benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-04 02:14:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge PR #1299 with current main for validation

Browse Source
This commit is contained in:
Sameh Khamis committed 2026-10-26 08:39:16 -07:00
2 parents a3a19883b4 5c3b66c263
commit aeee3652ce
39 files changed
+5274 -111
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

changelog.d/1299-voice-panel-settings.md
+3
View File
@@ -0,0 +1,3 @@
- Add a shared Voice Preview and Settings panel with browser-local preferences,
inactivity controls and accessible floating provider controls on compact layouts
(manjunath22466, #1299).
docs/CURRENT-STATE.md
+19
View File
@@ -571,6 +571,25 @@ replacement session; audio meters release failed initialization and reject revok
frames. Delayed action results and post-capture continuations cannot resume a
stopped conversation or send output into a replacement. See [voice ownership](VOICE-OWNERSHIP.md).
The opt-in voice panel shares Preview and Voice Settings, with browser-local
Preview and inactivity preferences. Voice-requested timeout changes apply directly
without a confirmation question; results distinguish a live change from a saved
browser preference. `set_voice_inactivity_timeout` accepts integer minutes from 1–60,
or 0 for None. Named `voice-settings` and `voice-preview` panel targets exit
Cockpit before opening the requested voice UI. `control_cockpit` also accepts
`voicePanel: "preview" | "settings"` for an explicit exit-and-show request.
Newer manual choices, Stop and provider replacement prevent an older operation
from revealing its UI. Ambiguous requests to open settings ask whether the user
means Voice Settings or Provider Settings before opening either panel. Named
Voice Settings applies directly; Provider Settings opens the existing local
provider dialog when available. Visual Presets targets the visual controls tray,
while Scenes targets saved scenes. Selecting Preview does not enable a disabled
Preview preference. Unverified speech captions use the localized Recent
speech label; ordinary user captions retain YOU. Caption attribution does not
control tool dispatch.
Automatic Voice Preview uses output from the active mounted provider, including late replies that may answer an older request. Manual panel choices remain until genuine local input rearms automatic presentation. Preview stays off when disabled, remains hidden in Cockpit, and retires on Stop or provider replacement. Display events do not grant command authority.
Contacts nearest-aircraft follow-ups retain the snapshot's distance basis even
after the camera moves. An explicit new scope replaces that basis. Provider
instructions require an analyst query for requested lists and use the requested
index.html
+1 -1
View File
@@ -26,7 +26,7 @@
with the sources: src/materialSymbolsSubset.test.mjs fails when a source
names a glyph this list is missing, because a missing glyph does not draw
a box — the ligature never forms and the element renders the literal word. -->
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20,400,0,0&icon_names=adjust,arrow_drop_down,arrow_forward,arrow_left,arrow_right,bolt,chevron_left,chevron_right,close,close_fullscreen,dark_mode,draw,east,flare,flight,layers,layers_clear,light_mode,local_fire_department,more_horiz,my_location,navigation,normal,on,open_in_full,public,radar,radio,right_panel_close,right_panel_open,rocket_launch,search,skip_next,skip_previous,upload_file,videocam,view_in_ar,visibility,visibility_off" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20,400,0,0&icon_names=adjust,arrow_drop_down,arrow_forward,arrow_left,arrow_right,bolt,chevron_left,chevron_right,close,close_fullscreen,dark_mode,draw,east,flare,flight,layers,layers_clear,light_mode,local_fire_department,more_horiz,my_location,navigation,normal,on,open_in_full,public,radar,radio,right_panel_close,right_panel_open,rocket_launch,search,settings,skip_next,skip_previous,upload_file,videocam,view_in_ar,visibility,visibility_off" rel="stylesheet" />
</head>
<body>
<!-- gev:template scene-chrome -->
scripts/package-boundaries.json
+12
View File
@@ -1656,6 +1656,12 @@
"src/voice/layerSummary.js",
"src/voice/voiceCost.js",
"src/voice/commands.js",
"src/voice/panel/actionObserver.js",
"src/voice/panel/host.js",
"src/voice/panel/inactivity.js",
"src/voice/panel/manualPanel.js",
"src/voice/panel/nativeActivity.js",
"src/voice/panel/preferences.js",
"src/voice/session.js",
"src/voice/sessionCommands.js",
"src/voice/realtimeSession.js",
@@ -2293,6 +2299,12 @@
"src/ui/visualPresets.js",
"src/ui/visualSettings.js",
"src/voice/commands.js",
"src/voice/panel/actionObserver.js",
"src/voice/panel/host.js",
"src/voice/panel/inactivity.js",
"src/voice/panel/manualPanel.js",
"src/voice/panel/nativeActivity.js",
"src/voice/panel/preferences.js",
"src/voice/control.js",
"src/voice/gevActions.js",
"src/agent/worldPacket.js",
server/providers/openai/instructions.js
+2 -3
View File
@@ -10,7 +10,7 @@ function realtimeInstructions(
'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
'Only control the app by calling the provided tools. Never invent tool names or arguments.',
'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
@@ -18,13 +18,12 @@ function realtimeInstructions(
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
'Do not mention disabled layers or stale selections.',
server/providers/openai/toolDescriptions.js
+13 -7
View File
@@ -44,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
},
select_nearest_aircraft: {
description:
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
$position: 1,
parameters: {
properties: {
@@ -111,12 +111,18 @@ export const ACTION_DESCRIPTIONS = {
},
},
set_panel_open: {
description: 'Open or close a GEV UI panel/dropdown.',
description:
'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
$position: 1,
},
set_voice_inactivity_timeout: {
description:
'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
$position: 1,
},
set_context_mode: {
description:
'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
$position: 1,
parameters: {
properties: {
@@ -129,23 +135,23 @@ export const ACTION_DESCRIPTIONS = {
},
control_cockpit: {
description:
'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
'Explicit Cockpit requests only: enter from a selected/tracked aircraft (establishes Contacts itself; no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
$position: 1,
parameters: {
properties: {
action: {
description:
'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
'previous/next (or prev) cycles nearby Cockpit contacts.',
$position: 2,
},
targetLayer: {
description:
'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
'Optional next/previous layer filter (e.g. military-only).',
$position: 2,
},
aircraftClass: {
description:
'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
'Optional next/previous aircraft class filter (e.g. helicopter).',
$position: 1,
},
},
src/firstRunExperience.test.mjs
+4 -3
View File
@@ -1052,11 +1052,12 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping
// Globe rotation adds one camera motion and its descriptions; Globe View
// framing/default guidance updates only move_camera and mode descriptions.
// Radio markup and the complete production-request comparison pin locality.
// Restore the base camera/Globe View wording with the globe-rotation append.
assert.equal(block.length, 30878, 'serialized tool schema length drifted');
// Panel adds timeout and named voice/provider targets; consolidated guidance
// preserves the mission mapping and globe-rotation arguments.
assert.equal(block.length, 31196, 'serialized tool schema length drifted');
assert.equal(
crypto.createHash('sha256').update(block).digest('hex'),
'7b95f0f4258896fe5c3316265a19f3a6602809d4e7e681ee7ecd8f2df514ef1f',
'56c91b309fb583c8737a9c3c1530553dcc8159ba2f3c5d4d2db99921f28d8f1e',
'the first-run missions must ride EXISTING tools: no schema edit, no cache bust',
);
const instructions = fs.readFileSync(
src/i18n/locales/es.json
+15 -1
View File
@@ -2952,5 +2952,19 @@
"Layers": "Capas",
"Search": "Buscar",
"More": "Más",
"Panels": "Paneles"
"Panels": "Paneles",
"Recent speech": "Habla reciente",
"Voice Settings": "Ajustes de voz",
"Preview": "Vista previa",
"Settings": "Ajustes",
"Close voice panel": "Cerrar panel de voz",
"Show Voice Preview": "Mostrar vista previa de voz",
"Voice inactivity timeout": "Tiempo de inactividad de voz",
"Custom": "Personalizado",
"Minutes": "Minutos",
"These preferences stay in this browser.": "Estas preferencias se guardan en este navegador.",
"Your choice applies now, but could not be saved.": "Tu elección se aplica ahora, pero no se pudo guardar.",
"Voice replies appear here.": "Las respuestas de voz aparecen aquí.",
"Preview is off.": "La vista previa está desactivada.",
"Automatic stop is unavailable for this voice connection.": "La parada automática no está disponible para esta conexión de voz."
}
src/i18n/locales/ru.json
+15 -1
View File
@@ -3019,5 +3019,19 @@
"Layers": "Слои",
"Search": "Поиск",
"More": "Ещё",
"Panels": "Панели"
"Panels": "Панели",
"Recent speech": "Недавняя речь",
"Voice Settings": "Настройки голоса",
"Preview": "Предпросмотр",
"Settings": "Настройки",
"Close voice panel": "Закрыть голосовую панель",
"Show Voice Preview": "Показывать предпросмотр голоса",
"Voice inactivity timeout": "Тайм-аут бездействия голоса",
"Custom": "Другое",
"Minutes": "Минуты",
"These preferences stay in this browser.": "Эти настройки сохраняются в этом браузере.",
"Your choice applies now, but could not be saved.": "Ваш выбор применён, но сохранить его не удалось.",
"Voice replies appear here.": "Здесь появляются голосовые ответы.",
"Preview is off.": "Предпросмотр отключён.",
"Automatic stop is unavailable for this voice connection.": "Автоматическая остановка недоступна для этого голосового подключения."
}
src/i18n/locales/zh-CN.json
+15 -1
View File
@@ -2818,5 +2818,19 @@
"Layers": "图层",
"Search": "搜索",
"More": "更多",
"Panels": "面板"
"Panels": "面板",
"Recent speech": "最近语音",
"Voice Settings": "语音设置",
"Preview": "预览",
"Settings": "设置",
"Close voice panel": "关闭语音面板",
"Show Voice Preview": "显示语音预览",
"Voice inactivity timeout": "语音空闲超时",
"Custom": "自定义",
"Minutes": "分钟",
"These preferences stay in this browser.": "这些偏好设置仅保存在此浏览器中。",
"Your choice applies now, but could not be saved.": "选择已生效，但无法保存。",
"Voice replies appear here.": "语音回复显示在此处。",
"Preview is off.": "预览已关闭。",
"Automatic stop is unavailable for this voice connection.": "此语音连接不支持自动停止。"
}
src/keySetup.js
+2
View File
@@ -395,6 +395,7 @@ export async function initKeySetup({
const destroy = () => {
if (disposed) return;
disposed = true;
delete root.dataset.ready;
unregisterUnsavedWork();
lifetime.abort();
signal?.removeEventListener('abort', destroy);
@@ -591,6 +592,7 @@ export async function initKeySetup({
chip.addEventListener('click', openDialog);
closeButton?.addEventListener('click', close);
applyButton?.addEventListener('click', onApply);
root.dataset.ready = 'true';
// Remove buttons are rendered per row; delegate so re-renders stay wired.
rowsHost?.addEventListener('click', (event) => {
const authButton = event.target?.closest?.(
src/radioMarkup.test.mjs
+18 -17
View File
@@ -1,5 +1,6 @@
import { readRealtimeSource } from './testSupport/readRealtimeSource.mjs';
import { GEV_REALTIME_TOOLS } from '../server/providers/openai/tools.js';
import { realtimeInstructions } from '../server/providers/openai/instructions.js';
import { readShellSource } from './testSupport/readShellSource.mjs';
import { expandApplicationHtml } from '../build/application-html.js';
import { readLayerSource } from './testSupport/readLayerSource.mjs';
@@ -25,13 +26,14 @@ const css = readStylesheet(new URL('../style.css', import.meta.url));
function realtimeTools() { return GEV_REALTIME_TOOLS; }
test('Realtime schema exposes the authoritative 30-tool inventory', () => {
test('Realtime schema exposes the authoritative 31-tool inventory', () => {
const tools = realtimeTools();
assert.equal(tools.length, 30);
assert.equal(tools.length, 31);
const names = tools.map((tool) => tool.name);
assert.equal(new Set(names).size, 30, 'tool names are unique');
assert.equal(new Set(names).size, 31, 'tool names are unique');
assert.ok(names.includes('set_context_mode'));
assert.ok(names.includes('control_cockpit'));
assert.ok(names.includes('set_voice_inactivity_timeout'));
assert.ok(names.includes('select_nearest_aircraft'));
assert.ok(names.includes('control_radio'));
assert.ok(names.includes('next_satellite_pass'));
@@ -77,13 +79,12 @@ test('the counting contract is stated in the Realtime instructions', () => {
});
test('Context panel opening stays distinct from Contacts activation', () => {
const start = voice.indexOf("'For requests to open, show, reveal, or focus a menu/panel");
assert.ok(start >= 0, 'panel-routing instruction is missing');
const text = voice.slice(start, voice.indexOf('\n', start));
assert.match(text, /"Open Context" means only set_panel_open/);
assert.match(text, /does not activate a Context sub-mode/);
assert.match(text, /"Open Contacts" means set_context_mode\{mode:"contacts"\}/);
assert.match(text, /expands the parent Context panel before activating Contacts/);
const text = realtimeInstructions().split('\n').find(line => line.startsWith('Panels:'));
assert.ok(text, 'panel-routing instruction is missing');
assert.match(text, /"Open Context" is only set_panel_open\{panelId:"global-context-panel",open:true\}/);
assert.match(text, /opening Context or selecting\/viewing an aircraft never enters them/);
assert.match(text, /"Open Contacts" is set_context_mode contacts/);
assert.match(text, /expands Context itself/);
});
test('nearest-aircraft selection stays out of Contacts and Cockpit', () => {
@@ -99,14 +100,14 @@ test('nearest-aircraft selection stays out of Contacts and Cockpit', () => {
assert.match(text, /nearest airborne result/);
assert.match(text, /healthy fallback feed is valid data/i);
assert.match(text, /Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity/);
assert.match(text, /SELECT\/FIND never implies Contacts or Cockpit/);
const routing = realtimeInstructions().split('\n').find(line => line.startsWith('Panels:'));
assert.match(routing, /Contacts, Space Missions and Cockpit need explicit naming/);
assert.match(text, /set_context_mode, or control_cockpit/);
const byName = new Map(realtimeTools().map((tool) => [tool.name, tool]));
assert.match(byName.get('set_context_mode').description, /explicitly requests/);
assert.match(byName.get('set_context_mode').description, /selecting an aircraft does not imply Context/i);
assert.match(byName.get('control_cockpit').description, /explicitly requests Cockpit/);
assert.match(byName.get('control_cockpit').description, /must not enter Cockpit/);
assert.match(byName.get('set_context_mode').description, /only when explicitly requested/);
assert.match(routing, /selecting\/viewing an aircraft never enters them/);
assert.match(byName.get('control_cockpit').description, /Explicit Cockpit requests only/);
assert.equal(
byName.get('fly_to_location').parameters.properties.waitForArrival.type,
'boolean',
@@ -117,7 +118,6 @@ test('nearest-aircraft selection stays out of Contacts and Cockpit', () => {
assert.match(nearest.description, /Atomically/);
assert.match(nearest.description, /exclude on-ground records/);
assert.match(nearest.description, /fallback feeds remain usable/);
assert.match(nearest.description, /does not open Contacts or Cockpit/);
});
test('the two Context/Cockpit tools pin their enums and required arguments', () => {
@@ -153,7 +153,7 @@ test('the edited existing tools changed exactly as intended', () => {
const panel = byName.get('set_panel_open');
assert.deepEqual(
panel.parameters.properties.panelId.enum,
['data-panel', 'location-bar', 'control-panel', 'cctv-panel', 'radio-panel', 'scene-panel', 'pp-toggles', 'global-context-panel', 'weather-panel', 'recent-imagery-panel'],
['data-panel', 'location-bar', 'control-panel', 'cctv-panel', 'radio-panel', 'scene-panel', 'pp-toggles', 'global-context-panel', 'weather-panel', 'recent-imagery-panel', 'voice-settings', 'voice-preview', 'provider-settings'],
);
assert.deepEqual(panel.parameters.required, ['panelId', 'open']);
@@ -183,6 +183,7 @@ test('no unchanged Realtime tool definition drifts silently', () => {
'set_context_mode',
'control_cockpit',
'set_panel_open',
'set_voice_inactivity_timeout',
'get_current_view_state',
'fly_to_location',
'select_nearest_aircraft',
src/standalone/tools.js
+1
View File
@@ -19,6 +19,7 @@ export function createStandaloneTools(options) {
...options,
voice: {
supportedProviders: ['openai', 'gemini'],
panel: true,
toolCatalog: loadToolCatalog,
...options?.voice,
},
src/ui/styles/voice-card.css
+246
View File
@@ -387,3 +387,249 @@ body.recording-mode .gev-pointer-reticle {
animation: none;
}
}
/* Opt-in shared panel: header stays fixed while only the body scrolls. */
#gev-voice-control .gev-voice-panel {
position: absolute;
bottom: calc(100% + 0.7rem);
left: 50%;
z-index: 10;
display: flex;
flex-direction: column;
width: clamp(18rem, 26vw, 22rem);
max-width: calc(100vw - 1.5rem);
max-height: min(
28rem,
max(7.5rem, calc(100dvh - 12rem - var(--gev-provider-lift, 0px)))
);
box-sizing: border-box;
padding: 0.6rem 0.7rem 0.65rem;
color: rgba(223, 245, 250, 0.92);
background: rgba(7, 22, 31, 0.96);
border: 1px solid rgba(54, 220, 255, 0.34);
border-radius: 0.6rem;
box-shadow:
0 0.85rem 2rem rgba(0, 0, 0, 0.58),
0 0 1.2rem rgba(0, 212, 255, 0.08);
backdrop-filter: blur(24px) saturate(1.3);
-webkit-backdrop-filter: blur(24px) saturate(1.3);
color-scheme: dark;
font: 0.75rem/1.45 var(--font-sans);
text-align: left;
transform: translateX(-50%);
}
:root[data-ui-theme='cyber'] #gev-voice-control .gev-voice-panel {
background-color: var(--cyber-panel-surface);
background-image: var(--cyber-panel-surface-detail);
border-color: var(--glass-border);
box-shadow: var(--cyber-panel-shadow);
}
#gev-voice-control .gev-voice-panel[hidden],
#gev-voice-control .gev-voice-panel [hidden] {
display: none;
}
#gev-voice-control .gev-voice-panel-header {
display: flex;
flex: 0 0 auto;
align-items: center;
justify-content: space-between;
gap: 0.35rem;
padding: 0 0 0.5rem;
border-bottom: 1px solid var(--glass-border);
}
.gev-voice-panel-title {
color: var(--voice-panel-accent, var(--accent));
font: 600 0.65rem/1 var(--font-mono);
letter-spacing: 0.14em;
}
.gev-voice-panel-tabs {
margin-left: auto;
}
#gev-voice-control .gev-voice-panel-header .gev-voice-panel-tab {
display: inline-flex;
align-items: center;
justify-content: center;
width: 2rem;
padding: 0;
}
#gev-voice-control .gev-voice-panel-tab-icon {
font-size: 18px;
}
.gev-voice-panel-header [role='tablist'] {
display: flex;
flex-wrap: wrap;
gap: 0.2rem;
min-width: 0;
}
#gev-voice-control
.gev-voice-panel
:is(.gev-voice-panel-header button, select, input[type='number']) {
min-height: 2rem;
box-sizing: border-box;
color: var(--text-primary);
background: rgba(255, 255, 255, 0.04);
border: 1px solid var(--glass-border);
border-radius: 0.3rem;
font: inherit;
}
#gev-voice-control .gev-voice-panel .gev-voice-panel-header button {
padding: 0 0.6rem;
color: var(--text-secondary);
font-family: var(--font-mono);
font-size: 0.68rem;
letter-spacing: 0.04em;
line-height: 1;
cursor: pointer;
}
#gev-voice-control #gev-voice-settings-close {
width: 2rem;
padding: 0;
font-size: 1rem;
}
#gev-voice-control .gev-voice-panel-header button:hover {
color: var(--text-primary);
border-color: var(--voice-panel-accent, var(--accent));
}
#gev-voice-control .gev-voice-panel [role='tab'][aria-selected='true'] {
color: var(--voice-panel-accent, var(--accent));
border-color: currentColor;
box-shadow: 0 -2px 0 currentColor inset;
}
#gev-voice-control .gev-voice-panel [role='tab']:focus,
#gev-voice-control .gev-voice-panel :focus-visible {
outline: 2px solid var(--voice-panel-accent, var(--accent));
outline-offset: 2px;
}
.gev-voice-panel-body {
min-height: 0;
overflow: auto;
overscroll-behavior: contain;
scrollbar-width: thin;
padding: 0.55rem 0.75rem 3px 3px;
line-height: 1.45;
overflow-wrap: anywhere;
}
.gev-voice-panel-body .gev-voice-card-who {
overflow-wrap: break-word;
}
.gev-voice-panel-body p {
margin: 0;
}
.gev-voice-preference {
display: flex;
align-items: center;
gap: 0.5rem;
margin: 0;
color: var(--text-primary);
}
.gev-voice-preference-toggle {
flex-direction: row-reverse;
justify-content: space-between;
min-height: 2.75rem;
font-weight: 600;
cursor: pointer;
}
.gev-voice-preference-toggle input {
flex: 0 0 auto;
width: 18px;
height: 18px;
margin: 0;
accent-color: var(--voice-panel-accent, var(--accent));
}
.gev-voice-setting-title {
margin-bottom: 0.4rem;
font-weight: 600;
}
.gev-voice-timeout-group {
margin-top: 0.65rem;
padding-top: 0.65rem;
border-top: 1px solid var(--glass-border);
}
.gev-voice-timeout-controls {
display: flex;
align-items: center;
gap: 0.4rem;
}
#gev-voice-timeout-select {
flex: 1 1 auto;
min-width: 0;
padding: 0 0.45rem;
}
#gev-voice-timeout-input {
flex: 0 0 4.5rem;
width: 4.5rem;
padding: 0 0.45rem;
font-variant-numeric: tabular-nums;
}
.gev-voice-timeout-unit {
order: 1;
color: var(--text-dim);
font-size: 0.63rem;
}
.gev-voice-timeout-controls:has(#gev-voice-timeout-input[hidden])
.gev-voice-timeout-unit {
display: none;
}
.gev-voice-settings-note {
color: rgba(190, 221, 230, 0.72);
font-size: 0.63rem;
line-height: 1.45;
}
.gev-voice-settings-note:not(:empty) {
margin-top: 0.25rem;
}
#gev-voice-control .gev-voice-panel .gev-voice-card {
position: static;
width: 100%;
max-height: none;
overflow: visible;
padding: 0;
border: 0;
background: transparent;
box-shadow: none;
transform: none;
animation: none;
}
#command-dock > #gev-voice-control:has(.gev-voice-panel:not([hidden])) {
z-index: 101;
}
#gev-voice-control:has(.gev-voice-panel:not([hidden])) .gev-voice-help-tray {
display: none;
}
#command-dock #gev-voice-control[data-provider-choice='true'] .gev-voice-panel {
bottom: min(
calc(100% + 4.8rem + var(--gev-provider-lift)),
calc(100dvh - 7.75rem)
);
}
#command-dock.dock-has-pinned-tray
#gev-voice-control:not([data-provider-choice='true'])
.gev-voice-panel {
bottom: min(
calc(100% + 1.7rem + var(--dock-pinned-stack-height)),
calc(100dvh - 18rem)
);
}
@media (pointer: coarse) {
#gev-voice-control
.gev-voice-panel
:is(
.gev-voice-panel-header button,
select,
input[type='number'],
.gev-voice-card-dismiss
) {
min-height: 44px;
}
#gev-voice-control
.gev-voice-panel
:is(.gev-voice-panel-header button, .gev-voice-card-dismiss) {
min-width: 44px;
}
}
/* A connection error remains reachable even while the manual panel is open. */
#gev-voice-control:has(.gev-voice-panel) .gev-voice-error-tray {
z-index: 12;
}
src/ui/styles/voice-cost.css
+324 -1
View File
@@ -111,7 +111,7 @@ html:not([lang='en']) #command-dock .gev-voice-cost {
padding-left: 0.1rem;
}
html:not([lang='en']) #command-dock .gev-voice-tier-btn {
html:not([lang='en']) #command-dock .gev-voice-heading .gev-voice-tier-btn {
min-width: 0;
padding: 0 0.2rem;
}
@@ -232,3 +232,326 @@ html:not([lang='en'])
pointer-events: auto;
}
}
/* The opt-in provider and gear share a discoverable row at the existing anchor. */
#gev-voice-control:has(> .gev-voice-settings-button) {
--gev-voice-settings-gap: 0.75rem;
--gev-voice-tray-clearance: 0px;
}
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field:not([hidden]) {
bottom: calc(
100% + var(--gev-provider-lift, 0px) + var(--gev-voice-settings-gap)
);
display: flex;
align-items: center;
gap: 6px;
padding: 6px 46px 6px 6px;
border-color: rgba(123, 189, 211, 0.28);
background: rgba(9, 18, 27, 0.88);
box-shadow: 0 8px 24px rgba(0, 0, 0, 0.46);
color-scheme: dark;
opacity: 1;
pointer-events: auto;
}
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field
> span {
position: absolute;
width: 1px;
height: 1px;
overflow: hidden;
clip-path: inset(50%);
white-space: nowrap;
}
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field
select {
min-height: 34px;
font-size: 0.7rem;
}
#gev-voice-control .gev-voice-settings-button {
position: absolute;
top: 1rem;
right: 0.25rem;
z-index: 11;
display: inline-flex;
align-items: center;
justify-content: center;
width: 34px;
height: 34px;
min-width: 34px;
min-height: 34px;
padding: 0;
color: rgba(190, 221, 230, 0.72);
background: rgba(9, 18, 27, 0.92);
border: 1px solid var(--glass-border);
border-radius: 0.3rem;
font-size: 18px;
line-height: 1;
cursor: pointer;
}
#gev-voice-control[data-provider-choice='true']:has(
.gev-voice-provider-field:not([hidden])
)
.gev-voice-settings-button {
top: auto;
right: 7px;
bottom: calc(
100% + var(--gev-provider-lift, 0px) + var(--gev-voice-settings-gap) + 7px
);
}
#gev-voice-control
.gev-voice-settings-button:is(:hover, [aria-expanded='true']) {
color: var(--voice-panel-accent, var(--accent));
border-color: currentColor;
background: rgba(0, 212, 255, 0.12);
}
#gev-voice-control .gev-voice-settings-button:focus-visible {
outline: 2px solid var(--voice-panel-accent, var(--accent));
outline-offset: 2px;
}
:root[data-ui-theme='cyber']
#gev-voice-control:has(> .gev-voice-settings-button) {
--voice-panel-accent: var(--cyber-red-bright);
}
:root[data-ui-theme='cyber']
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field {
background: var(--cyber-panel-surface);
border-color: var(--glass-border);
}
@media (pointer: coarse) {
#gev-voice-control .gev-voice-settings-button {
width: 44px;
height: 44px;
min-width: 44px;
min-height: 44px;
}
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field:not([hidden]) {
padding-right: 56px;
}
#gev-voice-control:has(> .gev-voice-settings-button)
.gev-voice-provider-field
select {
min-height: 44px;
}
}
/* Keep the row and popup on the same lift, with a small gap above pinned
trays. On very short screens the shared lift still yields to the viewport. */
#command-dock.dock-has-pinned-tray
#gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
) {
--gev-voice-settings-gap: 0.375rem;
--gev-provider-lift: min(
calc(var(--dock-pinned-stack-height) + var(--gev-voice-tray-clearance)),
max(0px, calc(100dvh - 20rem))
);
}
/* Match the attribution-aware tray offsets without moving the trays. The
popup can scroll before the shared viewport budget caps its lift. */
@media (max-width: 900px) {
#gev-voice-control:has(> .gev-voice-settings-button) {
--gev-voice-tray-clearance: calc(3rem - 1.2rem);
}
}
@media (max-width: 720px) {
#gev-voice-control:has(> .gev-voice-settings-button) {
--gev-voice-tray-clearance: calc(2vh + 2.5rem - 1.2rem);
}
}
/* Short touch landscapes have room beside the dock, not above its pinned
stack. Keep the same chooser and gear together in that empty lower gutter;
the open popup remains a bounded, scrollable overlay above the microphone. */
@media (pointer: coarse) and (orientation: landscape) and (min-width: 480px) and (max-height: 500px) {
#command-dock
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
),
#command-dock.dock-has-pinned-tray
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
) {
--gev-provider-lift: 0px;
--gev-landscape-row-width: clamp(
108px,
calc((100vw - var(--dock-unified-width)) / 2 - 0.875rem),
calc(100% - 2px)
);
}
#command-dock
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
)
> .gev-voice-provider-field:not([hidden]) {
left: calc(50% - 50vw + 0.5rem);
right: auto;
bottom: 0.375rem;
width: var(--gev-landscape-row-width);
box-sizing: border-box;
}
#command-dock
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
)
> .gev-voice-settings-button {
left: calc(50% - 50vw + 0.5rem + var(--gev-landscape-row-width) - 51px);
right: auto;
top: auto;
bottom: calc(0.375rem + 7px);
}
}
/* The revealed row clears pinned wings without making the Voice column's
noninteractive gaps cover their controls. Keep hit surfaces stable so hover
promotion cannot oscillate when the pointer crosses root padding. */
#command-dock
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
) {
pointer-events: none;
}
#command-dock
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
)
> :is(
#gev-voice-button,
.gev-voice-heading,
.gev-voice-readout,
.gev-voice-provider-field,
.gev-voice-settings-button,
.gev-voice-panel,
.gev-voice-error-tray
) {
pointer-events: auto;
}
@media (hover: hover) and (pointer: fine) {
#command-dock.dock-has-pinned-tray
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
):is(
:hover,
[data-row-grace],
:has(
:focus-visible,
#gev-voice-provider:focus,
.gev-voice-panel[data-view='settings']:not([hidden])
)
) {
z-index: 101;
}
}
@media (hover: none), (pointer: coarse) {
#command-dock.dock-has-pinned-tray
> #gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
) {
z-index: 101;
}
}
/* Pointer grace crosses pinned trays without covering their controls. Keyboard
focus, the provider select and an open panel hold the shared row visible. */
@media (hover: hover) and (pointer: fine) {
#command-dock
#gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
)
:is(.gev-voice-provider-field, .gev-voice-settings-button) {
transition:
opacity 160ms ease,
transform 180ms ease;
}
#command-dock
#gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
):not(:hover):not([data-row-grace]):not(
:has(
:focus-visible,
#gev-voice-provider:focus,
.gev-voice-panel[data-view='settings']:not([hidden])
)
)
:is(.gev-voice-provider-field, .gev-voice-settings-button) {
opacity: 0;
pointer-events: none;
transform: translateY(8px);
}
#command-dock:not(.dock-has-pinned-tray)
#gev-voice-control[data-provider-choice='true']:has(
> .gev-voice-settings-button
)
.gev-voice-provider-field::after {
content: '';
position: absolute;
top: 100%;
right: 0;
left: 0;
height: calc(var(--gev-voice-settings-gap) + 8px);
}
}
/* Existing tier and estimate controls are readable Settings content in panel mode. */
.gev-voice-cost-settings {
display: grid;
gap: 0.55rem;
margin-top: 0.75rem;
padding-top: 0.7rem;
border-top: 1px solid var(--glass-border);
}
.gev-voice-cost-settings:has(#gev-voice-tier[hidden]):has(
#gev-voice-cost-value[hidden]
),
.gev-voice-cost-setting-row:has(> [hidden]) {
display: none;
}
.gev-voice-cost-setting-row {
display: flex;
align-items: center;
justify-content: space-between;
gap: 1rem;
min-height: 2rem;
color: var(--text-primary);
font-size: 0.68rem;
}
.gev-voice-cost-setting-row > span {
min-width: 0;
}
#gev-voice-control .gev-voice-cost-settings .gev-voice-tier-btn {
flex: 0 0 auto;
min-height: 2rem;
min-width: 3.25rem;
padding: 0.3rem 0.55rem;
font-size: 0.67rem;
}
.gev-voice-cost-settings .gev-voice-cost-value {
flex: 0 0 auto;
font-size: 0.75rem;
}
.gev-voice-cost-settings .gev-voice-cost-value[data-level='ok'] {
color: var(--voice-panel-accent, var(--accent));
}
#gev-voice-control .gev-voice-settings-icon {
font-size: 18px;
line-height: 1;
}
@media (pointer: coarse) {
#gev-voice-control .gev-voice-cost-settings .gev-voice-tier-btn {
min-height: 44px;
min-width: 44px;
}
}
@media (prefers-reduced-motion: reduce) {
#command-dock
#gev-voice-control:has(> .gev-voice-settings-button)
:is(.gev-voice-provider-field, .gev-voice-settings-button) {
transition: none;
}
}
src/voice/actionSchemas.js
+13
View File
@@ -178,6 +178,9 @@ const schemas = [
'global-context-panel',
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
],
},
open: {
@@ -187,6 +190,15 @@ const schemas = [
required: ['panelId', 'open'],
},
},
{
name: 'set_voice_inactivity_timeout',
parameters: {
type: 'object',
additionalProperties: false,
properties: { minutes: { type: 'integer', minimum: 0, maximum: 60 } },
required: ['minutes'],
},
},
{
name: 'set_context_mode',
parameters: {
@@ -223,6 +235,7 @@ const schemas = [
aircraftClass: {
type: 'string',
},
voicePanel: { type: 'string', enum: ['preview', 'settings'] },
},
required: ['action'],
},
src/voice/actionSchemas.test.mjs
+30 -5
View File
@@ -32,7 +32,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
.digest('hex');
assert.equal(
digest,
// Re-derived for the voice layer manifest (generated layer enums and
// Re-derived for Provider Settings and explicit Visual Presets/Scenes routing,
// plus the deliberate direct Panel tool/target additions and voice layer manifest (generated layer enums and
// aliases), point-and-ask (pointer sentinels, referent args) and the
// consolidated tool wording (each policy stated once); model-facing tools
// omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves
@@ -41,8 +42,9 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// Air quality (ECCC AQHI) extends the manifest.
// Globe rotation adds one motion; visual keyhole fit and explicit
// reset guidance/defaults update only move_camera and mode descriptions.
// Restore base descriptions; append only globe-rotation defaults and fit guidance.
'967b5a043b812e2f8c1e190fc571cf12b960c9d4486df736902221aab4d99cf6',
// Panel adds named voice targets and timeout; shared Panel/Cockpit guidance
// is consolidated without changing either feature's arguments.
'b2cd2decb77fb3f8f0ab65e7907c221580e21ee3e7e77581f1b52bf62126ae86',
);
});
@@ -100,12 +102,18 @@ test('metadata cannot add tools, fields, types or enum values', () => {
test('all legacy action arguments are byte-identical after removing the deliberate additions', () => {
const legacy = structuredClone(GEV_ACTION_SCHEMAS).filter(
(tool) => !['next_satellite_pass', 'set_cyber_sonar'].includes(tool.name),
(tool) =>
![
'next_satellite_pass',
'set_cyber_sonar',
'set_voice_inactivity_timeout',
].includes(tool.name),
);
// Layer enums are generated from the voice layer manifest and pinned by
// layerManifest.test.mjs; the two shipped right-rail panels are additive.
const property = (name) =>
legacy.find((tool) => tool.name === name).parameters.properties;
delete property('control_cockpit').voicePanel;
delete property('set_layer_visibility').layerId.enum;
delete property('show_data_layers_menu').layerId.enum;
delete property('get_entity_context').layerId.enum;
@@ -123,7 +131,14 @@ test('all legacy action arguments are byte-identical after removing the delibera
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
const panels = property('set_panel_open').panelId;
panels.enum = panels.enum.filter(
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
(id) =>
![
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
].includes(id),
);
for (const tool of legacy) {
// Point-and-ask adds `referent` arguments and 'pointer' enum values.
@@ -228,3 +243,13 @@ test('globe reset guidance requires a separate explicit command after refusal',
assert.match(paragraph, /bare yes\/ok/);
assert.match(paragraph, /Do not ask a question/);
});
test('direct voice timeout schema accepts None and exact integers, rejects malformed duration', () => {
const tool = GEV_REALTIME_TOOLS.find(
(tool) => tool.name === 'set_voice_inactivity_timeout',
);
for (const minutes of [0, 1, 5, 60])
assert.equal(validateArgs(tool.parameters, { minutes }).length, 0);
for (const minutes of [-1, 61, 1.5, '5'])
assert.ok(validateArgs(tool.parameters, { minutes }).length > 0);
});
src/voice/commands.js
+39 -6
View File
@@ -2,17 +2,50 @@ import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createRealtimeSession } from './realtimeSession.js';
import { createGeminiSession } from './geminiSession.js';
import { createProviderCommands } from './providerCommands.js';
import { createVoicePanelHost } from './panel/host.js';
/** Default composition; callers may supply another session adapter factory. */
export function createVoiceCommands(options = {}) {
const panelHost =
options.panel === true
? createVoicePanelHost({
storage: options.panelStorage,
clock: options.panelClock,
readCockpitState: options.styleManager?.getCockpitState
? () => options.styleManager.getCockpitState()
: undefined,
})
: null;
const bind = (settings) =>
bindVoiceCommands({
...settings,
...(panelHost
? {
panelHost,
nativePanelProvider: settings.createController
? null
: settings.createSession === createRealtimeSession
? 'openai'
: settings.createSession === createGeminiSession
? 'gemini'
: null,
}
: {}),
});
let commands;
// Generic hosts retain the existing default; standalone opts in explicitly.
if (!options.createSession && options.supportedProviders !== undefined)
return createProviderCommands(options, {
bind: bindVoiceCommands,
commands = createProviderCommands(options, {
bind,
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
});
return bindVoiceCommands({
createSession: createRealtimeSession,
...options,
});
else commands = bind({ createSession: createRealtimeSession, ...options });
if (panelHost) {
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => panelHost.destroy(), {
once: true,
});
if (lifetime?.aborted) panelHost.destroy();
}
return commands;
}
src/voice/control.js
+73 -4
View File
@@ -27,7 +27,7 @@ export function voiceStatusLabel(status) {
}
/** Build the voice control independently of its connection backend. */
export function createVoiceControl({ reset = false } = {}) {
export function createVoiceControl({ reset = false, panel = false } = {}) {
let root = document.getElementById('gev-voice-control');
if (root && reset) {
root.remove();
@@ -43,13 +43,18 @@ export function createVoiceControl({ reset = false } = {}) {
<span>${t('Voice provider')}</span>
<select id="gev-voice-provider" aria-label="${t('Voice provider')}" title="${t('Changing provider reconnects voice if it is on.')}"></select>
</label>
${panel ? `<button id="gev-voice-settings-button" class="gev-voice-settings-button" type="button" aria-label="${t('Voice Settings')}" aria-controls="gev-voice-panel" aria-expanded="false"><span class="material-symbols-outlined gev-voice-settings-icon" aria-hidden="true">settings</span></button>` : ''}
<div class="gev-voice-heading">
<div class="gev-voice-kicker">${t('AI AGENT')}</div>
<div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
<div class="gev-voice-cost">
${
panel
? ''
: `<div class="gev-voice-cost">
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
</div>
</div>`
}
</div>
<button id="gev-voice-button" type="button" aria-label="${t('Voice control — activate to toggle voice; hold Space to speak')}" aria-describedby="gev-voice-help">
<span class="gev-mic-orbit"><img src="/mic.svg" alt="" /></span>
@@ -65,13 +70,29 @@ export function createVoiceControl({ reset = false } = {}) {
<span class="gev-voice-help-kicker">${t('VOICE CONTROL')}</span>
<span class="gev-voice-help-detail">${t('Hold Space to speak · tap Space to activate focused controls')}</span>
</div>
${
panel
? `<section id="gev-voice-panel" class="gev-voice-panel" hidden aria-label="${t('Voice Settings')}">
<div class="gev-voice-panel-header">
<span class="gev-voice-panel-title">${t('VOICE')}</span>
<div class="gev-voice-panel-tabs" role="tablist" aria-label="${t('Voice Settings')}">
<button id="gev-voice-preview-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Preview')}" title="${t('Preview')}" aria-selected="true" aria-controls="gev-voice-preview-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">visibility</span></button>
<button id="gev-voice-settings-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Settings')}" title="${t('Settings')}" aria-selected="false" tabindex="-1" aria-controls="gev-voice-settings-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">settings</span></button>
</div>
<button id="gev-voice-settings-close" type="button" aria-label="${t('Close voice panel')}">×</button>
</div>
<div class="gev-voice-panel-body">
<section id="gev-voice-preview-panel" role="tabpanel" aria-labelledby="gev-voice-preview-tab">
<p id="gev-voice-preview-empty">${t('Voice replies appear here.')}</p>`
: ''
}
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="${t('Voice assistant')}">
<div class="gev-voice-card-header">
<span class="gev-voice-card-kicker">${t('VOICE')}</span>
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="${t('Dismiss voice card')}">×</button>
</div>
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span class="gev-voice-card-who">${t('YOU')}</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span id="gev-voice-card-user-who" class="gev-voice-card-who">${t('YOU')}</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">${t('THIS')}</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">${ASSISTANT_NAME}</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="${t('Plan')}" hidden></ol>
@@ -87,6 +108,40 @@ export function createVoiceControl({ reset = false } = {}) {
</div>
<div id="gev-voice-card-live" class="gev-voice-card-sr" aria-live="polite" aria-atomic="true"></div>
</section>
${
panel
? `</section>
<section id="gev-voice-settings-panel" role="tabpanel" aria-labelledby="gev-voice-settings-tab" hidden>
<label class="gev-voice-preference gev-voice-preference-toggle"><input id="gev-voice-preview-enabled" type="checkbox" checked aria-describedby="gev-voice-preview-note" /><span>${t('Show Voice Preview')}</span></label>
<p id="gev-voice-preview-note" class="gev-voice-settings-note"></p>
<div class="gev-voice-timeout-group">
<label class="gev-voice-preference gev-voice-setting-title" for="gev-voice-timeout-select">${t('Voice inactivity timeout')}</label>
<div class="gev-voice-timeout-controls">
<select id="gev-voice-timeout-select" aria-describedby="gev-voice-timeout-note gev-voice-timeout-status">
<option value="3">3 ${t('Minutes')}</option><option value="5" selected>5 ${t('Minutes')}</option><option value="10">10 ${t('Minutes')}</option><option value="15">15 ${t('Minutes')}</option>
<option value="custom">${t('Custom')}</option><option value="0">${t('None')}</option>
</select>
<label class="gev-voice-timeout-unit" for="gev-voice-timeout-input">${t('Minutes')}</label>
<input id="gev-voice-timeout-input" type="number" min="1" max="60" step="1" value="5" aria-describedby="gev-voice-timeout-note" />
</div>
<p id="gev-voice-timeout-note" class="gev-voice-settings-note">${t('These preferences stay in this browser.')}</p>
<p id="gev-voice-timeout-status" class="gev-voice-settings-note" role="status" aria-live="polite"></p>
</div>
<div class="gev-voice-cost-settings">
<div class="gev-voice-cost-setting-row">
<span>${t('Voice model tier — applies next session')}</span>
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
</div>
<div class="gev-voice-cost-setting-row">
<span>${t('Estimated session cost')}</span>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
</div>
</div>
</section>
</div>
</section>`
: ''
}
<div class="gev-voice-error-tray" role="alert" aria-live="assertive">
<div class="gev-voice-error-header">
<span>${t('VOICE SYSTEM ERROR')}</span>
@@ -114,6 +169,20 @@ export function createVoiceControl({ reset = false } = {}) {
}
return {
root,
voicePanel: root.querySelector('#gev-voice-panel'),
previewPanel: root.querySelector('#gev-voice-preview-panel'),
voiceSettingsPanel: root.querySelector('#gev-voice-settings-panel'),
previewTab: root.querySelector('#gev-voice-preview-tab'),
settingsTab: root.querySelector('#gev-voice-settings-tab'),
voiceSettingsButton: root.querySelector('#gev-voice-settings-button'),
voiceSettingsClose: root.querySelector('#gev-voice-settings-close'),
previewEmpty: root.querySelector('#gev-voice-preview-empty'),
voicePreviewInput: root.querySelector('#gev-voice-preview-enabled'),
voicePreviewNote: root.querySelector('#gev-voice-preview-note'),
voiceTimeoutSelect: root.querySelector('#gev-voice-timeout-select'),
voiceTimeoutInput: root.querySelector('#gev-voice-timeout-input'),
voiceTimeoutNote: root.querySelector('#gev-voice-timeout-note'),
voiceTimeoutStatus: root.querySelector('#gev-voice-timeout-status'),
providerField: root.querySelector('#gev-voice-provider-field'),
providerSelect: root.querySelector('#gev-voice-provider'),
button: root.querySelector('#gev-voice-button'),
src/voice/gevActions.js
+188
View File
@@ -74,8 +74,17 @@ const PANEL_ALIASES = new Map([
['styles', 'control-panel'],
['filters', 'control-panel'],
['visual styles', 'control-panel'],
['visual preset', 'control-panel'],
['visual presets', 'control-panel'],
['provider settings', 'provider-settings'],
['cctv', 'cctv-panel'],
['cameras', 'cctv-panel'],
['voice settings', 'voice-settings'],
['voice-settings', 'voice-settings'],
['voice preview', 'voice-preview'],
['voice-preview', 'voice-preview'],
['voice control', 'voice-preview'],
['voice controls', 'voice-preview'],
['radio', 'radio-panel'],
['internet radio', 'radio-panel'],
['radio stations', 'radio-panel'],
@@ -110,6 +119,7 @@ const PANEL_IDS = new Set([
'pp-toggles',
'weather-panel',
'recent-imagery-panel',
'provider-settings',
]);
const CONTEXT_MODE_ALIASES = new Map([
['off', 'off'],
@@ -243,6 +253,7 @@ export function createGevActionRunner({
searchNavigation = searchAndFlyTo,
speechBuilders = null,
deixis = null,
readVoicePanelOperation = null,
}) {
// Voice enable times and analyst follow-up memory belong to this runner.
// Speech builders turn results into say/display envelopes; callers extend
@@ -268,6 +279,81 @@ export function createGevActionRunner({
});
// Nested calls (select_nearest_aircraft composing layer + flight) use the
// raw runner; only the outer call gets a voice envelope.
const voiceUnavailable = (action) => ({
ok: false,
action,
applied: false,
unavailable: true,
error: 'Voice panel controls are unavailable in this host.',
say: 'Voice panel controls are unavailable in this host.',
});
async function showVoicePanel(
view,
open,
options,
operation = readVoicePanelOperation?.(),
) {
const current = () =>
!options.signal?.aborted &&
(!options.isCurrent || options.isCurrent()) &&
operation?.isCurrent();
if (!operation) return voiceUnavailable('set_panel_open');
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
};
let exit = null;
if (open) {
if (typeof styleManager?.getCockpitState !== 'function')
return voiceUnavailable('set_panel_open');
if (styleManager.getCockpitState()?.active) {
if (typeof styleManager.controlCockpit !== 'function')
return voiceUnavailable('set_panel_open');
exit = await styleManager.controlCockpit('exit');
if (styleManager.getCockpitState()?.active)
return {
ok: false,
action: 'set_panel_open',
applied: false,
exit,
error: exit?.error || 'Cockpit did not exit.',
};
}
}
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
exit,
cockpitExited: Boolean(exit && !styleManager.getCockpitState()?.active),
say:
exit && !styleManager.getCockpitState()?.active
? 'Cockpit exited; the voice panel was left as you last set it.'
: 'The voice panel was left as you last set it.',
};
const result = operation.show(view, open);
return {
...result,
action: 'set_panel_open',
panelId: view === 'settings' ? 'voice-settings' : 'voice-preview',
exit,
state: styleManager?.getCockpitState?.() ?? null,
say: !result.ok
? 'The voice panel was not changed.'
: result.settingsOpen
? 'Voice Settings is open.'
: result.panelOpen
? result.previewEnabled
? 'Voice Preview is open.'
: 'Voice control is open; Preview remains off.'
: 'The voice panel is closed.',
};
}
const runGevAction = async function runGevAction(
name,
rawArgs = {},
@@ -639,6 +725,25 @@ export function createGevActionRunner({
return { ok: true, action: 'set_visual_style', style };
}
if (name === 'set_voice_inactivity_timeout') {
const operation = readVoicePanelOperation?.();
if (!operation) return voiceUnavailable(name);
if (!current() || !operation.isCurrent())
return { ok: false, action: name, applied: false, cancelled: true };
const result = operation.setMinutes(args.minutes);
const duration =
result.minutes === 0 ? 'disabled' : `${result.minutes} minutes`;
return {
...result,
action: name,
say: !result.applied
? 'Voice inactivity was not changed.'
: result.persisted
? `Voice inactivity is ${duration}, saved in this browser.`
: `Voice inactivity is ${duration} for now, but could not be saved in this browser.`,
};
}
if (name === 'set_panel_open') {
const panelId = normalizePanelId(args.panelId || args.panel);
if (!panelId)
@@ -646,6 +751,70 @@ export function createGevActionRunner({
`Unknown panel: ${args.panelId || args.panel || 'missing'}`,
);
const open = args.open !== false;
if (panelId === 'voice-settings' || panelId === 'voice-preview')
return showVoicePanel(
panelId === 'voice-settings' ? 'settings' : 'preview',
open,
runOptions,
);
if (panelId === 'provider-settings') {
if (!current())
return {
ok: false,
action: name,
panelId,
applied: false,
cancelled: true,
};
const doc = globalThis.document;
const panel = doc?.getElementById?.('key-setup');
const button = open
? doc?.getElementById?.('key-setup-chip')
: panel?.querySelector?.('[data-key-setup-close]');
const concealed =
open &&
[
'ui-clean-view',
'recording-mode',
'cockpit-mode',
'scene-playback-mode',
].some((mode) => doc?.body?.classList?.contains(mode));
if (
concealed ||
panel?.dataset?.ready !== 'true' ||
typeof button?.click !== 'function'
) {
const error = concealed
? 'Provider Settings is hidden in the current view. Leave that view before opening it.'
: 'Provider Settings is unavailable in this runtime.';
return {
ok: false,
action: name,
panelId,
applied: false,
unavailable: true,
error,
say: error,
};
}
button.click();
// Opening unhides immediately; closing removes visible before its fade ends.
const applied = open
? !panel.hidden
: !panel.classList.contains('visible');
const say = applied
? `Provider Settings is ${open ? 'open' : 'closed'}.`
: 'Provider Settings did not change.';
return {
ok: applied,
action: name,
panelId,
applied,
open: applied ? open : !panel.hidden,
say,
...(!applied ? { error: say } : {}),
};
}
setPanelOpen(styleManager, panelId, open);
return { ok: true, action: 'set_panel_open', panelId, open };
}
@@ -734,6 +903,25 @@ export function createGevActionRunner({
error: `Unknown cockpit action: ${args.action || args.command || 'missing'}`,
};
}
if (args.voicePanel !== undefined) {
if (
action !== 'exit' ||
!['preview', 'settings'].includes(args.voicePanel)
)
return {
ok: false,
action: 'control_cockpit',
applied: false,
error: 'Voice panel reveal requires exit and a valid view.',
};
const result = await showVoicePanel(args.voicePanel, true, runOptions);
return {
...result,
action: 'control_cockpit',
cockpitAction: 'exit',
voicePanel: args.voicePanel,
};
}
const inferred = normalizeCockpitNavigationHints(rawAction);
const targetLayer = normalizeCockpitTargetLayer(
args.targetLayer || inferred.targetLayer || args.layer || args.layerId,
src/voice/gevRealtime.js
+7 -2
View File
@@ -50,12 +50,17 @@ export function initGevVoiceCommands(options) {
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
const commands = createVoiceCommands({
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({ ...options, deixis }),
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
src/voice/panel/actionObserver.js
+178
View File
@@ -0,0 +1,178 @@
/**
* Private action accounting prototype. Keys describe invocations, never input
* provenance, successful effects, provider admission or physical settlement.
* The supplied runAction continues to own execution and currentness.
*/
export function createPanelActionObserver({
runAction,
onObservation,
signal,
}) {
if (typeof runAction !== 'function')
throw new TypeError('runAction is required');
const pending = new Map();
let disposed = false;
const publish = (type, record, details = {}) => {
try {
onObservation?.(
Object.freeze({
type,
key: record.key,
name: record.name,
pendingCount: pending.size,
...details,
}),
);
} catch {
// A presentation observer cannot change execution or its result.
}
};
const release = (record) => {
if (!pending.delete(record.key)) return false;
try {
record.signal?.removeEventListener('abort', record.abort);
} catch {
/* Observer cleanup only. */
}
return true;
};
const retireOne = (record, reason) => {
if (release(record)) publish('ownership-retired', record, { reason });
};
const retire = (reason = 'owner-retired') => {
for (const record of [...pending.values()])
retireOne(record, typeof reason === 'string' ? reason : 'owner-retired');
};
const dispose = () => {
if (disposed) return;
disposed = true;
try {
signal?.removeEventListener('abort', dispose);
} catch {
/* Observer cleanup only. */
}
retire('mount-retired');
};
signal?.addEventListener('abort', dispose, { once: true });
if (signal?.aborted) dispose();
function observedAction(...args) {
if (disposed) return Reflect.apply(runAction, this, args);
const record = {
key: Object.freeze({}),
name: typeof args[0] === 'string' ? args[0] : undefined,
};
const originalOptions = args[2];
let forwarded = args;
// Main supplies ordinary data options. Leave exotic/accessor options intact
// rather than evaluating getters or pretending all custom factories fit.
try {
const prototype =
originalOptions == null ? null : Object.getPrototypeOf(originalOptions);
const descriptors =
originalOptions == null
? {}
: Object.getOwnPropertyDescriptors(originalOptions);
const callerSignal = descriptors.signal?.value;
if (callerSignal instanceof AbortSignal) record.signal = callerSignal;
const dataOnly = Object.values(descriptors).every((d) => 'value' in d);
const originalProgress = descriptors.progress?.value;
if (
originalOptions !== undefined &&
originalOptions !== null &&
(prototype === Object.prototype || prototype === null) &&
dataOnly &&
(originalProgress == null || typeof originalProgress === 'function')
) {
const progressDescriptor = descriptors.progress;
delete descriptors.progress;
const options = Object.create(prototype, descriptors);
Object.defineProperty(options, 'progress', {
configurable: true,
enumerable: progressDescriptor?.enumerable ?? true,
writable: true,
value: function (...updates) {
const receiver = this === options ? originalOptions : this;
const returned =
originalProgress == null
? undefined
: Reflect.apply(originalProgress, receiver, updates);
if (pending.has(record.key)) {
// No raw mutable result/options/update object escapes to observers.
try {
const step = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'step',
)?.value;
const label = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'label',
)?.value;
if (typeof step === 'string')
publish('progress', record, {
step,
...(typeof label === 'string' ? { label } : {}),
});
} catch {
/* Private metadata inspection cannot change progress. */
}
}
return returned;
},
});
forwarded = [...args];
forwarded[2] = options;
}
} catch {
// Unsupported option introspection is not an execution failure.
}
if (!record.signal?.aborted) {
pending.set(record.key, record);
record.abort = () => retireOne(record, 'caller-aborted');
try {
record.signal?.addEventListener('abort', record.abort, { once: true });
} catch {
retireOne(record, 'unobserved-signal');
}
if (pending.has(record.key)) publish('attempt-started', record);
}
const settle = (outcome) => {
if (release(record)) publish('attempt-settled', record, { outcome });
};
let result;
try {
result = Reflect.apply(runAction, this, forwarded);
} catch (error) {
settle('synchronous-throw');
throw error;
}
// The actual common session returns a same-realm native Promise. Do not
// assimilate arbitrary thenables, read their .then getter or invent their
// settlement; custom return values still reach the caller unchanged.
try {
if (result instanceof Promise) {
Promise.prototype.then.call(
result,
() => settle('fulfilled'),
() => settle('rejected'),
);
} else if (
result === null ||
!['object', 'function'].includes(typeof result)
) {
settle('synchronous-return');
} else retireOne(record, 'unobserved-return-kind');
} catch {
retireOne(record, 'unobserved-return-kind');
}
return result;
}
return {
runAction: observedAction,
retire,
dispose,
get pendingCount() {
return pending.size;
},
};
}
src/voice/panel/actionObserver.test.mjs
+498
View File
@@ -0,0 +1,498 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { getEventListeners } from 'node:events';
import { createPanelActionObserver } from './actionObserver.js';
import { createVoiceSession } from '../session.js';
function deferred() {
let resolve, reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
test('sync value/throw, receiver, arguments and opaque unsupported result stay exact', () => {
const receiver = {},
args = {},
options = { extra: 7 };
const error = Error('original');
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
onObservation: (event) => events.push(event),
runAction(name, received, opts, tail) {
assert.equal(this, receiver);
assert.equal(received, args);
assert.equal(opts.extra, 7);
assert.equal(tail, 'tail');
calls++;
if (name === 'throw') throw error;
return 42;
},
});
assert.equal(
observer.runAction.call(receiver, 'value', args, options, 'tail'),
42,
);
assert.throws(
() => observer.runAction.call(receiver, 'throw', args, options, 'tail'),
(e) => e === error,
);
assert.equal(calls, 2);
assert.equal(observer.pendingCount, 0);
assert.deepEqual(
events.filter((e) => e.type === 'attempt-settled').map((e) => e.outcome),
['synchronous-return', 'synchronous-throw'],
);
const opaque = Object.defineProperty({}, 'then', {
get() {
throw Error('must not read then');
},
});
const another = createPanelActionObserver({ runAction: () => opaque });
assert.equal(another.runAction('opaque'), opaque);
assert.equal(another.pendingCount, 0);
});
test('native Promise identity, fulfillment object and rejection object are preserved', async () => {
for (const reject of [false, true]) {
const d = deferred(),
value = { exact: true },
events = [];
const observer = createPanelActionObserver({
runAction: () => d.promise,
onObservation: (e) => events.push(e),
});
const returned = observer.runAction('promise');
assert.equal(returned, d.promise);
assert.equal(observer.pendingCount, 1);
if (reject) {
d.reject(value);
await assert.rejects(returned, (e) => e === value);
} else {
d.resolve(value);
assert.equal(await returned, value);
}
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).outcome, reject ? 'rejected' : 'fulfilled');
assert.equal('result' in events.at(-1), false);
assert.equal('error' in events.at(-1), false);
}
});
test('null/reused callIds have independent keys and settlement cannot retire a sibling', async () => {
const a = deferred(),
b = deferred(),
events = [];
const observer = createPanelActionObserver({
runAction: (name) => (name === 'a' ? a.promise : b.promise),
onObservation: (e) => events.push(e),
});
const pa = observer.runAction('a', {}, { callId: null });
const pb = observer.runAction('b', {}, { callId: null });
const starts = events.filter((e) => e.type === 'attempt-started');
assert.notEqual(starts[0].key, starts[1].key);
assert.equal(observer.pendingCount, 2);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 1);
assert.equal(events.at(-1).key, starts[1].key);
a.resolve('a');
await pa;
assert.equal(observer.pendingCount, 0);
});
test('progress preserves public-before-original-before-private order, receiver and option references', async () => {
const order = [],
signal = new AbortController().signal,
symbol = Symbol('unknown');
let currentCalls = 0;
const isCurrent = () => {
currentCalls++;
return true;
};
const options = {
signal,
isCurrent,
progress(update) {
assert.equal(this, options);
order.push('original');
return update;
},
};
Object.defineProperty(options, 'hidden', { value: 8 });
options[symbol] = 9;
const custom = {};
const observer = createPanelActionObserver({
onObservation(e) {
if (e.type === 'progress') order.push('private');
},
runAction(name, args, received) {
assert.equal(received.signal, signal);
assert.equal(received.isCurrent, isCurrent);
assert.equal(received.hidden, 8);
assert.equal(received[symbol], 9);
assert.equal(
Object.getOwnPropertyDescriptor(received, 'hidden').enumerable,
false,
);
order.push('public');
assert.equal(received.progress({ step: 'running' }).step, 'running');
return Promise.resolve();
},
});
await observer.runAction('action', {}, options);
assert.deepEqual(order, ['public', 'original', 'private']);
assert.equal(currentCalls, 0);
const explicit = createPanelActionObserver({
runAction(name, args, received) {
received.progress.call(custom, { step: 'running' });
return 0;
},
});
let originalReceiver;
explicit.runAction(
'explicit',
{},
{
progress() {
originalReceiver = this;
},
},
);
assert.equal(originalReceiver, custom);
});
test('observer faults and private metadata faults cannot replace original callback errors', async () => {
const originalError = Error('callback');
const observer = createPanelActionObserver({
onObservation() {
throw Error('observer');
},
runAction(name, args, options) {
options.progress({ step: 'work' });
return Promise.resolve('ok');
},
});
await assert.rejects(
Promise.resolve().then(() =>
observer.runAction(
'error',
{},
{
progress() {
throw originalError;
},
},
),
),
(e) => e === originalError,
);
const hostile = new Proxy(
{},
{
getOwnPropertyDescriptor() {
throw Error('private read');
},
},
);
const safe = createPanelActionObserver({
runAction(name, args, options) {
options.progress(hostile);
return Promise.resolve('unchanged');
},
onObservation() {
throw Error('observer');
},
});
assert.equal(await safe.runAction('safe', {}, {}), 'unchanged');
});
test('accessor/custom options are forwarded untouched without extra getter/currentness probes', async () => {
let reads = 0,
predicates = 0;
const options = Object.defineProperty(
{
isCurrent() {
predicates++;
return true;
},
},
'progress',
{
get() {
reads++;
return () => {};
},
},
);
const observer = createPanelActionObserver({
runAction(name, args, received) {
assert.equal(received, options);
received.progress({});
received.isCurrent();
return Promise.resolve();
},
});
await observer.runAction('accessor', {}, options);
assert.equal(reads, 1);
assert.equal(predicates, 1);
assert.equal(observer.pendingCount, 0);
});
test('real caller abort retires ownership only; late A cannot release or report over B', async () => {
const a = deferred(),
b = deferred(),
caller = new AbortController(),
events = [],
callbacks = [];
let oldOptions;
const observer = createPanelActionObserver({
onObservation: (e) => events.push(e),
runAction(name, args, options) {
if (name === 'a') oldOptions = options;
return name === 'a' ? a.promise : b.promise;
},
});
const pa = observer.runAction(
'a',
{},
{ signal: caller.signal, progress: () => callbacks.push('original') },
);
caller.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
assert.equal(
events.some((e) => e.type === 'attempt-settled'),
false,
);
const pb = observer.runAction('b');
const count = events.length;
oldOptions.progress({ step: 'late' });
a.resolve('a');
await pa;
assert.deepEqual(callbacks, ['original']);
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 0);
assert.equal(getEventListeners(caller.signal, 'abort').length, 0);
});
test('already-aborted caller and disposed mount still invoke original exactly once', async () => {
const caller = new AbortController(),
mount = new AbortController();
caller.abort();
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction: () => {
calls++;
return Promise.resolve('original');
},
});
assert.equal(
await observer.runAction('aborted', {}, { signal: caller.signal }),
'original',
);
assert.equal(observer.pendingCount, 0);
mount.abort();
assert.equal(await observer.runAction('disposed'), 'original');
assert.equal(calls, 2);
assert.equal(events.length, 0);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('reentrant start retirement retains only fresh ownership and never skips execution', async () => {
const a = deferred(),
b = deferred(),
called = [];
let observer, pb;
observer = createPanelActionObserver({
runAction(name) {
called.push(name);
return name === 'a' ? a.promise : b.promise;
},
onObservation(e) {
if (e.type === 'attempt-started' && e.name === 'a') {
observer.retire('actual-interruption');
pb = observer.runAction('b');
}
},
});
const pa = observer.runAction('a');
assert.deepEqual(called, ['b', 'a']);
assert.equal(observer.pendingCount, 1);
a.resolve();
await pa;
assert.equal(observer.pendingCount, 1);
b.resolve();
await pb;
assert.equal(observer.pendingCount, 0);
});
test('actual pinned-main common session: private invocation stream is separate from public events', async (t) => {
const publicEvents = [],
privateEvents = [],
order = [];
let hooks, observer;
const result = { ok: true, unchanged: true };
const session = createVoiceSession({
runner: async (name, args, options) => {
assert.equal(options.isCurrent(), true);
options.progress({ step: 'actual', label: 'Actual' });
return result;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation(e) {
privateEvents.push(e);
if (e.type === 'progress') order.push('private');
},
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
session.subscribe((e) => {
publicEvents.push(e);
if (e.type === 'progress') order.push('public');
});
await session.start();
assert.equal(
await observer.runAction(
'actual',
{},
{
progress() {
order.push('caller');
},
},
),
result,
);
assert.deepEqual(order, ['public', 'caller', 'private']);
const call = publicEvents.find((e) => e.type === 'action-call');
assert.deepEqual(call, {
type: 'action-call',
name: 'actual',
callId: null,
arguments: {},
});
assert.equal(
publicEvents.some((e) => e.type === 'action-settled'),
false,
);
assert.equal(privateEvents.at(-1).type, 'attempt-settled');
assert.equal(observer.pendingCount, 0);
assert.equal(hooks.signal.aborted, false);
});
test('actual common interruption aborts resources; injected real-lifecycle observation retires uncooperative A', async (t) => {
const a = deferred(),
b = deferred(),
events = [];
let hooks, observer, aSignal;
const session = createVoiceSession({
runner: (name, args, options) => {
if (name === 'a') aSignal = options.signal;
return name === 'a' ? a.promise : b.promise;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation: (e) => events.push(e),
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
await session.start();
const pa = observer.runAction('a');
const rejection = assert.rejects(pa, { name: 'AbortError' });
hooks.emit({ type: 'interruption', reason: 'actual-common-event' });
assert.equal(aSignal.aborted, true);
// This bridge is explicitly supplied by the test, not automatically inferred.
observer.retire('observed-common-interruption');
assert.equal(observer.pendingCount, 0);
const pb = observer.runAction('b');
const count = events.length;
a.resolve({ ok: true });
await rejection;
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve({ ok: true });
await pb;
assert.equal(observer.pendingCount, 0);
});
test('null-prototype option descriptors survive instrumentation; mount abort does not fabricate settlement', async () => {
const mount = new AbortController(),
d = deferred(),
events = [],
originalProgress = [];
const options = Object.assign(Object.create(null), {
progress: () => originalProgress.push('original'),
});
let received;
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction(name, args, opts) {
received = opts;
return d.promise;
},
});
const promise = observer.runAction('pending', {}, options);
assert.equal(Object.getPrototypeOf(received), null);
mount.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
const count = events.length;
received.progress({ step: 'late' });
d.resolve('done');
assert.equal(await promise, 'done');
assert.deepEqual(originalProgress, ['original']);
assert.equal(events.length, count);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('omitted and explicit undefined options keep original arity and values', () => {
const calls = [];
const observer = createPanelActionObserver({
runAction: function (...args) {
calls.push(args);
return 1;
},
});
assert.equal(observer.runAction('omitted', {}), 1);
assert.equal(observer.runAction('explicit', {}, undefined), 1);
assert.equal(calls[0].length, 2);
assert.equal(calls[1].length, 3);
assert.equal(calls[0][2], undefined);
assert.equal(calls[1][2], undefined);
});
src/voice/panel/host.js
+304
View File
@@ -0,0 +1,304 @@
import { t } from '../../i18n/index.js';
import { createManualVoicePanel } from './manualPanel.js';
import {
createVoicePreferences,
bindPreferenceControls,
} from './preferences.js';
import { createPanelActionObserver } from './actionObserver.js';
import { observeNativeActivity } from './nativeActivity.js';
import { createVoiceInactivity } from './inactivity.js';
/** One UI/preference owner across provider mounts; no semantic voice routes. */
export function createVoicePanelHost({
storage,
clock = {},
readCockpitState = () => ({
active:
globalThis.document?.body?.classList?.contains('cockpit-mode') === true,
}),
} = {}) {
const preferences = createVoicePreferences({ storage });
let current = null;
let disposed = false;
let saved = { view: 'preview', panelOpen: false, automatic: true };
function observeActions(hooks) {
const observer = createPanelActionObserver({
runAction: hooks.runAction,
signal: hooks.signal,
onObservation: () => {
if (current?.observer === observer) current.workChanged();
},
});
return observer;
}
function mount({ ui, card, session, observer, nativeProvider }) {
if (disposed || !ui.voicePanel || !card?.setEnabled || !card?.setPresented)
return null;
current?.destroy();
let dead = false;
let revision = 0;
let operationEpoch = 0;
let operationWrite = false;
let panel;
let idle;
let refreshQueued = false;
let automatic = saved.automatic;
let inputBusy = false;
let finalTranscript = null;
const presentationLifetime = new AbortController();
const rearm = () => {
if (!owns() || !session.isActive()) return;
automatic = true;
saved.automatic = true;
card.rearm?.();
// Unknown IDs cannot distinguish a fresh identical answer after input.
if (!finalTranscript?.id) finalTranscript = null;
};
const owns = () =>
!dead && !disposed && current === binding && !session.disposed;
const remember = ({ automatic: auto = false } = {}) => {
if (!auto) {
revision++;
automatic = false;
}
if (panel) {
const value = panel.snapshot();
saved = { view: value.view, panelOpen: value.panelOpen, automatic };
}
};
const renderActivity = () => {
if (ui.voiceTimeoutStatus)
ui.voiceTimeoutStatus.textContent =
nativeProvider === 'openai' || nativeProvider === 'gemini'
? ''
: t('Automatic stop is unavailable for this voice connection.');
};
const changed = () => {
if (!owns()) return;
const busy = activity.snapshot().inputBusy;
if (busy && !inputBusy) rearm();
inputBusy = busy;
idle?.update({ activity: true });
renderActivity();
};
const activity = observeNativeActivity({
adapter: session.adapter,
nativeProvider,
signal: session.signal,
onChange: changed,
onInput: rearm,
});
const closeAfterStop = (ticket) => {
if (owns() && ticket === revision && !session.isActive()) panel.close();
};
const binding = {
observer,
get panel() {
return panel;
},
stopTicket: () => revision,
closeAfterStop,
workChanged: changed,
captureOperation() {
const stamp = revision;
const epoch = operationEpoch;
const isCurrent = () =>
owns() && stamp === revision && epoch === operationEpoch;
const cancelled = () => ({
ok: false,
applied: false,
cancelled: true,
});
return {
isCurrent,
setMinutes(value) {
if (!isCurrent()) return cancelled();
if (typeof value !== 'number' || !Number.isInteger(value))
return {
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
minutes: preferences.snapshot().minutes,
};
let result;
operationWrite = true;
try {
result = preferences.setMinutes(value);
} finally {
operationWrite = false;
}
return {
ok: result.ok,
applied: result.applied,
changed: result.changed,
persisted: result.persisted,
minutes: result.minutes,
...(result.reason ? { reason: result.reason } : {}),
};
},
show(view, open = true) {
if (!isCurrent()) return cancelled();
const result = open ? panel.select(view) : panel.close(view);
return { ...result, applied: result.ok };
},
};
},
handle(event) {
if (!owns()) return;
if (event.type !== 'state' && !session.isActive()) return;
if (
event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state)
) {
finalTranscript = null;
if (event.state !== 'connecting') saved.automatic = automatic = false;
}
if (
event.type === 'interruption' &&
!event.preserveAdvisoryCaption &&
['user-speech', 'user-text', 'push-to-talk'].includes(event.reason)
)
rearm();
if (
event.type === 'transcript' &&
event.role === 'assistant' &&
event.text
) {
const id = event.responseId || event.itemId || null;
if (
finalTranscript &&
(id ? id === finalTranscript.id : !finalTranscript.id) &&
((!event.final && id) ||
(event.final && event.text === finalTranscript.text))
)
return;
if (event.final) finalTranscript = { id, text: event.text };
if (
automatic &&
preferences.snapshot().previewEnabled &&
!readCockpitState()?.active
)
panel.select('preview', { automatic: true });
}
// A retired native boundary is not fresh input and must not revive
// dismissal or erase the advisory caption it explicitly preserves.
if (!(event.type === 'interruption' && event.preserveAdvisoryCaption))
card.handle(event);
if (
event.type === 'interruption' ||
(event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state))
) {
operationEpoch++;
observer.retire('session-boundary');
}
activity.handle(event);
idle?.update();
// Native controllers finish processing after forwarding some events.
// One guarded microtask refresh reads the final current local state.
if (!refreshQueued) {
refreshQueued = true;
queueMicrotask(() => {
refreshQueued = false;
if (!owns()) return;
activity.handle({ type: 'refresh' });
idle?.update();
});
}
},
snapshot: () => ({
...panel.snapshot(),
preferences: preferences.snapshot(),
activity: activity.snapshot(),
inactivity: idle.snapshot(),
}),
destroy() {
if (dead) return;
dead = true;
presentationLifetime.abort();
idle?.destroy();
activity.destroy();
fields.dispose();
observer.dispose();
panel.destroy();
session.signal.removeEventListener('abort', binding.destroy);
if (current === binding) current = null;
},
};
current = binding;
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preferences.snapshot().previewEnabled,
getCockpitActive: () => readCockpitState()?.active === true,
copy: {
previewEmpty: t('Voice replies appear here.'),
previewDisabled: t('Preview is off.'),
},
onPresentationIntent: remember,
});
const fields = bindPreferenceControls({
ui,
preferences,
copy: {
previewNote: '',
timeoutNote: t('These preferences stay in this browser.'),
notSaved: t('Your choice applies now, but could not be saved.'),
},
onChange: (state) => {
if (!owns()) return;
if (!operationWrite) revision++;
card.setEnabled(state.previewEnabled);
panel.sync();
idle?.setMinutes(state.minutes);
},
});
globalThis.window?.addEventListener(
'gev:cockpit-mode-changed',
() => {
if (owns()) panel.sync();
},
{ signal: presentationLifetime.signal },
);
ui.root?.querySelector('#gev-voice-card-dismiss')?.addEventListener(
'click',
() => {
remember();
if (panel.snapshot().panelOpen) ui.previewTab?.focus();
},
{ signal: presentationLifetime.signal },
);
card.setEnabled(preferences.snapshot().previewEnabled);
panel.sync();
idle = createVoiceInactivity({
readActivity: activity.snapshot,
readPending: () => observer.pendingCount,
minutes: preferences.snapshot().minutes,
signal: session.signal,
...clock,
onExpire: () => {
if (!owns()) return;
const ticket = revision;
session.stop();
closeAfterStop(ticket);
},
});
if (saved.panelOpen) panel.select(saved.view, { automatic: true });
renderActivity();
session.signal.addEventListener('abort', binding.destroy, { once: true });
if (session.signal.aborted) binding.destroy();
return binding;
}
return {
observeActions,
mount,
snapshot: () => preferences.snapshot(),
destroy() {
if (disposed) return;
current?.destroy();
disposed = true;
},
};
}
src/voice/panel/host.test.mjs
+756
View File
@@ -0,0 +1,756 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceControl } from '../control.js';
import { createGevActionRunner } from '../gevActions.js';
import { createVoiceCommands as bind } from '../sessionCommands.js';
import { createVoiceCommands as compose } from '../commands.js';
import { createProviderCommands } from '../providerCommands.js';
import { createVoicePanelHost } from './host.js';
import { installPanelDom, controlledClock } from './testDom.mjs';
function factory(instances, { stopError = null, native = false } = {}) {
return (hooks) => {
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {},
};
const controller = native
? {
input,
_input: input,
_turns: { observeSpeech() {} },
status: 'idle',
dc: null,
sendTextCommand(text) {
if (!text) return false;
hooks.emit({
type: 'pointer',
pointer: { kind: 'here', label: 'Typed target' },
});
},
createAudio(options) {
this.audioCallbacks = options;
return {
pending: false,
play() {},
clearPlayback() {},
close() {},
};
},
handleMessage() {},
userActivityOpen: false,
pendingTools: 0,
isActive: () => true,
}
: {};
const adapter = {
controller,
capabilities: { pushToTalk: true },
starts: 0,
stops: 0,
bindControls() {},
start() {
this.starts++;
if (native) {
hooks.emit({ type: 'state', state: 'connecting' });
controller.audio = controller.createAudio({ pushToTalk: false });
hooks.emit({ type: 'state', state: 'connecting' });
// Native open-mic startup marks presentation pending before speech.
controller.userTurnPending = true;
}
hooks.emit({ type: 'state', state: 'listening' });
},
stop() {
this.stops++;
if (stopError) throw stopError;
},
sendText: () => false,
sendMapEvent() {},
getStartOptions: () => ({ pushToTalk: false }),
hooks,
input,
};
instances.push(adapter);
return adapter;
};
}
function mounted(t, options = {}) {
const doc = installPanelDom(t);
const clock = controlledClock();
const host = createVoicePanelHost({
storage: options.storage ?? null,
clock,
readCockpitState: options.readCockpitState,
});
const instances = [];
const commands = bind({
runner: options.runner ?? (async () => ({ ok: true })),
dataManager: { layers: new Map() },
createSession: factory(instances, options),
panelHost: host,
nativePanelProvider: options.native
? (options.nativeProvider ?? 'gemini')
: null,
});
t.after(() => {
try {
commands.session.destroy();
} catch {}
host.destroy();
});
return { doc, clock, host, instances, commands, ui: commands.ui };
}
test('actual control default remains opt-out; panel markup has unique controlled panes and an outside alert', (t) => {
const doc = installPanelDom(t);
const baseline = createVoiceControl();
assert.equal(baseline.voicePanel, null);
const ui = createVoiceControl({ reset: true, panel: true });
assert.equal(doc.getElementById('gev-voice-panel'), ui.voicePanel);
assert.equal(
ui.voiceSettingsButton.getAttribute('aria-controls'),
ui.voicePanel.id,
);
assert.equal(ui.previewTab.getAttribute('aria-controls'), ui.previewPanel.id);
assert.equal(
ui.settingsTab.getAttribute('aria-controls'),
ui.voiceSettingsPanel.id,
);
assert.equal(ui.previewPanel.contains(ui.card), true);
assert.equal(
ui.voicePanel.contains(ui.root.querySelector('.gev-voice-error-tray')),
false,
);
assert.equal(ui.root.querySelectorAll('#gev-voice-card').length, 1);
});
test('actual binder/card has launcher toggle, idempotent tabs, focus and disabled Preview', (t) => {
const { ui, commands, doc } = mounted(t);
ui.voiceSettingsButton.fire('click');
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
ui.voiceSettingsButton.fire('click');
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
ui.settingsTab.fire('click');
ui.settingsTab.fire('keydown', { key: 'End' });
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
ui.settingsTab.fire('click');
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A visible reply',
});
const dismiss = ui.root.querySelector('#gev-voice-card-dismiss');
dismiss.focus();
dismiss.fire('click');
assert.equal(ui.card.hidden, true);
assert.equal(commands.voicePanel.snapshot().panelOpen, true);
assert.equal(doc.activeElement?.id, ui.previewTab.id);
ui.voicePanel.fire('keydown', { key: 'Escape', target: doc.activeElement });
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
commands.voicePanel.select('settings');
ui.voicePreviewInput.checked = false;
ui.voicePreviewInput.fire('change');
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A hidden reply',
});
assert.equal(ui.card.hidden, true);
assert.equal(ui.previewEmpty.textContent, 'Preview is off.');
ui.previewTab.focus();
ui.voicePanel.fire('keydown', { key: 'Escape' });
assert.equal(doc.activeElement, ui.voiceSettingsButton);
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'Later reply',
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
});
test('actual facade remount keeps failed-save live preferences/view and manual close, with no restore writes or focus theft', async (t) => {
const doc = installPanelDom(t);
const writes = [];
const storage = {
getItem: () => null,
setItem(key, value) {
writes.push([key, value]);
throw Error('blocked');
},
};
const host = createVoicePanelHost({ storage });
const instances = [];
const commands = createProviderCommands(
{
runner: async () => ({}),
dataManager: { layers: new Map() },
supportedProviders: ['openai', 'gemini'],
},
{
bind: (options) => bind({ ...options, panelHost: host }),
factories: { openai: factory(instances), gemini: factory(instances) },
storage,
},
);
t.after(() => {
commands.destroy();
host.destroy();
});
const old = commands.ui;
commands.voicePanel.select('settings');
old.voicePreviewInput.checked = false;
old.voicePreviewInput.fire('change');
old.voiceTimeoutInput.value = '7';
old.voiceTimeoutSelect.value = 'custom';
old.voiceTimeoutSelect.fire('change');
const preferenceWrites = () =>
writes.filter(([key]) => key.startsWith('godsEyeView.voice.')).length;
assert.equal(preferenceWrites(), 2);
old.providerSelect.value = 'gemini';
old.providerSelect.fire('change');
assert.notEqual(commands.ui, old);
assert.equal(old.root.isConnected, false);
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
assert.deepEqual(commands.getVoicePreferences(), {
previewEnabled: false,
minutes: 7,
previewPersisted: false,
timeoutPersisted: false,
});
assert.equal(commands.ui.voicePreviewInput.checked, false);
assert.match(commands.ui.voiceTimeoutNote.textContent, /could not be saved/);
assert.equal(preferenceWrites(), 2);
assert.equal(doc.activeElement, commands.ui.providerSelect);
commands.ui.voicePreviewInput.checked = true;
commands.ui.voicePreviewInput.fire('change');
commands.voicePanel.close();
await commands.start();
const retired = instances[1];
commands.ui.providerSelect.value = 'openai';
commands.ui.providerSelect.fire('change');
await commands.start();
assert.equal(commands.session.isActive(), true);
retired.hooks.emit({
type: 'transcript',
role: 'assistant',
text: 'Retired output',
final: true,
});
const emit = instances[2].hooks.emit;
emit({
type: 'transcript',
role: 'assistant',
text: 'Current output after remount',
final: true,
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
emit({
type: 'transcript',
role: 'assistant',
text: 'Reply after genuine input',
final: true,
});
assert.equal(commands.voicePanel.snapshot().previewVisible, true);
});
test('explicit successful mic Stop closes; failed Stop idle and startup error do not', async (t) => {
const good = mounted(t);
await good.commands.session.start();
good.commands.voicePanel.select('settings');
good.ui.button.fire('click');
assert.equal(good.commands.voicePanel.snapshot().panelOpen, false);
const error = Error('stop failed');
const bad = mounted(t, { stopError: error });
await bad.commands.session.start();
bad.commands.voicePanel.select('settings');
assert.throws(
() => bad.ui.button.fire('click'),
(value) => value === error,
);
assert.equal(bad.commands.session.state, 'idle');
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
bad.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'startup failed',
});
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
});
test('reentrant manual choice/new start during Stop cannot close the successor choice', async (t) => {
const f = mounted(t);
await f.commands.session.start();
f.commands.voicePanel.select('settings');
const unsubscribe = f.commands.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'idle')
f.commands.voicePanel.select('preview');
});
f.ui.button.fire('click');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
unsubscribe();
});
test('integrated inactivity rechecks held PTT at expiry, then normal stop closes on later quiet expiry', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
f.commands.voicePanel.select('settings');
f.ui.voiceTimeoutInput.value = '1';
f.ui.voiceTimeoutSelect.value = 'custom';
f.ui.voiceTimeoutSelect.fire('change');
f.instances[0].input.pushToTalkKeyHeld = true;
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0);
f.instances[0].input.releasePushToTalkKey();
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1);
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('None cancels integrated app timer and unknown custom hosts never guess expiry', async (t) => {
const native = mounted(t, { native: true });
await native.commands.session.start();
native.ui.voiceTimeoutSelect.value = '0';
native.ui.voiceTimeoutSelect.fire('change');
native.clock.advance(600000);
assert.equal(native.instances[0].stops, 0);
assert.equal(native.commands.getVoicePreferences().minutes, 0);
const unknown = mounted(t);
await unknown.commands.session.start();
unknown.clock.advance(600000);
assert.equal(unknown.instances[0].stops, 0);
assert.match(unknown.ui.voiceTimeoutStatus.textContent, /unavailable/);
});
test('transient native OpenAI response activity does not announce connection unavailability', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai' });
await f.commands.session.start();
f.ui.voiceTimeoutInput.value = '1';
f.ui.voiceTimeoutSelect.value = 'custom';
f.ui.voiceTimeoutSelect.fire('change');
f.instances[0].controller._turns.observeSpeech({ type: 'response.created' });
assert.equal(f.ui.voiceTimeoutStatus.textContent, '');
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0, 'unknown output still prevents expiry');
f.instances[0].controller._turns.observeSpeech({
type: 'output_audio_buffer.cleared',
});
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1, 'known quiet output permits expiry');
});
test('native factory with custom controller remains uninstrumented despite panel opt-in', (t) => {
installPanelDom(t);
let inspected = 0;
const commands = compose({
panel: true,
panelStorage: null,
runner: async () => ({}),
dataManager: { layers: new Map() },
createController: (options) => ({
ui: options.ui,
get _turns() {
inspected++;
throw Error('custom internal getter');
},
start() {},
stop() {},
sendTextCommand() {},
notifyMapEvent() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
toggleVoiceTier() {},
}),
});
t.after(() => commands.session.destroy());
assert.equal(inspected, 0);
assert.equal(commands.getVoicePanelState().activity.known, false);
assert.equal(inspected, 0);
});
test('native connecting phases retain output observation; idle starts before first speech and waits for drain', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
const adapter = f.instances[0];
const controller = adapter.controller;
assert.equal(
f.commands.getVoicePanelState().inactivity.eligible,
true,
'open microphone presentation-pending is not admitted speech',
);
controller.audio.pending = true;
controller.handleMessage({});
f.clock.advance(300000);
assert.equal(adapter.stops, 0);
assert.equal(f.commands.getVoicePanelState().activity.outputBusy, true);
controller.audio.pending = false;
controller.audioCallbacks.onDrain();
f.clock.advance(300000);
assert.equal(adapter.stops, 1);
});
async function directActions(
t,
{ storage = null, delayExit = false, exposePanel = true } = {},
) {
installPanelDom(t);
let f, releaseExit;
const state = { active: true, exits: 0, visual: [] };
const styleManager = {
getCockpitState: () => ({ active: state.active }),
controlCockpit(action) {
assert.equal(action, 'exit');
state.exits++;
state.active = false;
const result = { ok: true, state: { active: false } };
return delayExit
? new Promise((resolve) => {
releaseExit = () => resolve(result);
})
: result;
},
setPanelCollapsed(...args) {
state.visual.push(args);
},
};
const runner = createGevActionRunner({
viewer: {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
},
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
readVoicePanelOperation: () =>
exposePanel ? f?.commands.captureVoicePanelOperation() : undefined,
});
f = mounted(t, { native: true, runner, storage });
await f.commands.session.start();
return {
...f,
runner,
state,
run: f.instances[0].hooks.runAction,
releaseExit: () => releaseExit(),
};
}
test('actual runner applies exact timeout directly and reports live-unsaved truth, invalid/cancelled attempts stay inert', async (t) => {
const f = await directActions(t);
const result = await f.run('set_voice_inactivity_timeout', { minutes: 7 });
assert.equal(result.applied, true);
assert.equal(result.persisted, false);
assert.equal(result.minutes, 7);
assert.match(result.say, /could not be saved/);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const invalid = await f.run('set_voice_inactivity_timeout', { minutes: 7.5 });
assert.equal(invalid.applied, false);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const abort = new AbortController();
abort.abort();
const cancelled = await f.runner(
'set_voice_inactivity_timeout',
{ minutes: 2 },
{ signal: abort.signal },
);
assert.equal(cancelled.cancelled, true);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const operation = f.commands.captureVoicePanelOperation();
f.commands.session.stop();
assert.equal(operation.setMinutes(2).cancelled, true);
});
test('named voice settings exits Cockpit directly; Preview stays disabled and visual presets stay separate from Scenes', async (t) => {
const f = await directActions(t);
const result = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(f.state.exits, 1);
assert.equal(result.settingsOpen, true);
f.ui.voicePreviewInput.checked = false;
f.ui.voicePreviewInput.fire('change');
const preview = await f.run('set_panel_open', {
panelId: 'voice control',
open: true,
});
assert.equal(preview.view, 'preview');
assert.equal(preview.previewEnabled, false);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
for (const panelId of [
'visual styles',
'visual preset',
'visual presets',
'scenes',
])
await f.run('set_panel_open', { panelId, open: true });
assert.equal(f.state.exits, 1);
assert.deepEqual(
f.state.visual.map(([id]) => id),
['control-panel', 'control-panel', 'control-panel', 'scene-panel'],
);
await assert.rejects(
() => f.run('set_panel_open', { panelId: 'settings', open: true }),
/Unknown panel: settings/,
);
assert.equal(
f.state.visual.length,
4,
'an unnamed settings target never guesses a panel',
);
});
test('canonical exit reveals requested voice UI; newer manual choice or Stop defeats pending reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
f.commands.voicePanel.select('preview');
f.releaseExit();
const result = await pending;
assert.equal(result.cancelled, true);
assert.equal(f.state.active, false);
assert.equal(result.exit.ok, true);
assert.equal(result.cockpitExited, true);
assert.match(result.say, /Cockpit exited/);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
f.state.active = true;
const alias = f.run('control_cockpit', {
action: 'leave cockpit',
voicePanel: 'settings',
});
f.releaseExit();
assert.equal((await alias).settingsOpen, true);
f.commands.voicePanel.select('preview');
f.state.active = true;
const stopped = f.run('control_cockpit', {
action: 'quit cockpit',
voicePanel: 'settings',
});
f.commands.session.stop();
f.releaseExit();
await assert.rejects(stopped, { name: 'AbortError' });
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
});
test('unavailable host refuses direct voice operations before Cockpit or preference effects', async (t) => {
const f = await directActions(t, { exposePanel: false });
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 1 });
assert.equal(timeout.unavailable, true);
assert.equal(timeout.applied, false);
const panel = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(panel.unavailable, true);
assert.equal(f.state.exits, 0);
assert.equal(f.commands.getVoicePreferences().minutes, 5);
});
test('voice timeout change preserves a pending voice reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 0 });
assert.equal(timeout.minutes, 0);
assert.equal(timeout.applied, true);
f.releaseExit();
const result = await pending;
assert.equal(result.settingsOpen, true);
assert.equal(f.commands.getVoicePreferences().minutes, 0);
});
test('active provider output reveals Preview after done; manual choice wins until genuine input, replays do not reset idle', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai' });
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text, responseId = 'r1', extra = {}) =>
emit({
type: 'transcript',
role: 'assistant',
text,
responseId,
final: true,
...extra,
});
emit({ type: 'completion', responseId: 'r1' });
reply('Late answer');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
const timer = f.commands.voiceCard.hideTimer;
reply('Late answer');
reply('Late', 'r1', { final: false });
assert.equal(f.commands.voiceCard.hideTimer, timer);
assert.equal(f.commands.voiceCard.state.assistant, 'Late answer');
f.commands.voicePanel.close();
emit({
type: 'transcript',
role: 'user',
text: 'advisory caption',
final: false,
});
f.commands.voiceCard.handle({ type: 'dismiss' });
const advisory = f.commands.voiceCard.state.user;
emit({
type: 'interruption',
reason: 'user-speech',
preserveAdvisoryCaption: true,
});
assert.equal(f.commands.voiceCard.state.dismissed, true);
assert.equal(f.commands.voiceCard.state.user, advisory);
reply('Older reply', 'r2');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
reply('A reply with no newest-request attribution', 'older-id');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
f.commands.voicePanel.select('settings');
reply('Another reply', 'r3');
assert.equal(f.commands.voicePanel.snapshot().settingsOpen, true);
f.commands.voicePanel.close();
f.commands.voiceCard.handle({ type: 'dismiss' });
f.commands.sendTextCommand('Show the typed target');
reply('Typed reply', 'typed-id');
assert.equal(f.commands.voicePanel.snapshot().previewVisible, true);
assert.equal(f.commands.voiceCard.state.dismissed, false);
assert.equal(f.commands.voiceCard.state.pointer.label, 'Typed target');
f.commands.session.stop();
reply('Stopped stale output', 'r4');
assert.equal(f.commands.voiceCard.state.assistant, null);
f.commands.session.destroy();
reply('Disposed stale output', 'r5');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('Preview off and actual Cockpit state suppress automatic presentation without enabling preference', async (t) => {
let cockpit = false;
const f = mounted(t, {
native: true,
readCockpitState: () => ({ active: cockpit }),
});
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text) =>
emit({ type: 'transcript', role: 'assistant', text, final: true });
reply('Visible');
f.ui.previewTab.focus();
cockpit = true;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
f.commands.voicePanel.select('settings', { focus: true });
assert.equal(f.doc.activeElement, f.ui.settingsTab);
f.commands.voicePanel.select('preview', { focus: true });
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
const outside = f.doc.createElement('button');
f.doc.body.appendChild(outside);
outside.focus();
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.doc.activeElement, outside);
assert.equal(f.commands.voicePanel.snapshot().previewVisible, false);
reply('Suppressed in Cockpit');
assert.equal(f.ui.voicePanel.hidden, true);
f.commands.voicePanel.close();
cockpit = false;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.voicePanel.hidden, true);
f.ui.voicePreviewInput.checked = false;
f.ui.voicePreviewInput.fire('change');
emit({ type: 'interruption', reason: 'user-speech' });
reply('Preference stays off');
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
});
test('Provider Settings uses its ready dialog handlers and refuses unavailable or cancelled requests', async (t) => {
const f = await directActions(t);
t.mock.timers.enable({ apis: ['setTimeout'] });
const request = (open, options) =>
f.run('set_panel_open', { panelId: 'provider-settings', open }, options);
assert.equal((await request(true)).unavailable, true);
const doc = globalThis.document;
const chip = doc.createElement('button');
chip.id = 'key-setup-chip';
const panel = doc.createElement('aside');
panel.id = 'key-setup';
panel.hidden = true;
const close = doc.createElement('button');
panel.appendChild(close);
doc.body.append(chip, panel);
const query = panel.querySelector.bind(panel);
panel.querySelector = (selector) =>
selector === '[data-key-setup-close]' ? close : query(selector);
chip.click = () => chip.fire('click');
close.click = () => close.fire('click');
let frames = 0;
const previousFrame = globalThis.requestAnimationFrame;
globalThis.requestAnimationFrame = (callback) => {
frames++;
callback();
};
t.after(() => {
if (previousFrame === undefined) delete globalThis.requestAnimationFrame;
else globalThis.requestAnimationFrame = previousFrame;
});
const { initKeySetup } = await import('../../keySetup.js');
let finishStatus;
const pending = initKeySetup({
documentRef: doc,
fetchImpl: () =>
new Promise((resolve) => {
finishStatus = resolve;
}),
});
assert.equal(
(await request(true)).unavailable,
true,
'initializing is not ready',
);
finishStatus({
ok: true,
json: async () => ({ keys: [], setCount: 0, total: 1 }),
});
const setup = await pending;
t.after(() => setup.destroy());
doc.body.classList.add('cockpit-mode');
assert.equal((await request(true)).unavailable, true);
assert.equal(
panel.hidden,
true,
'a concealed view cannot schedule a later surprise reveal',
);
doc.body.classList.remove('cockpit-mode');
const abort = new AbortController();
abort.abort();
await assert.rejects(() => request(true, { signal: abort.signal }), {
name: 'AbortError',
});
assert.equal(panel.hidden, true);
assert.equal((await request(true)).ok, true);
assert.equal(panel.hidden, false);
assert.equal(panel.classList.contains('visible'), true);
assert.equal((await request(true)).ok, true);
assert.equal(frames, 1, 'repeated open reuses the dialog');
assert.equal((await request(false)).ok, true);
assert.equal(panel.classList.contains('visible'), false);
t.mock.timers.tick(400);
assert.equal(panel.hidden, true);
assert.equal((await request(false)).ok, true);
assert.equal(
f.state.visual.length,
0,
'provider setup never opens a visual or scene panel',
);
});
src/voice/panel/inactivity.js
+77
View File
@@ -0,0 +1,77 @@
/** Local inactivity only. Unknown activity never authorizes expiry. */
export function createVoiceInactivity({
readActivity,
readPending,
minutes = 5,
onExpire,
signal,
setTimer = setTimeout,
clearTimer = clearTimeout,
now = Date.now,
}) {
if (!Number.isInteger(minutes) || minutes < 0 || minutes > 60) minutes = 5;
let disposed = false,
timer = null,
deadline = null,
eligible = false,
epoch = 0;
function cancel() {
epoch++;
if (timer !== null) clearTimer(timer);
timer = deadline = null;
}
function quiet() {
const a = readActivity();
return (
a.known &&
a.connected &&
!a.held &&
!a.inputBusy &&
!a.outputBusy &&
!a.generationBusy &&
!readPending()
);
}
function arm() {
cancel();
if (!minutes || disposed) return;
const id = epoch;
deadline = now() + minutes * 60000;
timer = setTimer(() => {
if (disposed || id !== epoch) return;
timer = deadline = null;
eligible = quiet();
if (eligible) onExpire?.();
}, minutes * 60000);
}
function update({ activity = false } = {}) {
if (disposed) return;
const next = quiet();
if (!next) cancel();
else if (!eligible || activity) arm();
eligible = next;
}
function setMinutes(value) {
if (!Number.isInteger(value) || value < 0 || value > 60) return false;
if (disposed) return false;
minutes = value;
eligible = false;
update();
return true;
}
function destroy() {
if (disposed) return;
disposed = true;
cancel();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
update();
return {
update,
setMinutes,
snapshot: () => ({ minutes, deadline, eligible, disposed }),
destroy,
};
}
src/voice/panel/inactivity.test.mjs
+108
View File
@@ -0,0 +1,108 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
function fixture() {
let activity = {
known: true,
connected: true,
held: false,
inputBusy: false,
outputBusy: false,
generationBusy: false,
};
let pending = 0,
expired = 0,
time = 100,
serial = 0;
const timers = new Map();
const idle = createVoiceInactivity({
readActivity: () => activity,
readPending: () => pending,
onExpire: () => expired++,
now: () => time,
setTimer: (fn, delay) => {
timers.set(++serial, { fn, delay });
return serial;
},
clearTimer: (id) => timers.delete(id),
});
return {
idle,
activity,
timers,
setPending: (n) => (pending = n),
expired: () => expired,
fire: () => {
const entry = [...timers][0];
timers.delete(entry[0]);
entry[1].fn();
},
tick: (n) => (time += n),
};
}
test('one clock; quiet arms default five minutes; busy blocks and quiescence rearms', () => {
const f = fixture();
assert.equal([...f.timers.values()][0].delay, 300000);
assert.equal(f.idle.snapshot().deadline, 300100);
f.activity.outputBusy = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.outputBusy = false;
f.tick(200);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, 300300);
f.fire();
assert.equal(f.expired(), 1);
});
test('actual activity resets but silent unchanged snapshots do not', () => {
const f = fixture();
const deadline = f.idle.snapshot().deadline;
f.tick(1000);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, deadline);
f.idle.update({ activity: true });
assert.equal(f.idle.snapshot().deadline, deadline + 1000);
});
test('None cancels; invalid values cannot replace setting; explicit retry rearms', () => {
const f = fixture();
assert.equal(f.idle.setMinutes(0), true);
assert.equal(f.timers.size, 0);
for (const bad of [1.5, -1, 61, '5', NaN])
assert.equal(f.idle.setMinutes(bad), false);
assert.equal(f.idle.setMinutes(1), true);
assert.equal([...f.timers.values()][0].delay, 60000);
});
test('expiry rereads held, unknown and pending counts despite missing update', () => {
for (const kind of ['held', 'unknown', 'pending']) {
const f = fixture();
if (kind === 'held') f.activity.held = true;
if (kind === 'unknown') f.activity.known = false;
if (kind === 'pending') f.setPending(2);
f.fire();
assert.equal(f.expired(), 0);
}
});
test('pending siblings and held-before-connect forbid timer; stale callback cannot expire', () => {
const f = fixture();
const stale = [...f.timers.values()][0].fn;
f.setPending(2);
f.idle.update();
stale();
assert.equal(f.expired(), 0);
f.setPending(1);
f.idle.update();
assert.equal(f.timers.size, 0);
f.setPending(0);
f.activity.connected = false;
f.activity.held = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.connected = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.held = false;
f.idle.update();
assert.equal(f.timers.size, 1);
f.idle.destroy();
assert.equal(f.timers.size, 0);
});
src/voice/panel/manualPanel.js
+182
View File
@@ -0,0 +1,182 @@
const ROW_GRACE_MS = 700;
/** Manual presentation only. Session and action events belong to the host. */
export function createManualVoicePanel({
ui,
card,
getPreviewEnabled,
getCockpitActive = () => false,
copy,
onPresentationIntent,
}) {
if (!ui?.voicePanel || typeof getPreviewEnabled !== 'function')
throw new TypeError(
'A voice panel and Preview preference reader are required.',
);
if (
typeof copy?.previewEmpty !== 'string' ||
typeof copy?.previewDisabled !== 'string'
)
throw new TypeError('Localized Preview messages are required.');
const lifetime = new AbortController();
const panel = ui.voicePanel;
const tabs = [ui.previewTab, ui.settingsTab];
const doc = panel.ownerDocument;
let disposed = false;
let open = false;
let view = 'preview';
const enabled = () => getPreviewEnabled() === true;
const suppressed = () => view === 'preview' && getCockpitActive() === true;
const visible = (element) =>
Boolean(
element &&
!element.hidden &&
element.isConnected !== false &&
element.checkVisibility?.({
opacityProperty: true,
visibilityProperty: true,
}) !== false,
);
const snapshot = () => ({
view,
panelOpen: !disposed && open,
settingsOpen: !disposed && open && view === 'settings',
previewEnabled: enabled(),
previewVisible:
!disposed &&
open &&
view === 'preview' &&
!suppressed() &&
enabled() &&
visible(ui.card),
});
const sync = () => {
if (disposed) return;
const hidePanel = !open || suppressed();
const restoreFocus = hidePanel && panel.contains?.(doc?.activeElement);
panel.hidden = hidePanel;
panel.dataset.view = view;
if (ui.previewPanel)
ui.previewPanel.hidden = !open || view !== 'preview' || suppressed();
if (ui.voiceSettingsPanel)
ui.voiceSettingsPanel.hidden = !open || view !== 'settings';
ui.voiceSettingsButton?.setAttribute(
'aria-expanded',
String(open && view === 'settings'),
);
tabs.forEach((tab, index) => {
if (!tab) return;
const selected = view === (index === 0 ? 'preview' : 'settings');
tab.setAttribute('aria-selected', String(selected));
tab.tabIndex = selected ? 0 : -1;
});
card?.setPresented?.(open && view === 'preview' && !suppressed());
if (ui.previewEmpty) {
ui.previewEmpty.hidden = Boolean(ui.card && !ui.card.hidden);
ui.previewEmpty.textContent = enabled()
? copy.previewEmpty
: copy.previewDisabled;
}
if (restoreFocus)
[ui.voiceSettingsButton, ui.button].find(visible)?.focus?.();
};
const refuse = (reason) => ({
ok: false,
changed: false,
reason,
...snapshot(),
});
const select = (next, { focus = false, automatic = false } = {}) => {
if (disposed) return refuse('disposed');
if (!['preview', 'settings'].includes(next)) return refuse('unknown-view');
const changed = !open || view !== next;
const outgoing =
view === 'preview' ? ui.previewPanel : ui.voiceSettingsPanel;
const transferFocus = changed && outgoing?.contains?.(doc?.activeElement);
open = true;
view = next;
onPresentationIntent?.({ automatic });
sync();
const tab = tabs[next === 'preview' ? 0 : 1];
if ((focus || transferFocus) && visible(panel) && visible(tab))
tab.focus?.();
return { ok: true, changed, ...snapshot() };
};
const close = (onlyView) => {
if (disposed) return refuse('disposed');
if (onlyView && !['preview', 'settings'].includes(onlyView))
return refuse('unknown-view');
if (onlyView && onlyView !== view)
return { ok: true, changed: false, ...snapshot() };
const changed = open;
open = false;
onPresentationIntent?.();
sync();
return { ok: true, changed, ...snapshot() };
};
const listen = (element, type, handler) =>
element?.addEventListener(type, handler, { signal: lifetime.signal });
const toggleSettings = () =>
open && view === 'settings'
? close('settings')
: select('settings', { focus: true });
try {
listen(ui.voiceSettingsButton, 'click', toggleSettings);
tabs.forEach((tab, index) => {
listen(tab, 'click', () =>
select(index === 0 ? 'preview' : 'settings', { focus: true }),
);
listen(tab, 'keydown', (event) => {
const next =
event.key === 'Home'
? 0
: event.key === 'End'
? 1
: ['ArrowLeft', 'ArrowRight'].includes(event.key)
? 1 - index
: null;
if (next === null) return;
event.preventDefault();
select(next === 0 ? 'preview' : 'settings', { focus: true });
});
});
listen(ui.voiceSettingsClose, 'click', () => close());
listen(panel, 'keydown', (event) => {
if (event.key !== 'Escape' || !open) return;
event.preventDefault();
close();
});
// Keep the raised row reachable across pinned trays without intercepting
// their controls. The same lifetime owns the listeners and pending timer.
let graceTimer;
const endGrace = () => {
clearTimeout(graceTimer);
if (ui.root) delete ui.root.dataset.rowGrace;
};
listen(ui.root, 'pointerleave', (event) => {
if (event.pointerType === 'touch') return;
endGrace();
ui.root.dataset.rowGrace = 'true';
graceTimer = setTimeout(endGrace, ROW_GRACE_MS);
});
listen(ui.root, 'pointerenter', endGrace);
lifetime.signal.addEventListener('abort', endGrace, { once: true });
sync();
} catch (error) {
lifetime.abort();
throw error;
}
return {
select,
close,
sync,
snapshot,
destroy() {
if (disposed) return;
open = false;
sync();
disposed = true;
lifetime.abort();
},
};
}
src/voice/panel/manualPanel.test.mjs
+198
View File
@@ -0,0 +1,198 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createManualVoicePanel } from './manualPanel.js';
function fixture(t) {
const doc = { activeElement: null };
const node = () =>
Object.assign(new EventTarget(), {
hidden: false,
isConnected: true,
dataset: {},
attributes: {},
ownerDocument: doc,
setAttribute(name, value) {
this.attributes[name] = value;
},
focus() {
doc.activeElement = this;
},
contains(other) {
return other === this;
},
checkVisibility() {
return !this.hidden;
},
});
const ui = Object.fromEntries(
[
'root',
'voicePanel',
'previewPanel',
'voiceSettingsPanel',
'previewTab',
'settingsTab',
'voiceSettingsButton',
'voiceSettingsClose',
'previewEmpty',
'card',
'button',
].map((key) => [key, node()]),
);
ui.voicePanel.contains = (element) =>
Object.values(ui).includes(element) &&
![ui.root, ui.voiceSettingsButton, ui.button].includes(element);
let enabled = true,
content = false,
intents = 0;
const card = {
setPresented(value) {
this.presented = value;
ui.card.hidden = !(value && enabled && content);
},
};
const copy = {
previewEmpty: 'Habla para ver resultados.',
previewDisabled: 'Vista previa desactivada.',
};
const panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => enabled,
copy,
onPresentationIntent: () => intents++,
});
t.after(() => panel.destroy());
return {
ui,
panel,
doc,
card,
copy,
intents: () => intents,
enabled(value) {
enabled = value;
panel.sync();
},
content(value) {
content = value;
panel.sync();
},
};
}
const click = (element) => element.dispatchEvent(new Event('click'));
function key(element, value) {
const event = new Event('keydown', { cancelable: true });
Object.defineProperty(event, 'key', { value });
element.dispatchEvent(event);
return event;
}
test('Settings launcher toggles while tab and programmatic selection remain idempotent', (t) => {
const f = fixture(t);
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.ui.voiceSettingsButton);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.doc.activeElement, f.ui.settingsTab);
assert.equal(f.ui.voiceSettingsButton.attributes['aria-expanded'], 'true');
click(f.ui.voiceSettingsButton);
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.ui.settingsTab);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('settings').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
click(f.ui.settingsTab);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('unknown').reason, 'unknown-view');
});
test('tab keyboard navigation and Escape preserve focus and accessibility state', (t) => {
const f = fixture(t);
click(f.ui.voiceSettingsButton);
assert.equal(key(f.ui.settingsTab, 'ArrowLeft').defaultPrevented, true);
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(f.doc.activeElement, f.ui.previewTab);
assert.equal(f.ui.previewTab.tabIndex, 0);
assert.equal(f.ui.settingsTab.tabIndex, -1);
assert.equal(f.ui.previewTab.attributes['aria-selected'], 'true');
key(f.ui.previewTab, 'End');
assert.equal(f.panel.snapshot().view, 'settings');
key(f.ui.settingsTab, 'Home');
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, true);
assert.equal(f.panel.snapshot().panelOpen, false);
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, false);
});
test('manual switching transfers hidden-panel focus without stealing unrelated focus', (t) => {
const f = fixture(t);
f.panel.select('settings');
f.ui.voiceSettingsPanel.focus();
f.panel.select('preview');
assert.equal(f.doc.activeElement, f.ui.previewTab);
const unrelated = (f.doc.activeElement = {});
f.panel.select('settings');
assert.equal(f.doc.activeElement, unrelated);
assert.equal(f.panel.close('preview').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
});
test('manual Preview selection never enables a disabled preference and uses injected copy', (t) => {
const f = fixture(t);
f.enabled(false);
f.content(true);
f.panel.select('preview');
assert.equal(f.panel.snapshot().previewEnabled, false);
assert.equal(f.panel.snapshot().previewVisible, false);
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewDisabled);
assert.equal(f.ui.previewEmpty.hidden, false);
f.enabled(true);
assert.equal(f.panel.snapshot().previewVisible, true);
assert.equal(f.ui.previewEmpty.hidden, true);
f.content(false);
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewEmpty);
});
test('disposal hides owned presentation and removes listeners without destroying the card', (t) => {
const f = fixture(t);
f.panel.select('settings');
const count = f.intents();
f.panel.destroy();
f.panel.destroy();
click(f.ui.voiceSettingsButton);
click(f.ui.settingsTab);
assert.equal(f.intents(), count);
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.card.presented, false);
assert.equal(f.panel.select('preview').reason, 'disposed');
assert.equal(f.panel.close().reason, 'disposed');
});
test('pointer grace expires, resets on reentry, ignores touch and clears on disposal', (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const f = fixture(t);
const pointer = (type, pointerType = 'mouse') => {
const event = new Event(type);
Object.defineProperty(event, 'pointerType', { value: pointerType });
f.ui.root.dispatchEvent(event);
};
pointer('pointerleave');
t.mock.timers.tick(699);
assert.equal(f.ui.root.dataset.rowGrace, 'true');
pointer('pointerenter');
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
t.mock.timers.tick(699);
assert.equal(f.ui.root.dataset.rowGrace, 'true');
t.mock.timers.tick(1);
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave', 'touch');
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
f.panel.destroy();
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
t.mock.timers.tick(700);
assert.equal(f.ui.root.dataset.rowGrace, undefined);
});
src/voice/panel/nativeActivity.js
+212
View File
@@ -0,0 +1,212 @@
/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
export function observeNativeActivity({
adapter,
nativeProvider,
signal,
onChange,
onInput,
}) {
const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
// Do not even inspect custom controller properties.
const controller = native ? adapter.controller : null;
const input = native
? nativeProvider === 'gemini'
? controller.input
: controller._input
: null;
let dead = false,
connected = false,
inputBusy = false;
function cleanNativeOutput() {
if (nativeProvider === 'gemini') return true;
if (nativeProvider !== 'openai') return false;
const turns = controller._turns;
return (
['idle', 'connecting', 'error'].includes(controller.status) &&
controller.dc == null &&
!turns.assistantAudioPlaying &&
!turns.responseActive &&
!turns.responseCreatePending &&
!turns.userTurnPending &&
!turns.activeResponseId &&
!turns.progressResponse
);
}
let outputKnown = cleanNativeOutput();
let audio = null,
version = 0;
const restores = [];
function snapshot() {
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
return {
known: native && outputKnown && !dead,
connected: connected && !dead,
held: Boolean(input?.pushToTalkKeyHeld),
inputBusy: Boolean(
inputBusy ||
(nativeProvider === 'gemini' && controller?.userActivityOpen),
),
outputBusy: Boolean(
nativeProvider === 'gemini'
? audio && audio === controller.audio && audio.pending
: turns?.assistantAudioPlaying,
),
generationBusy: Boolean(
turns?.responseActive ||
turns?.responseCreatePending ||
(nativeProvider === 'openai' && turns?.userTurnPending) ||
(nativeProvider === 'gemini' &&
(turns?.replyPending ||
turns?.retiredReplyPending ||
turns?.deferredInput ||
turns?.automaticTurnOpen ||
turns?.pendingTools ||
turns?.controlQueue?.length)),
),
};
}
function notify() {
if (!dead) {
try {
onChange?.(snapshot());
} catch {}
}
}
function wrap(object, key, after) {
const original = object?.[key];
if (typeof original !== 'function') return;
function wrapped(...args) {
const stamp = version;
const result = Reflect.apply(original, this, args);
if (!dead && stamp === version) {
try {
after?.(args, result);
} catch {}
}
return result;
}
object[key] = wrapped;
restores.push(() => {
if (object[key] === wrapped) object[key] = original;
});
}
function handle(event) {
if (dead || !native) return;
if (event?.type === 'state') {
const state = event.state;
const wasConnected = connected;
connected = !['idle', 'connecting', 'error'].includes(state);
// Connecting has multiple native phases around audio allocation. Retire
// a previous connected generation once, not each phase of this start.
if (
state === 'idle' ||
state === 'error' ||
(state === 'connecting' && wasConnected)
) {
version++;
audio = null;
inputBusy = false;
outputKnown = cleanNativeOutput();
}
}
notify();
}
if (native) {
wrap(controller, 'sendTextCommand', ([text], result) => {
if (String(text || '').trim() && result !== false) onInput?.();
});
for (const key of [
'setMicrophoneEnabled',
'releasePushToTalkKey',
'resetSession',
])
wrap(input, key, notify);
if (nativeProvider === 'openai') {
wrap(controller._turns, 'observeSpeech', ([event]) => {
if (event?.type === 'input_audio_buffer.speech_started')
inputBusy = true;
if (event?.type === 'input_audio_buffer.speech_stopped')
inputBusy = false;
// A previous turn's drain does not prove the next transport emits output events.
if (event?.type === 'response.created') outputKnown = false;
if (
event?.type === 'output_audio_buffer.started' ||
event?.type === 'output_audio_buffer.cleared'
)
outputKnown = true;
notify();
});
} else {
wrap(controller, 'handleMessage', notify);
const original = controller.createAudio;
function createAudio(...args) {
const stamp = version;
const options = args[0];
let current;
const decorated = { ...options };
for (const key of [
'onAudioStart',
'onAudioEnd',
'onAudioDiscard',
'onDrain',
]) {
const callback = options?.[key];
decorated[key] = function (...values) {
const result =
typeof callback === 'function'
? Reflect.apply(
callback,
this === decorated ? options : this,
values,
)
: undefined;
if (
!dead &&
stamp === version &&
audio === current &&
controller.audio === current
) {
if (
key === 'onAudioStart' &&
options.pushToTalk &&
controller.captureOwner?.admitted
)
inputBusy = true;
if (key === 'onAudioEnd' || key === 'onAudioDiscard')
inputBusy = false;
notify();
}
return result;
};
}
const forwarded = args.slice();
forwarded[0] = decorated;
current = Reflect.apply(original, this, forwarded);
if (!dead && stamp === version) {
audio = current;
inputBusy = false;
for (const key of ['play', 'clearPlayback', 'close'])
wrap(current, key, () => {
if (audio === current && controller.audio === current) notify();
});
}
return current;
}
controller.createAudio = createAudio;
restores.push(() => {
if (controller.createAudio === createAudio)
controller.createAudio = original;
});
}
}
function destroy() {
if (dead) return;
dead = true;
version++;
for (const restore of restores.reverse()) restore();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
return { snapshot, handle, destroy };
}
src/voice/panel/nativeActivity.test.mjs
+473
View File
@@ -0,0 +1,473 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
import { RealtimeTurns } from '../realtimeTurns.js';
import { observeNativeActivity } from './nativeActivity.js';
// Deliberately small native-shaped fixtures exercise wrapper contracts, not devices.
function gemini(onChange = () => {}) {
const callbacks = [];
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {
this.pushToTalkKeyHeld = false;
},
};
const c = {
input,
captureOwner: { admitted: false },
createAudio(opts) {
callbacks.push(opts);
return {
pending: 0,
play() {
this.pending++;
return 42;
},
clearPlayback() {
this.pending = 0;
},
close() {
this.pending = 0;
},
};
},
handleMessage() {},
};
const observer = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
onChange,
});
const start = () => {
observer.handle({ type: 'state', state: 'connecting' });
c.audio = c.createAudio({ pushToTalk: true });
observer.handle({ type: 'state', state: 'listening' });
return c.audio;
};
return { c, input, observer, callbacks, start };
}
test('custom host never inspected, including hostile controller getter', () => {
const adapter = {
get controller() {
throw Error('must not inspect');
},
};
const o = observeNativeActivity({ adapter, nativeProvider: null });
assert.equal(o.snapshot().known, false);
o.handle({ type: 'state', state: 'listening' });
o.destroy();
});
test('PTT hold is direct before connect; silent automatic stream is not admitted speech', () => {
const f = gemini();
f.input.pushToTalkKeyHeld = true;
assert.equal(f.observer.snapshot().held, true);
assert.equal(f.observer.snapshot().connected, false);
f.start();
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, false);
f.c.captureOwner.admitted = true;
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, true);
f.callbacks[0].onAudioDiscard();
f.input.releasePushToTalkKey();
assert.equal(f.observer.snapshot().inputBusy, false);
assert.equal(f.observer.snapshot().held, false);
});
test('queue-to-drain, clear and close use current instance, without an acoustic claim', () => {
const f = gemini();
const a = f.start();
assert.equal(a.play(), 42);
assert.equal(f.observer.snapshot().outputBusy, true);
a.clearPlayback();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.close();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.pending = 0;
f.callbacks[0].onDrain();
assert.equal(f.observer.snapshot().outputBusy, false);
});
test('same-mount restart old callbacks cannot clear new admitted capture/output', () => {
const f = gemini();
f.start();
const old = f.callbacks[0];
const b = f.start();
f.c.captureOwner.admitted = true;
f.callbacks[1].onAudioStart();
b.play();
old.onAudioDiscard();
old.onDrain();
assert.equal(f.observer.snapshot().inputBusy, true);
assert.equal(f.observer.snapshot().outputBusy, true);
});
test('native callbacks original first with receiver, arguments, return and thrown identity; observation failure isolated', () => {
const f = gemini(() => {
throw Error('observer');
});
const receiver = {};
const argument = {};
let seen;
f.c.audio = f.c.createAudio({
pushToTalk: true,
onAudioEnd: function (...args) {
seen = [this, args];
return argument;
},
});
const cb = f.callbacks[0];
assert.equal(cb.onAudioEnd.call(receiver, argument), argument);
assert.equal(seen[0], receiver);
assert.equal(seen[1][0], argument);
const error = Error('same');
f.c.audio = f.c.createAudio({
onDrain() {
throw error;
},
});
assert.throws(
() => f.callbacks[1].onDrain(),
(e) => e === error,
);
});
test('reentrant Stop in original callback suppresses stale post-callback observation; teardown preserves newer replacement', () => {
let count = 0;
const f = gemini(() => count++);
f.c.audio = f.c.createAudio({
onDrain() {
f.observer.handle({ type: 'state', state: 'idle' });
},
});
const before = count;
f.callbacks[0].onDrain();
assert.equal(count, before + 1);
assert.equal(f.observer.snapshot().connected, false);
const newer = () => {};
f.input.releasePushToTalkKey = newer;
f.observer.destroy();
assert.equal(f.input.releasePushToTalkKey, newer);
});
test('OpenAI response.done does not drain output; absent lifecycle remains unknown', () => {
const turns = {
assistantAudioPlaying: false,
responseActive: false,
observeSpeech(e) {
if (e.type === 'output_audio_buffer.started')
this.assistantAudioPlaying = true;
if (
e.type === 'output_audio_buffer.stopped' ||
e.type === 'output_audio_buffer.cleared'
)
this.assistantAudioPlaying = false;
},
};
const c = { _turns: turns, _input: { pushToTalkKeyHeld: false } };
const original = turns.observeSpeech;
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'openai',
});
o.handle({ type: 'state', state: 'listening' });
assert.equal(o.snapshot().known, false);
turns.observeSpeech({ type: 'output_audio_buffer.started' });
assert.equal(o.snapshot().known, true);
turns.responseActive = false;
o.handle({ type: 'completion' });
assert.equal(o.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.cleared' });
assert.equal(o.snapshot().outputBusy, false);
o.destroy();
assert.equal(turns.observeSpeech, original);
});
test('method receiver/args/result and throw are exact; abort restores wrappers', () => {
const abort = new AbortController();
const result = Promise.resolve();
let seen;
const error = Error('original');
const input = {
setMicrophoneEnabled(...args) {
seen = [this, args];
if (args[0] === 'throw') throw error;
return result;
},
};
const original = input.setMicrophoneEnabled;
const c = {
input,
createAudio() {
return {};
},
handleMessage() {},
};
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
signal: abort.signal,
onChange() {
throw Error('observe');
},
});
assert.equal(input.setMicrophoneEnabled(1, 2), result);
assert.equal(seen[0], input);
assert.deepEqual(seen[1], [1, 2]);
assert.throws(
() => input.setMicrophoneEnabled('throw'),
(e) => e === error,
);
abort.abort();
assert.equal(input.setMicrophoneEnabled, original);
assert.equal(o.snapshot().known, false);
});
test('options-method callback receiver maps to original options, explicit receiver is preserved', () => {
const f = gemini();
let receiver;
const options = {
onDrain() {
receiver = this;
return 17;
},
};
f.c.audio = f.c.createAudio(options);
assert.equal(f.callbacks[0].onDrain(), 17);
assert.equal(receiver, options);
const explicit = {};
f.callbacks[0].onDrain.call(explicit);
assert.equal(receiver, explicit);
});
test('actual native RealtimeTurns output lifecycle remains busy beyond generation', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: { controller: { _turns: turns, _input: {} } },
nativeProvider: 'openai',
});
observer.handle({ type: 'state', state: 'listening' });
turns.observeSpeech({ type: 'output_audio_buffer.started' });
turns.responseActive = false;
observer.handle({ type: 'completion' });
assert.equal(observer.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().outputBusy, false);
turns.observeSpeech({ type: 'response.created' });
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('verified clean native mount can arm initial idle, eventless response invalidates it', () => {
const turns = new RealtimeTurns({});
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
let armed = 0,
cleared = 0;
const idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer() {
armed++;
return 1;
},
clearTimer() {
cleared++;
},
});
assert.equal(observer.snapshot().known, true);
assert.equal(armed, 0);
controller.status = 'listening';
controller.dc = { readyState: 'open' };
observer.handle({ type: 'state', state: 'listening' });
idle.update();
assert.equal(armed, 1);
turns.observeSpeech({ type: 'response.created' });
idle.update();
assert.equal(observer.snapshot().known, false);
assert.equal(cleared, 1);
turns.responseActive = false;
observer.handle({ type: 'completion' });
idle.update();
assert.equal(armed, 1);
assert.equal(observer.snapshot().known, false);
controller.status = 'idle';
controller.dc = null;
turns.reset();
observer.handle({ type: 'state', state: 'idle' });
assert.equal(observer.snapshot().known, true);
idle.destroy();
observer.destroy();
});
test('attaching to already active native transport never assumes empty playback', () => {
const turns = new RealtimeTurns({});
for (const controller of [
{ status: 'listening', dc: {}, _turns: turns, _input: {} },
{ status: 'idle', dc: {}, _turns: turns, _input: {} },
]) {
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
}
turns.responseActive = true;
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('Gemini queue-only work blocks idle, queue drain rearms, expiry rereads delayed queue', () => {
const f = gemini();
f.start();
f.c.controlQueue = [];
let callback,
arms = 0,
clears = 0,
expired = 0;
const idle = createVoiceInactivity({
readActivity: f.observer.snapshot,
readPending: () => 0,
onExpire() {
expired++;
},
setTimer(fn) {
callback = fn;
return ++arms;
},
clearTimer() {
clears++;
},
});
assert.equal(arms, 1);
f.c.controlQueue.push({});
assert.equal(f.observer.snapshot().generationBusy, true);
idle.update();
assert.equal(clears, 1);
f.c.controlQueue.shift();
assert.equal(f.observer.snapshot().generationBusy, false);
idle.update();
assert.equal(arms, 2);
// A queue may change after the last event-driven snapshot and before the deadline.
f.c.controlQueue.push({});
callback();
assert.equal(expired, 0);
f.c.controlQueue = [];
idle.update();
assert.equal(arms, 3);
callback();
assert.equal(expired, 1);
idle.destroy();
f.observer.destroy();
});
test('Gemini presentation-only pending does not block idle; retired/deferred work still does', () => {
const f = gemini();
f.start();
f.c.userTurnPending = true;
assert.equal(f.observer.snapshot().generationBusy, false);
f.c.retiredReplyPending = true;
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.retiredReplyPending = false;
f.c.deferredInput = {};
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.deferredInput = null;
assert.equal(f.observer.snapshot().generationBusy, false);
f.observer.destroy();
});
test('actual Gemini startup phases keep playback observation and permit quiet open-mic expiry', async () => {
const { harness } = await import('../../testSupport/geminiNativeHarness.mjs');
let observer,
idle,
deadline,
expiries = 0;
const h = harness({ emit: (event) => observer?.handle(event) });
observer = observeNativeActivity({
adapter: { controller: h.c },
nativeProvider: 'gemini',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer(fn) {
deadline = fn;
return fn;
},
clearTimer() {},
onExpire() {
expiries++;
},
});
try {
await h.c.start();
assert.equal(h.c.userTurnPending, true);
assert.equal(idle.snapshot().eligible, true);
const beforeOutput = deadline;
h.sockets[0].message({
serverContent: {
modelTurn: {
parts: [
{
inlineData: { data: 'output', mimeType: 'audio/pcm;rate=24000' },
},
],
},
turnComplete: true,
},
});
assert.equal(observer.snapshot().outputBusy, true);
assert.equal(idle.snapshot().eligible, false);
beforeOutput();
assert.equal(expiries, 0);
h.captures[0].drain();
assert.equal(observer.snapshot().outputBusy, false);
assert.equal(idle.snapshot().eligible, true);
deadline();
assert.equal(expiries, 1);
} finally {
idle.destroy();
observer.destroy();
h.c.stop({ removeUi: true });
}
});
test('native typed input observation preserves receiver/result, refuses failed submissions and retires old wrapper', () => {
let inputs = 0;
const controller = {
input: {},
handleMessage() {},
createAudio() {},
sendTextCommand(text) {
assert.equal(this, controller);
if (text === 'throw') throw new Error('rejected');
return text === 'reject' ? false : undefined;
},
};
const original = controller.sendTextCommand;
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'gemini',
onInput: () => inputs++,
});
const old = controller.sendTextCommand;
assert.equal(controller.sendTextCommand('typed request'), undefined);
controller.sendTextCommand('reject');
controller.sendTextCommand('');
assert.throws(() => controller.sendTextCommand('throw'), /rejected/);
assert.equal(inputs, 1);
observer.destroy();
assert.equal(controller.sendTextCommand, original);
old.call(controller, 'late');
assert.equal(inputs, 1);
});
src/voice/panel/preferences.js
+194
View File
@@ -0,0 +1,194 @@
export const VOICE_PREVIEW_STORAGE_KEY = 'godsEyeView.voice.previewEnabled';
export const VOICE_TIMEOUT_STORAGE_KEY = 'godsEyeView.voice.inactivityMinutes';
const PRESETS = ['3', '5', '10', '15'];
function parseMinutes(value) {
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
if (typeof value === 'string' && !/^\d+$/.test(value)) return undefined;
const number = Number(value);
return Number.isInteger(number) && number >= 0 && number <= 60
? number
: undefined;
}
/** Outer, browser-local live state. Construction never writes storage. */
export function createVoicePreferences({ storage } = {}) {
const listeners = new Set();
const target = () =>
storage === undefined ? globalThis.localStorage : storage;
const read = (key) => {
try {
return target()?.getItem?.(key);
} catch {
return undefined;
}
};
const rawMinutes = read(VOICE_TIMEOUT_STORAGE_KEY);
let state = {
previewEnabled: read(VOICE_PREVIEW_STORAGE_KEY) !== 'false',
minutes: rawMinutes === 'none' ? 0 : (parseMinutes(rawMinutes) ?? 5),
previewPersisted: null,
timeoutPersisted: null,
};
const snapshot = () => ({ ...state });
function apply(field, status, key, value, serialized) {
const changed = state[field] !== value;
let persisted = false;
state = { ...state, [field]: value };
try {
const destination = target();
if (
typeof destination?.setItem === 'function' &&
typeof destination?.getItem === 'function'
) {
destination.setItem(key, serialized);
persisted = destination.getItem(key) === serialized;
}
} catch {
/* The applied live value remains authoritative across mounts. */
}
state = { ...state, [status]: persisted };
const result = {
ok: true,
applied: true,
changed,
persisted,
...snapshot(),
};
for (const listener of [...listeners]) listener(snapshot());
return result;
}
const invalid = () => ({
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
...snapshot(),
});
return {
snapshot,
setPreviewEnabled(value) {
return typeof value === 'boolean'
? apply(
'previewEnabled',
'previewPersisted',
VOICE_PREVIEW_STORAGE_KEY,
value,
String(value),
)
: invalid();
},
setMinutes(value) {
const parsed = parseMinutes(value);
return parsed === undefined
? invalid()
: apply(
'minutes',
'timeoutPersisted',
VOICE_TIMEOUT_STORAGE_KEY,
parsed,
parsed === 0 ? 'none' : String(parsed),
);
},
subscribe(listener) {
if (typeof listener !== 'function')
throw new TypeError('Preference listener required');
listeners.add(listener);
return () => listeners.delete(listener);
},
};
}
/** Mount-local controls, backed by an independently owned preference model. */
export function bindPreferenceControls({ ui, preferences, copy, onChange }) {
const {
voicePreviewInput: preview,
voicePreviewNote: previewNote,
voiceTimeoutSelect: select,
voiceTimeoutInput: input,
voiceTimeoutNote: timeoutNote,
} = ui;
let disposed = false;
const initial = preferences.snapshot();
const displayed = parseMinutes(input?.value);
let custom =
!PRESETS.includes(String(initial.minutes)) && initial.minutes > 0
? initial.minutes
: displayed > 0
? displayed
: 5;
function render(state) {
if (disposed) return;
if (preview) preview.checked = state.previewEnabled;
if (previewNote)
previewNote.textContent = `${copy.previewNote}${state.previewPersisted === false ? ` ${copy.notSaved}` : ''}`;
if (timeoutNote)
timeoutNote.textContent = `${copy.timeoutNote}${state.timeoutPersisted === false ? ` ${copy.notSaved}` : ''}`;
const value = String(state.minutes);
const customSelected = state.minutes > 0 && !PRESETS.includes(value);
if (customSelected) custom = state.minutes;
if (select)
select.value =
state.minutes === 0 ? '0' : customSelected ? 'custom' : value;
if (input) {
input.value = String(custom);
input.hidden = !customSelected;
}
}
const unsubscribe = preferences.subscribe((state) => {
render(state);
onChange?.(state);
});
const previewChange = () => {
if (!disposed && !preferences.setPreviewEnabled(preview.checked).ok)
render(preferences.snapshot());
};
const selectChange = () => {
if (disposed) return;
const value = select.value;
if (value === 'custom') {
const parsed = parseMinutes(input?.value);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
// A custom value may equal a preset: retain the user's explicit editor choice.
select.value = 'custom';
if (input) input.hidden = false;
} else if (value === '0' || PRESETS.includes(value))
preferences.setMinutes(value);
else render(preferences.snapshot());
};
const inputChange = () => {
if (disposed) return;
const parsed = parseMinutes(input.value);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
if (select) select.value = 'custom';
input.hidden = false;
};
const bindings = [
[preview, previewChange],
[select, selectChange],
[input, inputChange],
];
for (const [element, listener] of bindings)
element?.addEventListener('change', listener);
render(initial);
return {
dispose() {
if (disposed) return;
disposed = true;
unsubscribe();
for (const [element, listener] of bindings)
element?.removeEventListener('change', listener);
},
};
}
src/voice/panel/preferences.test.mjs
+272
View File
@@ -0,0 +1,272 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createVoicePreferences,
bindPreferenceControls,
VOICE_PREVIEW_STORAGE_KEY as P,
VOICE_TIMEOUT_STORAGE_KEY as T,
} from './preferences.js';
function storage(entries = {}, mode = 'normal') {
const values = new Map(Object.entries(entries));
const writes = [];
return {
writes,
values,
getItem(key) {
if (mode === 'readthrow') throw Error('blocked');
return values.get(key) ?? null;
},
setItem(key, value) {
writes.push([key, value]);
if (mode === 'throw') throw Error('blocked');
if (mode !== 'noop' && mode !== 'readthrow')
values.set(key, mode === 'mismatch' ? `${value}!` : value);
},
};
}
class Element {
value = '';
checked = false;
textContent = '';
hidden = false;
listeners = new Set();
addEventListener(type, listener) {
assert.equal(type, 'change');
this.listeners.add(listener);
}
removeEventListener(type, listener) {
assert.equal(type, 'change');
this.listeners.delete(listener);
}
change(value) {
if (typeof value === 'boolean') this.checked = value;
else this.value = value;
for (const listener of [...this.listeners]) listener();
}
}
function controls(custom = '7') {
const ui = Object.fromEntries(
[
'voicePreviewInput',
'voicePreviewNote',
'voiceTimeoutSelect',
'voiceTimeoutInput',
'voiceTimeoutNote',
].map((key) => [key, new Element()]),
);
ui.voiceTimeoutInput.value = custom;
return ui;
}
const copy = {
previewNote: 'Preview local',
timeoutNote: 'Provider limits still apply',
notSaved: 'Not saved',
};
function bind(ui, preferences, onChange) {
return bindPreferenceControls({ ui, preferences, copy, onChange });
}
test('defaults and restore are passive; legacy None and stored custom restore', () => {
for (const [entries, minutes, previewEnabled] of [
[{}, 5, true],
[{ [P]: 'false', [T]: 'none' }, 0, false],
[{ [T]: '17' }, 17, true],
[{ [T]: 'broken' }, 5, true],
]) {
const disk = storage(entries);
const prefs = createVoicePreferences({ storage: disk });
assert.deepEqual(prefs.snapshot(), {
previewEnabled,
minutes,
previewPersisted: null,
timeoutPersisted: null,
});
const ui = controls();
bind(ui, prefs).dispose();
assert.deepEqual(disk.writes, []);
}
});
test('exact valid values, presets, None and serialized readback', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
for (const value of [3, 5, 10, 15, '1', '60', 0]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, true);
assert.equal(result.applied, true);
assert.equal(result.persisted, true);
assert.equal(result.minutes, Number(value));
assert.deepEqual(disk.writes.at(-1), [
T,
Number(value) === 0 ? 'none' : String(Number(value)),
]);
}
assert.equal(prefs.setPreviewEnabled(false).previewEnabled, false);
assert.deepEqual(disk.writes.at(-1), [P, 'false']);
});
test('fractional, malformed, coercible and out-of-range inputs never write or notify', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
let notices = 0;
prefs.subscribe(() => notices++);
const before = prefs.snapshot();
for (const value of [
null,
undefined,
true,
{},
[],
'',
' ',
' 3',
'3 ',
'3.0',
'1e1',
'0x10',
'none',
-1,
61,
3.2,
NaN,
Infinity,
]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, false, String(value));
assert.equal(result.applied, false);
assert.equal(result.changed, false);
assert.deepEqual(prefs.snapshot(), before);
}
for (const value of [0, 'true', null])
assert.equal(prefs.setPreviewEnabled(value).ok, false);
assert.equal(notices, 0);
assert.deepEqual(disk.writes, []);
});
for (const mode of ['throw', 'noop', 'mismatch', 'readthrow', 'missing']) {
test(`storage ${mode} preserves live value and exposes unsaved state across A → B`, () => {
const disk = mode === 'missing' ? null : storage({}, mode);
const prefs = createVoicePreferences({ storage: disk });
const a = controls();
const bindingA = bind(a, prefs);
a.voicePreviewInput.change(false);
a.voiceTimeoutSelect.change('custom');
a.voiceTimeoutInput.change('19');
assert.deepEqual(prefs.snapshot(), {
previewEnabled: false,
minutes: 19,
previewPersisted: false,
timeoutPersisted: false,
});
bindingA.dispose();
const writeCount = disk?.writes.length;
const b = controls();
const bindingB = bind(b, prefs);
assert.equal(b.voicePreviewInput.checked, false);
assert.equal(b.voiceTimeoutInput.value, '19');
assert.match(b.voicePreviewNote.textContent, /Not saved/);
assert.match(b.voiceTimeoutNote.textContent, /Not saved/);
assert.equal(disk?.writes.length, writeCount);
a.voicePreviewInput.change(true);
a.voiceTimeoutSelect.change('3');
assert.equal(prefs.snapshot().minutes, 19);
assert.equal(prefs.snapshot().previewEnabled, false);
bindingB.dispose();
});
}
test('same-value retry really writes and clears warnings after storage recovers', () => {
const disk = storage();
const original = disk.setItem;
disk.setItem = () => {
throw Error('blocked');
};
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
prefs.setPreviewEnabled(false);
prefs.setMinutes(12);
assert.match(ui.voiceTimeoutNote.textContent, /Not saved/);
disk.setItem = original;
const timeout = prefs.setMinutes(12);
const preview = prefs.setPreviewEnabled(false);
assert.equal(timeout.changed, false);
assert.equal(preview.changed, false);
assert.equal(timeout.persisted, true);
assert.equal(preview.persisted, true);
assert.deepEqual(disk.writes, [
[T, '12'],
[P, 'false'],
]);
assert.equal(ui.voiceTimeoutNote.textContent, copy.timeoutNote);
assert.equal(ui.voicePreviewNote.textContent, copy.previewNote);
});
test('Custom commits displayed valid value immediately, even when equal to preset', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
const ui = controls('10');
bind(ui, prefs);
assert.equal(disk.writes.length, 0);
ui.voiceTimeoutSelect.change('custom');
assert.equal(prefs.snapshot().minutes, 10);
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
assert.equal(ui.voiceTimeoutInput.hidden, false);
assert.deepEqual(disk.writes, [[T, '10']]);
ui.voiceTimeoutInput.change('23');
ui.voiceTimeoutSelect.change('5');
assert.equal(ui.voiceTimeoutInput.value, '23');
assert.equal(ui.voiceTimeoutInput.hidden, true);
ui.voiceTimeoutSelect.change('custom');
assert.equal(prefs.snapshot().minutes, 23);
assert.equal(ui.voiceTimeoutInput.value, '23');
});
test('invalid control edits restore valid state without writes; None remains distinct', () => {
const disk = storage({ [T]: '17' });
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
assert.equal(ui.voiceTimeoutInput.value, '17');
for (const value of ['1.5', '', '61', '0']) {
ui.voiceTimeoutInput.change(value);
assert.equal(ui.voiceTimeoutInput.value, '17');
assert.equal(prefs.snapshot().minutes, 17);
}
ui.voiceTimeoutSelect.change('malformed');
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
assert.equal(disk.writes.length, 0);
ui.voiceTimeoutSelect.change('0');
assert.equal(prefs.snapshot().minutes, 0);
assert.equal(ui.voiceTimeoutInput.hidden, true);
ui.voiceTimeoutInput.value = 'bad';
ui.voiceTimeoutSelect.change('custom');
assert.equal(ui.voiceTimeoutSelect.value, '0');
assert.equal(prefs.snapshot().minutes, 0);
assert.deepEqual(disk.writes, [[T, 'none']]);
});
test('model snapshots cannot mutate state; unsubscribe and binder disposal release ownership', () => {
const prefs = createVoicePreferences({ storage: storage() });
const ui = controls();
let calls = 0;
const binding = bind(ui, prefs, () => calls++);
const snap = prefs.snapshot();
snap.minutes = 41;
assert.equal(prefs.snapshot().minutes, 5);
const unsub = prefs.subscribe((state) => {
state.minutes = 22;
});
prefs.setMinutes(11);
assert.equal(calls, 1);
assert.equal(prefs.snapshot().minutes, 11);
unsub();
binding.dispose();
binding.dispose();
prefs.setMinutes(12);
assert.equal(calls, 1);
assert.equal(ui.voiceTimeoutInput.value, '11');
for (const element of Object.values(ui))
assert.equal(element.listeners.size, 0);
});
src/voice/panel/testDom.mjs
+204
View File
@@ -0,0 +1,204 @@
// Minimal DOM fixture for actual control/card/binder composition, not layout proof.
class Element {
constructor(doc, tag = 'div') {
Object.assign(this, {
ownerDocument: doc,
tagName: tag.toUpperCase(),
children: [],
attributes: new Map(),
dataset: {},
hidden: false,
value: '',
checked: false,
listeners: new Map(),
style: { setProperty() {}, removeProperty() {} },
});
this.classList = {
add: (...names) => names.forEach((n) => this.classes.add(n)),
remove: (...names) => names.forEach((n) => this.classes.delete(n)),
contains: (name) => this.classes.has(name),
};
this.classes = new Set();
}
set id(value) {
this.attributes.set('id', value);
}
get id() {
return this.attributes.get('id') || '';
}
set className(value) {
this.classes = new Set(value.split(/\s+/).filter(Boolean));
}
get className() {
return [...this.classes].join(' ');
}
setAttribute(name, value) {
this.attributes.set(name, String(value));
if (name === 'class') this.className = value;
if (name === 'hidden') this.hidden = true;
if (name === 'value') this.value = value;
if (name.startsWith('data-'))
this.dataset[
name.slice(5).replace(/-([a-z])/g, (_, c) => c.toUpperCase())
] = value;
}
getAttribute(name) {
return this.attributes.get(name) ?? null;
}
get isConnected() {
return (
this === this.ownerDocument.body || Boolean(this.parentNode?.isConnected)
);
}
get parentElement() {
return this.parentNode;
}
set innerHTML(html) {
this.replaceChildren();
const stack = [this];
const voids = new Set(['INPUT', 'IMG', 'BR', 'HR']);
for (const match of html.matchAll(/<(\/?)([a-z][\w-]*)\b([^>]*)>/gi)) {
if (match[1]) {
if (stack.length > 1) stack.pop();
continue;
}
const node = new Element(this.ownerDocument, match[2]);
for (const attr of match[3].matchAll(/([^\s=\/]+)(?:="([^"]*)")?/g))
node.setAttribute(attr[1], attr[2] ?? '');
stack.at(-1).appendChild(node);
if (!voids.has(node.tagName)) stack.push(node);
}
}
set textContent(value) {
this.text = String(value);
}
get textContent() {
return this.text ?? '';
}
appendChild(node) {
node.remove();
this.children.push(node);
node.parentNode = this;
return node;
}
append(...nodes) {
nodes.forEach((node) => this.appendChild(node));
}
insertBefore(node, before) {
node.remove();
const at = this.children.indexOf(before);
this.children.splice(at < 0 ? this.children.length : at, 0, node);
node.parentNode = this;
}
replaceChildren(...nodes) {
for (const child of this.children) child.parentNode = null;
this.children = [];
this.append(...nodes);
}
remove() {
if (this.parentNode)
this.parentNode.children = this.parentNode.children.filter(
(n) => n !== this,
);
this.parentNode = null;
}
contains(node) {
return node === this || this.children.some((child) => child.contains(node));
}
matches(selector) {
if (selector.startsWith('#')) return this.id === selector.slice(1);
if (selector.startsWith('.')) return this.classes.has(selector.slice(1));
return this.tagName === selector.toUpperCase();
}
querySelectorAll(selector) {
return this.children.flatMap((n) => [
...(n.matches(selector) ? [n] : []),
...n.querySelectorAll(selector),
]);
}
querySelector(selector) {
return this.querySelectorAll(selector)[0] || null;
}
checkVisibility() {
return (
this.isConnected &&
!this.hidden &&
(!this.parentNode || this.parentNode.checkVisibility())
);
}
focus() {
this.ownerDocument.activeElement = this;
}
addEventListener(type, listener, options = {}) {
if (options.signal?.aborted) return;
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
this.listeners.get(type).add(listener);
if (options.signal)
options.signal.addEventListener(
'abort',
() => this.removeEventListener(type, listener),
{ once: true },
);
}
removeEventListener(type, listener) {
this.listeners.get(type)?.delete(listener);
}
fire(type, fields = {}) {
const event = {
type,
target: this,
preventDefault() {
this.defaultPrevented = true;
},
...fields,
};
for (const listener of [...(this.listeners.get(type) || [])])
listener.call(this, event);
return event;
}
}
const installed = new WeakMap();
export function installPanelDom(t) {
if (installed.has(t)) return installed.get(t);
const previous = {
window: globalThis.window,
document: globalThis.document,
Element: globalThis.Element,
};
const doc = new EventTarget();
doc.body = new Element(doc, 'body');
doc.documentElement = new Element(doc, 'html');
doc.documentElement.lang = 'en';
doc.activeElement = doc.body;
doc.createElement = (tag) => new Element(doc, tag);
doc.getElementById = (id) => doc.body.querySelector(`#${id}`);
doc.querySelector = (selector) => doc.body.querySelector(selector);
const win = new EventTarget();
globalThis.window = win;
globalThis.document = doc;
globalThis.Element = Element;
installed.set(t, doc);
t.after(() => Object.assign(globalThis, previous));
return doc;
}
export function controlledClock() {
let now = 0,
next = 0;
const tasks = new Map();
return {
now: () => now,
setTimer(fn, ms) {
const id = ++next;
tasks.set(id, { fn, at: now + ms });
return id;
},
clearTimer: (id) => tasks.delete(id),
advance(ms) {
now += ms;
for (const [id, task] of [...tasks])
if (task.at <= now && tasks.delete(id)) task.fn();
},
};
}
src/voice/sessionCommands.js
+45 -9
View File
@@ -36,15 +36,24 @@ export function createVoiceCommands({
pointer = null,
referents = null,
createReticle = createDefaultReticle,
panelHost = null,
nativePanelProvider = null,
}) {
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({ reset: true });
const ui = createControl({
reset: true,
...(panelHost ? { panel: true } : {}),
});
let actionObserver;
let panelMount;
const session = createVoiceSession({
runner,
signal,
createAdapter: (hooks) =>
createSession({
createAdapter: (hooks) => {
if (panelHost) actionObserver = panelHost.observeActions(hooks);
return createSession({
...hooks,
...(actionObserver ? { runAction: actionObserver.runAction } : {}),
runner,
ui,
dataManager,
@@ -53,7 +62,8 @@ export function createVoiceCommands({
createController,
pointer,
radioLayer: dataManager?.layers?.get('radio')?.module || null,
}),
});
},
});
const adapter = session.adapter;
const capabilities = adapter.capabilities || {};
@@ -117,11 +127,31 @@ export function createVoiceCommands({
});
// Captions, plan and result card nested in the control; it only reads
// provider-neutral session events.
const card = ui.card ? createCard({ root: ui.root }) : null;
const updateCard = card
? session.subscribe((event) => card.handle(event))
const card = ui.card
? createCard({
root: ui.root,
...(panelHost ? { onRender: () => panelMount?.panel.sync() } : {}),
})
: null;
panelMount = panelHost?.mount({
ui,
card,
session,
observer: actionObserver,
nativeProvider: nativePanelProvider,
});
const updatePanel = panelMount ? session.subscribe(panelMount.handle) : null;
const updateCard =
card && !panelMount
? session.subscribe((event) => card.handle(event))
: null;
controls.voiceCard = card;
if (panelMount) {
controls.voicePanel = panelMount.panel;
controls.captureVoicePanelOperation = panelMount.captureOperation;
controls.getVoicePanelState = panelMount.snapshot;
controls.getVoicePreferences = panelHost.snapshot;
}
// Point-and-ask: the reticle follows turn events; pointer and referents are
// session-scoped, so a session that ends forgets both.
const reticle = pointer ? createReticle(pointer) : null;
@@ -151,8 +181,11 @@ export function createVoiceCommands({
);
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (session.isActive()) session.stop();
else void session.start({ pushToTalk: false });
if (session.isActive()) {
const ticket = panelMount?.stopTicket();
session.stop();
panelMount?.closeAfterStop(ticket);
} else void session.start({ pushToTalk: false });
};
ui.button.addEventListener('click', buttonHandler);
let cleaned = false;
@@ -165,6 +198,9 @@ export function createVoiceCommands({
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePanel?.();
panelMount?.destroy();
actionObserver?.dispose();
updatePointer();
forwardEvents?.();
reticle?.destroy();
src/voice/speech.js
+1
View File
@@ -848,6 +848,7 @@ const STEP_TITLES = Object.freeze({
zoom_to_globe: msg('Zoom out to globe'),
show_data_layers_menu: msg('Open data layers'),
set_panel_open: msg('Open or close a panel'),
set_voice_inactivity_timeout: msg('Voice inactivity timeout'),
set_context_mode: msg('Change context mode'),
control_cockpit: msg('Control cockpit'),
set_visual_style: msg('Change visual style'),
src/voice/voiceCard.js
+56 -3
View File
@@ -13,6 +13,7 @@ const ELEMENT_IDS = Object.freeze({
dismiss: 'gev-voice-card-dismiss',
user: 'gev-voice-card-user',
userText: 'gev-voice-card-user-text',
userWho: 'gev-voice-card-user-who',
pointer: 'gev-voice-card-pointer',
pointerKind: 'gev-voice-card-pointer-kind',
pointerText: 'gev-voice-card-pointer-text',
@@ -70,8 +71,9 @@ export class VoiceCardControls {
idleMs = VOICE_CARD_IDLE_MS,
setTimer = (fn, ms) => setTimeout(fn, ms),
clearTimer = (id) => clearTimeout(id),
onRender,
}) {
Object.assign(this, { idleMs, setTimer, clearTimer });
Object.assign(this, { idleMs, setTimer, clearTimer, onRender });
this.elements = Object.fromEntries(
Object.entries(ELEMENT_IDS).map(([key, id]) => [
key,
@@ -79,6 +81,10 @@ export class VoiceCardControls {
]),
);
this.state = initialVoiceCardState();
this.enabled = true;
this.presented = true;
this.announcementRevision = 0;
this.consumedPresentationAnnouncement = -1;
this.hideTimer = null;
this.destroyed = false;
this.listeners = new AbortController();
@@ -96,17 +102,53 @@ export class VoiceCardControls {
if (this.destroyed || !this.elements.card) return;
const next = reduceVoiceCard(this.state, event);
if (next === this.state) return;
if (
next.announce !== this.state.announce ||
(next.announce &&
((event.type === 'transcript' &&
event.role === 'assistant' &&
event.final) ||
(event.type === 'action-result' &&
next.result !== this.state.result) ||
(event.type === 'pointer' &&
next.pointer &&
next.pointer !== this.state.pointer)))
)
this.announcementRevision++;
this.state = next;
this.render();
if (this.state.visible) this.scheduleIdle();
else this.cancelIdle();
}
/** A local submission allows future output after dismissal; keep its content. */
rearm() {
if (this.destroyed || !this.elements.card) return;
this.state = { ...this.state, dismissed: false };
}
/** An annotation outline resolved or failed after its tool result. */
outline(event) {
this.handle({ type: 'outline', id: event?.id, status: event?.status });
}
/** Change presentation only; session events and the idle deadline continue. */
setEnabled(enabled) {
if (this.destroyed || !this.elements.card) return;
this.enabled = Boolean(enabled);
this.render();
}
/** View selection changes presentation without changing the saved preference. */
setPresented(presented) {
if (this.destroyed || !this.elements.card || this.presented === presented)
return;
if (!presented)
this.consumedPresentationAnnouncement = this.announcementRevision;
this.presented = presented;
this.render();
}
scheduleIdle() {
this.cancelIdle();
this.hideTimer = this.setTimer(() => {
@@ -129,12 +171,13 @@ export class VoiceCardControls {
render() {
const el = this.elements;
const view = voiceCardView(this.state);
const view = this.snapshot();
setHidden(el.card, !view.visible);
el.card.dataset.busy = String(view.busy);
setText(el.phase, view.phase);
setHidden(el.user, !view.user.visible);
setText(el.userText, view.user.text);
setText(el.userWho, view.user.label);
setHidden(el.pointer, !view.pointer.visible);
setText(el.pointerKind, view.pointer.kind);
setText(el.pointerText, view.pointer.text);
@@ -186,11 +229,21 @@ export class VoiceCardControls {
node.textContent = note;
});
setText(el.live, view.announce);
this.onRender?.();
}
/** Test/diagnostic view of what the card shows. */
snapshot() {
return voiceCardView(this.state);
const view = voiceCardView(this.state);
return this.enabled && this.presented
? {
...view,
announce:
this.consumedPresentationAnnouncement === this.announcementRevision
? ''
: view.announce,
}
: { ...view, visible: false, announce: '' };
}
/** Idempotent: removes listeners and the idle timer; markup stays owned by the control. */
src/voice/voiceCard.test.mjs
+464 -46
View File
@@ -3,13 +3,17 @@ import { readFileSync } from 'node:fs';
import test from 'node:test';
import { readStylesheet } from '../testSupport/readStylesheet.mjs';
import { VoiceCardControls } from './voiceCard.js';
import { getLocale, setLocale } from '../i18n/index.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
const controlSource = readFileSync(new URL('./control.js', import.meta.url), 'utf8');
const controlSource = readFileSync(
new URL('./control.js', import.meta.url),
'utf8',
);
const css = readStylesheet(new URL('../../style.css', import.meta.url));
function fold(events) {
@@ -19,11 +23,37 @@ function fold(events) {
test('presentation: a slow annotate turn shows captions, plan steps and the result card', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'transcript', role: 'user', text: 'Annotate the Texas State Capitol', final: true },
{ type: 'transcript', role: 'assistant', text: 'Finding the Capitol, ', responseId: 'r1' },
{ type: 'transcript', role: 'assistant', text: 'then outlining it.', responseId: 'r1' },
{ type: 'action-call', name: 'annotate_map', callId: 'call_a', arguments: { annotations: [{ target: 'Texas State Capitol' }] } },
{ type: 'progress', name: 'annotate_map', callId: 'call_a', step: 'resolve', label: 'Texas State Capitol' },
{
type: 'transcript',
role: 'user',
text: 'Annotate the Texas State Capitol',
final: true,
},
{
type: 'transcript',
role: 'assistant',
text: 'Finding the Capitol, ',
responseId: 'r1',
},
{
type: 'transcript',
role: 'assistant',
text: 'then outlining it.',
responseId: 'r1',
},
{
type: 'action-call',
name: 'annotate_map',
callId: 'call_a',
arguments: { annotations: [{ target: 'Texas State Capitol' }] },
},
{
type: 'progress',
name: 'annotate_map',
callId: 'call_a',
step: 'resolve',
label: 'Texas State Capitol',
},
]);
let view = voiceCardView(state);
assert.equal(view.visible, true);
@@ -31,9 +61,16 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'WORKING');
assert.equal(view.user.text, 'Annotate the Texas State Capitol');
assert.equal(view.assistant.text, 'Finding the Capitol, then outlining it.');
assert.deepEqual(view.steps.map(({ label, status, detail }) => ({ label, status, detail })), [
{ label: 'Mark Texas State Capitol', status: 'running', detail: 'Finding places: Texas State Capitol' },
]);
assert.deepEqual(
view.steps.map(({ label, status, detail }) => ({ label, status, detail })),
[
{
label: 'Mark Texas State Capitol',
status: 'running',
detail: 'Finding places: Texas State Capitol',
},
],
);
const done = [
{
type: 'action-result',
@@ -43,7 +80,12 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-1', outlinePending: true }],
display: { title: 'Marked 1 place', lines: ['Texas State Capitol'], notes: ['Tracing outlines'], sources: [{ label: 'OpenStreetMap' }] },
display: {
title: 'Marked 1 place',
lines: ['Texas State Capitol'],
notes: ['Tracing outlines'],
sources: [{ label: 'OpenStreetMap' }],
},
referents: [{ n: 1, id: 'anno-1', label: 'Texas State Capitol' }],
},
},
@@ -53,35 +95,80 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'DONE');
assert.equal(view.steps[0].detail, 'Outline traced');
assert.equal(view.result.title, 'Marked 1 place');
assert.deepEqual(view.result.referents, [{ n: 1, label: 'Texas State Capitol' }]);
assert.deepEqual(view.result.notes, ['Tracing outlines', 'Source: OpenStreetMap']);
assert.equal(view.announce, 'Marked 1 place', 'the live region carries the result title');
assert.deepEqual(view.result.referents, [
{ n: 1, label: 'Texas State Capitol' },
]);
assert.deepEqual(view.result.notes, [
'Tracing outlines',
'Source: OpenStreetMap',
]);
assert.equal(
view.announce,
'Marked 1 place',
'the live region carries the result title',
);
});
test('presentation: silent lookups, dismissal and session end', () => {
const lookup = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'get_current_view_state', arguments: {} },
{ type: 'action-result', name: 'get_current_view_state', result: { ok: true, schedule: 'silent', display: { title: 'View state' } } },
{
type: 'action-result',
name: 'get_current_view_state',
result: {
ok: true,
schedule: 'silent',
display: { title: 'View state' },
},
},
]);
assert.equal(voiceCardView(lookup).result.visible, false, 'a silent lookup does not take the result card');
assert.equal(
voiceCardView(lookup).result.visible,
false,
'a silent lookup does not take the result card',
);
const dismissed = reduceVoiceCard(lookup, { type: 'dismiss' });
assert.equal(voiceCardView(dismissed).visible, false);
const stillDismissed = reduceVoiceCard(dismissed, { type: 'transcript', role: 'assistant', text: 'Hi', final: true });
assert.equal(voiceCardView(stillDismissed).visible, false, 'dismissed until the next turn');
assert.equal(voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' })).visible, true);
assert.deepEqual(reduceVoiceCard(lookup, { type: 'state', state: 'idle' }), initialVoiceCardState());
assert.equal(reduceVoiceCard(lookup, { type: 'turn-metrics' }), lookup, 'unrelated events are no-ops');
const stillDismissed = reduceVoiceCard(dismissed, {
type: 'transcript',
role: 'assistant',
text: 'Hi',
final: true,
});
assert.equal(
voiceCardView(stillDismissed).visible,
false,
'dismissed until the next turn',
);
assert.equal(
voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' }))
.visible,
true,
);
assert.deepEqual(
reduceVoiceCard(lookup, { type: 'state', state: 'idle' }),
initialVoiceCardState(),
);
assert.equal(
reduceVoiceCard(lookup, { type: 'turn-metrics' }),
lookup,
'unrelated events are no-ops',
);
});
test('markup: the card nests inside the voice control with every element and a polite live region', () => {
const root = controlSource.indexOf("root.id = 'gev-voice-control'");
const card = controlSource.indexOf('id="gev-voice-card"');
assert.ok(root >= 0 && card > root, 'the card is part of the voice control markup');
assert.ok(
root >= 0 && card > root,
'the card is part of the voice control markup',
);
for (const id of [
'gev-voice-card-phase',
'gev-voice-card-dismiss',
'gev-voice-card-user-text',
'gev-voice-card-user-who',
'gev-voice-card-assistant-text',
'gev-voice-card-plan',
'gev-voice-card-result-title',
@@ -92,11 +179,22 @@ test('markup: the card nests inside the voice control with every element and a p
]) {
assert.match(controlSource, new RegExp(`id="${id}"`), `${id} is missing`);
}
assert.match(controlSource, /id="gev-voice-card-live"[^>]*aria-live="polite"/);
assert.match(
controlSource,
/id="gev-voice-card-live"[^>]*aria-live="polite"/,
);
assert.match(controlSource, /aria-label="\$\{t\('Dismiss voice card'\)\}"/);
assert.match(css, /#gev-voice-control \.gev-voice-card \{/);
assert.match(css, /body\.recording-mode #gev-voice-control/, 'recording mode hides the control, card included');
assert.match(css, /body\.ui-clean-view #command-dock/, 'Clean-UI hides the dock, card included');
assert.match(
css,
/body\.recording-mode #gev-voice-control/,
'recording mode hides the control, card included',
);
assert.match(
css,
/body\.ui-clean-view #command-dock/,
'Clean-UI hides the dock, card included',
);
});
function fakeRoot() {
@@ -111,7 +209,12 @@ function fakeRoot() {
ownerDocument: null,
addEventListener(type, handler, options) {
listeners.set(type, handler);
options?.signal?.addEventListener('abort', () => listeners.delete(type));
options?.signal?.addEventListener('abort', () =>
listeners.delete(type),
);
},
removeEventListener(type, handler) {
if (listeners.get(type) === handler) listeners.delete(type);
},
fire: (type) => listeners.get(type)?.({}),
listenerCount: () => listeners.size,
@@ -128,6 +231,7 @@ function fakeRoot() {
const nodes = new Map();
const doc = { createElement: (tag) => ({ ...make(tag), tag }) };
const root = {
...make('gev-voice-control'),
querySelector(selector) {
const id = selector.slice(1);
if (!nodes.has(id)) {
@@ -160,12 +264,26 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
});
assert.equal(nodes.get('gev-voice-card').hidden, true, 'starts hidden');
card.handle({ type: 'interruption' });
card.handle({ type: 'action-call', name: 'fly_to_location', arguments: { query: 'Tokyo' } });
card.handle({ type: 'action-result', name: 'fly_to_location', result: { ok: true, display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] } } });
card.handle({
type: 'action-call',
name: 'fly_to_location',
arguments: { query: 'Tokyo' },
});
card.handle({
type: 'action-result',
name: 'fly_to_location',
result: {
ok: true,
display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] },
},
});
assert.equal(nodes.get('gev-voice-card').hidden, false);
assert.equal(nodes.get('gev-voice-card-result-title').textContent, 'Tokyo');
assert.equal(nodes.get('gev-voice-card-plan').children.length, 1);
assert.equal(nodes.get('gev-voice-card-chips').children[0].textContent, 'stale');
assert.equal(
nodes.get('gev-voice-card-chips').children[0].textContent,
'stale',
);
assert.equal(timers.size, 1, 'one idle timer at a time');
[...timers.values()][0]();
assert.equal(nodes.get('gev-voice-card').hidden, true, 'hides when idle');
@@ -177,7 +295,11 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
assert.equal(timers.size, 0);
assert.equal(nodes.get('gev-voice-card-dismiss').listenerCount(), 0);
card.handle({ type: 'interruption' });
assert.equal(nodes.get('gev-voice-card').hidden, true, 'a destroyed card ignores late events');
assert.equal(
nodes.get('gev-voice-card').hidden,
true,
'a destroyed card ignores late events',
);
});
test('controls: a control without card markup is inert', () => {
@@ -189,17 +311,41 @@ test('controls: a control without card markup is inert', () => {
test('presentation: concurrent calls of one tool update their own steps', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'fly_to_location', callId: 'c1', arguments: { query: 'Tokyo' } },
{ type: 'action-call', name: 'fly_to_location', callId: 'c2', arguments: { query: 'Paris' } },
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c1',
arguments: { query: 'Tokyo' },
},
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c2',
arguments: { query: 'Paris' },
},
// Out of order: the first call finishes last.
{ type: 'action-result', name: 'fly_to_location', callId: 'c2', result: { ok: false, error: 'x' } },
{ type: 'progress', name: 'fly_to_location', callId: 'c1', step: 'search', label: 'Tokyo' },
{
type: 'action-result',
name: 'fly_to_location',
callId: 'c2',
result: { ok: false, error: 'x' },
},
{
type: 'progress',
name: 'fly_to_location',
callId: 'c1',
step: 'search',
label: 'Tokyo',
},
]);
const steps = voiceCardView(state).steps;
assert.deepEqual(steps.map(({ label, status }) => [label, status]), [
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
]);
assert.deepEqual(
steps.map(({ label, status }) => [label, status]),
[
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
],
);
assert.equal(steps[0].detail, 'Looking up place: Tokyo');
});
@@ -207,27 +353,299 @@ test('presentation: stale and unmatched events never reopen the card', () => {
const ended = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-9', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-9', outlinePending: true }],
},
},
{ type: 'state', state: 'idle' },
]);
for (const late of [
{ type: 'outline', id: 'anno-9', status: 'resolved' },
{ type: 'outline', status: 'resolved' },
{ type: 'progress', name: 'annotate_map', callId: 'c1', step: 'resolve' },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, display: { title: 'Marked 1 place' } } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: { ok: true, display: { title: 'Marked 1 place' } },
},
]) {
assert.equal(reduceVoiceCard(ended, late), ended, `${late.type} after stop is a no-op`);
assert.equal(
reduceVoiceCard(ended, late),
ended,
`${late.type} after stop is a no-op`,
);
}
const live = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-1', outlinePending: true }, { ok: true, id: 'anno-2', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [
{ ok: true, id: 'anno-1', outlinePending: true },
{ ok: true, id: 'anno-2', outlinePending: true },
],
},
},
{ type: 'dismiss' },
]);
assert.equal(reduceVoiceCard(live, { type: 'outline', id: 'anno-7', status: 'resolved' }), live, 'another mark is ignored');
const first = reduceVoiceCard(live, { type: 'outline', id: 'anno-1', status: 'resolved' });
assert.equal(voiceCardView(first).steps[0].detail, 'Tracing outline', 'one of two outlines still pending');
assert.equal(voiceCardView(first).visible, false, 'a dismissed card stays hidden');
const both = reduceVoiceCard(first, { type: 'outline', id: 'anno-2', status: 'failed' });
assert.equal(
reduceVoiceCard(live, {
type: 'outline',
id: 'anno-7',
status: 'resolved',
}),
live,
'another mark is ignored',
);
const first = reduceVoiceCard(live, {
type: 'outline',
id: 'anno-1',
status: 'resolved',
});
assert.equal(
voiceCardView(first).steps[0].detail,
'Tracing outline',
'one of two outlines still pending',
);
assert.equal(
voiceCardView(first).visible,
false,
'a dismissed card stays hidden',
);
const both = reduceVoiceCard(first, {
type: 'outline',
id: 'anno-2',
status: 'failed',
});
assert.equal(voiceCardView(both).steps[0].detail, 'Outline unavailable');
});
test('presentation controls preserve real reducer content and the existing 14-second idle deadline', () => {
const { root, nodes } = fakeRoot();
const timers = new Map();
let next = 0,
renders = 0;
const card = new VoiceCardControls({
root,
setTimer(fn, ms) {
const id = ++next;
timers.set(id, { fn, ms });
return id;
},
clearTimer(id) {
timers.delete(id);
},
onRender() {
renders++;
},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A current answer',
final: true,
responseId: 'r1',
});
const content = card.state;
const timer = [...timers.keys()][0];
assert.equal(timers.get(timer).ms, 14000);
card.setPresented(false);
assert.equal(card.snapshot().visible, false);
assert.equal(nodes.get('gev-voice-card-live').textContent, '');
card.setPresented(true);
assert.equal(card.snapshot().visible, true);
assert.equal(card.snapshot().assistant.text, 'A current answer');
assert.equal(
card.snapshot().announce,
'',
'returning to a tab does not repeat an already presented announcement',
);
assert.equal(card.state, content);
assert.equal(
[...timers.keys()][0],
timer,
'presentation switches do not reset idle',
);
card.setEnabled(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A later answer',
final: true,
responseId: 'r2',
});
assert.equal(card.snapshot().visible, false);
card.setEnabled(true);
assert.equal(card.snapshot().assistant.text, 'A later answer');
assert.equal(card.snapshot().announce, 'A later answer');
assert.ok(renders > 4);
const idle = [...timers.values()][0];
idle.fn();
assert.equal(
card.snapshot().visible,
false,
'original idle still ends content presentation',
);
card.destroy();
const lastRenders = renders;
card.setPresented(false);
card.setEnabled(false);
assert.equal(renders, lastRenders);
});
test('a new hidden reply with identical words is still a fresh announcement', () => {
const { root } = fakeRoot();
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r1',
});
card.setPresented(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r2',
});
card.setPresented(true);
assert.equal(card.snapshot().announce, 'Same words');
card.destroy();
});
test('additive presentation gates retain the current localized reducer labels', () => {
const previous = getLocale();
try {
setLocale('es');
const { root } = fakeRoot();
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'action-call',
name: 'annotate_map',
callId: 'a',
arguments: {},
});
assert.equal(card.snapshot().phase, 'EN CURSO');
const steps = card.state.steps;
const result = card.state.result;
const who = root.querySelector('#gev-voice-card-user-who');
card.handle({ type: 'transcript', role: 'user', text: 'Same words', attribution: 'unverified' });
assert.equal(card.state.userAttribution, 'unverified');
assert.equal(card.snapshot().user.label, 'Habla reciente');
assert.equal(who.textContent, 'Habla reciente');
card.handle({ type: 'transcript', role: 'user', text: 'Same words' });
assert.equal(card.state.userAttribution, null);
assert.equal(card.snapshot().user.label, 'TÚ');
assert.equal(who.textContent, 'TÚ');
assert.equal(card.state.steps, steps);
assert.equal(card.state.result, result);
card.setPresented(false);
card.setPresented(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.setEnabled(false);
card.setEnabled(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.destroy();
} finally {
setLocale(previous);
}
});
test('manual panel and preference compose with the actual card without changing voice content or enabling Preview', async () => {
const { createManualVoicePanel } = await import('./panel/manualPanel.js');
const { createVoicePreferences, bindPreferenceControls } =
await import('./panel/preferences.js');
const { root } = fakeRoot();
let panel;
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
onRender: () => panel?.sync(),
});
const ui = {
root,
card: root.querySelector('#gev-voice-card'),
voicePanel: root.querySelector('#panel'),
previewPanel: root.querySelector('#preview-panel'),
voiceSettingsPanel: root.querySelector('#settings-panel'),
previewEmpty: root.querySelector('#empty'),
voicePreviewInput: root.querySelector('#preview-input'),
voicePreviewNote: root.querySelector('#preview-note'),
};
const preference = createVoicePreferences({ storage: null });
const fields = bindPreferenceControls({
ui,
preferences: preference,
copy: {
previewNote: 'Mostrar respuestas.',
timeoutNote: '',
notSaved: 'No guardado.',
},
onChange: (state) => {
card.setEnabled(state.previewEnabled);
panel?.sync();
},
});
card.setEnabled(preference.snapshot().previewEnabled);
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preference.snapshot().previewEnabled,
copy: { previewEmpty: 'Sin respuesta.', previewDisabled: 'Desactivada.' },
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Actual current answer',
final: true,
});
const content = card.state;
assert.equal(
card.snapshot().visible,
false,
'manual panel stays closed despite content',
);
panel.select('preview');
assert.equal(card.snapshot().assistant.text, 'Actual current answer');
assert.equal(card.snapshot().visible, true);
preference.setPreviewEnabled(false);
assert.equal(card.snapshot().visible, false);
assert.equal(ui.previewEmpty.textContent, 'Desactivada.');
panel.select('settings');
panel.select('preview');
assert.equal(preference.snapshot().previewEnabled, false);
assert.equal(card.snapshot().visible, false);
assert.equal(card.state, content);
fields.dispose();
panel.destroy();
card.destroy();
});
src/voice/voiceCardPresentation.js
+12 -1
View File
@@ -44,6 +44,7 @@ export function initialVoiceCardState() {
visible: false,
dismissed: false,
user: null,
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -69,6 +70,7 @@ function newTurn(state) {
visible: true,
dismissed: false,
user: '',
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -143,7 +145,14 @@ export function reduceVoiceCard(state, event) {
case 'transcript': {
const text = event.text || '';
if (event.role === 'user') {
return text ? reveal({ ...state, user: caption(text) }) : state;
return text
? reveal({
...state,
user: caption(text),
userAttribution:
event.attribution === 'unverified' ? 'unverified' : null,
})
: state;
}
if (event.role !== 'assistant' || !text) return state;
const sameResponse =
@@ -287,6 +296,8 @@ export function voiceCardView(state) {
busy: running,
phase: running ? t('WORKING') : state.steps.length ? t('DONE') : '',
user: {
label:
state.userAttribution === 'unverified' ? t('Recent speech') : t('YOU'),
visible: state.user !== null,
text: state.user || '…',
},