diff --git a/.env.example b/.env.example index 070a05d1b..102b447b7 100644 --- a/.env.example +++ b/.env.example @@ -83,6 +83,12 @@ OPENAI_REALTIME_TRANSCRIBE_MODEL=gpt-4o-mini-transcribe # keep them while debugging; delete the log when you are done. GEV_VOICE_LOG_CONTENT= OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano +# Optional: Gemini Live voice, selected in the GEV MIC provider chooser. +# Server-only key. Do not prefix with VITE_. Provider Settings can save it. +GEMINI_API_KEY= +GEMINI_LIVE_MODEL=gemini-3.8-live +# Token starts per IP per minute; 0 disables the throttle, not provider quotas. +# GEV_RATELIMIT_GEMINI_PER_MIN=30 # Per-IP rate limit for the OpenAI cost endpoints # (/api/realtime/token + /api/openai/hud-summary), in requests per minute. # DEFAULTS TO 30, the value the Pinokio build also ships. That clears ordinary diff --git a/README.md b/README.md index 55cf47189..3b005c8dc 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,10 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain β€” an ## πŸŽ™οΈ Talk to It +The standalone app offers **OpenAI** and **Gemini** in the GEV MIC provider chooser. OpenAI is the initial default; your choice is saved in this browser. To use Gemini, add `GEMINI_API_KEY` in Provider Settings or your local `.env` file, then select Gemini. `GEMINI_LIVE_MODEL` defaults to `gemini-3.8-live`. The permanent key stays on the local server, which issues a constrained, single-use session credential. + +Changing provider ends the current conversation. If voice is on, the new provider starts with the same open-mic or push-to-talk mode; a held Space key is not carried over. Gemini uses the same map actions and captions, but has no OpenAI cost meter or MINI tier. When Gemini reports that its session has expired, start voice again for a new conversation. The provider chooser does not change the OpenAI-backed AI HUD summary. + > Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. The OAuth option is experimental. OpenAI documents ChatGPT plan usage for the [Responses API](https://developers.openai.com/siwc/token-sharing-open-source/models-and-inference); a supported third-party Realtime contract for reusing Codex tokens has not been confirmed. A successful token mint does not establish billing or support terms. diff --git a/changelog.d/1200-cctv-live-icon-color.md b/changelog.d/1200-cctv-live-icon-color.md new file mode 100644 index 000000000..30d67e1d4 --- /dev/null +++ b/changelog.d/1200-cctv-live-icon-color.md @@ -0,0 +1,5 @@ +- Color CCTV cameras with a video feed violet. Cameras with an HLS, MP4 or + WebM feed were the same cyan as stills, so a DelDOT or Statens vegvesen + video camera could only be found by opening it. Stills stay cyan and the + active camera stays amber, and focus de-emphasis fades the violet the same + way (Danielslay86, #1200). diff --git a/changelog.d/1201-cctv-stills-aspect.md b/changelog.d/1201-cctv-stills-aspect.md new file mode 100644 index 000000000..fdb560064 --- /dev/null +++ b/changelog.d/1201-cctv-stills-aspect.md @@ -0,0 +1,4 @@ +- Draw CCTV stills at their source aspect on the monitor plane instead of + stretching them to the 16:9 canvas. Most DOT cameras publish 4:3 frames, so + every still was widened; they are now centered and pillarboxed on the dark + the placeholder already ends on (Danielslay86, #1201). diff --git a/changelog.d/1220-openai-typed-send.md b/changelog.d/1220-openai-typed-send.md new file mode 100644 index 000000000..cd1b5b470 --- /dev/null +++ b/changelog.d/1220-openai-typed-send.md @@ -0,0 +1,3 @@ +- Avoid starting or queuing a reply when the local channel rejects a typed OpenAI + command, while preserving an earlier accepted deferred input + (Manjunath, #1220). diff --git a/changelog.d/1234-cctv-catalog-background-refresh.md b/changelog.d/1234-cctv-catalog-background-refresh.md new file mode 100644 index 000000000..2d3706b14 --- /dev/null +++ b/changelog.d/1234-cctv-catalog-background-refresh.md @@ -0,0 +1,5 @@ +- Serve the camera catalog while it refreshes. An expired catalog made every + camera request wait for the slowest pack to reload; it is now answered at + once while one background refresh runs, so only the first load waits. Estonia's + slow Tarktee feeds get 30 s instead of 15 s, so the pack no longer drops out + on about a third of loads (quanticsoul4772, #1234). diff --git a/changelog.d/816-radio-owner-restoration.md b/changelog.d/816-radio-owner-restoration.md new file mode 100644 index 000000000..df38617ee --- /dev/null +++ b/changelog.d/816-radio-owner-restoration.md @@ -0,0 +1,7 @@ +- Close the compact Radio disclosure when an adaptive tactical right-rail + restore directly expands Context, before Space Missions can be covered. + A repeated Context-open request also clears stale disclosure state. + Real PanelChrome constructor wiring and the rail reflow path are covered + by focused regressions. Credits: kvnloo (#816), raysung-2026's original + #804 report, asturksever's review, and quanticsoul4772's independent + Windows diagnosis. diff --git a/changelog.d/872-gemini-live-voice.md b/changelog.d/872-gemini-live-voice.md new file mode 100644 index 000000000..519e4d37b --- /dev/null +++ b/changelog.d/872-gemini-live-voice.md @@ -0,0 +1,4 @@ +- Add optional Gemini Live voice to the standalone app, with a provider chooser, + server-held API key, microphone streaming, shared map actions and captions. + Switching providers ends the old conversation and preserves the input mode + (Manjunath, #872). diff --git a/changelog.d/880-world-packet-boundary.md b/changelog.d/880-world-packet-boundary.md new file mode 100644 index 000000000..7ee574e35 --- /dev/null +++ b/changelog.d/880-world-packet-boundary.md @@ -0,0 +1,6 @@ +- Extract a reusable model-free World Packet from the existing voice + `get_current_view_state` owner without restoring disabled layers to the + model context. Keep the enabled-only payload, disabled-layer count, source + provenance, action result and view-state contract; read the layer roster once + per packet; voice retains the speech/card envelope at its own boundary. + Credits: original GEV voice/layer contributors and #880 implementation. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 5ee781db4..75f492df7 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -496,6 +496,11 @@ to layer toggles, fly_to_location, frame_overhead, select_nearest_aircraft, get_entity_context, annotate_map and analyst_query. Analyst speech and the card share one deterministic count/scope headline; lower bounds use "At least" and partial answers name the unanswered layers. `get_current_view_state` lists enabled layers only. +The model-free `gods-eye-view/agent/world-packet` export exposes +`getWorldPacket({ viewer, styleManager, dataManager, sceneDirector })` to +non-voice consumers. It reads the same enabled-only state and feed +provenance before voice adds speech/card fields, with no new network requests +or mutations. The voice card nested in the mic control (`voiceCard.js`) shows captions, plan steps and the result. `resultDisplay.js` is the one adapter from a result to the card: builder results show as built, and analyst results show a title @@ -613,7 +618,9 @@ Reference feed construction is exported through `sources/reference`; the cable s Source factories have dedicated `layers//source` exports. ALPR and earthquake record normalization and CCTV source policy no longer pull rendering into source consumers. Catalog construction and voice feed reads use their focused owners; compatibility entries remain available. Settings filesystem hardening lives under `server/standalone/`. Import-direction checks complement export ownership checks; source behavior, settings policy and rendering are unchanged. -Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. Alternate adapters can use the same controls and action runner. +Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. + +The standalone app offers OpenAI and Gemini Live through an explicit provider chooser. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 30 tools using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields. @@ -1203,6 +1210,10 @@ Automatic collapse remains presentation only, prioritizes the latest explicit panel, and honors keyboard focus on the right. Mobile layout still releases desktop allocation styles. Stable Display allocation avoids unnecessary style writes. This extraction does not change panel positions or layout defaults. +Context expansion through the panel owner or adaptive right-rail restoration +closes an open compact Radio disclosure before Context/Space Missions becomes +visible. The mini remains available while Context is collapsed. This fixes the +Radio-overlap path reported in #804 without changing z-index or layout policy. The right rail skips panels hidden by other CSS (computed `display: none`), but keeps the collapsed panels its own exclusive presentation hides, so automatic collapse can still be undone. @@ -4408,7 +4419,7 @@ silently demoting every later lookup for the session. - **FIRMS**: no ground clamping (zero 3D-tiles height sampling), ≀18 screen-decluttered ambient labels, click-to-inspect detail card, 2.5k/3k sprite budgets viewport-clipped by FRP. - **CCTV v2 foundation:** a pitched frustum wireframe (4 corner rays + far-cap rectangle) with a monitor plane at the frustum's - far cap, retargeting the existing video/canvas texture pipeline. Manual calibration only β€” + far cap, retargeting the existing video/canvas texture pipeline. Stills are drawn at their source aspect, centered on the placeholder dark (`#01070c`), instead of being stretched to the 16:9 canvas. Manual calibration only β€” auto-calibration and the drape mesh pipeline are deleted. A one-shot activation obstruction probe (`pickFromRay` on camera activation, clamping the plane short of the first hit) remains; ground placement is superseded by the shared-floor v3 behavior below. Calibration persists to @@ -4426,7 +4437,7 @@ silently demoting every later lookup for the session. ground clamp now lifts the cap _center_ only so the wireframe stays a true pyramid welded to the plane (was a flattened fan / the ~47.5 m divergence β€” RESOLVED), re-selecting the active camera is a no-op (killed a click-flash), and texture swaps gate on canvas content (killed a - periodic white flash). Coverage is now **metro-wide: 250 cameras** (`CCTV_AUSTIN_MAX_SOURCES` + periodic white flash). Camera icons are cyan for stills, violet for cameras with a video feed, and amber for the active camera. Coverage is now **metro-wide: 250 cameras** (`CCTV_AUSTIN_MAX_SOURCES` default 36 β†’ 250, hard bound 300), filtered to `camera_status === TURNED_ON` (~815 live of 1,003 rows). City packs (2026-07-04): Caltrans (districts 4/7/11/3 β€” SF, LA, San Diego, Sacramento; cap 300) and TfL London JamCams (cap 250) join Austin (cap 250) as keyless default @@ -4439,7 +4450,13 @@ silently demoting every later lookup for the session. GATHERING stations carry 2,256 in-collection presets, prioritized to 300 against seven anchors on the main road spine. `CCTV_MAX_SOURCES` is a 4,000 catalog-wide ceiling shared round-robin across packs, so a lower global cap thins every region instead of starving the last pack. - ~1,100 cameras total, all RAW PRIOR poses, stills-first. + ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once + it expires, `/api/cctv` keeps answering from it while one background refresh reloads every + pack, so only the very first load waits for the slowest pack (a live first load took about + 22 s, and is bounded by Tarktee's 30 s). A background refresh that fails or comes back + empty keeps the old catalog and is retried after another 15 minutes, not on the next + request. Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s + (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. - **CCTV v3 UX β€” viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a tri-state cycle `OFF β†’ ON β†’ VIEWSHED`; viewshed mode renders each visible camera's frustum @@ -4756,7 +4773,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Every selectable layer writes that one slot** (2026-08-21). The tracking layers β€” flights, military, satellites β€” publish selection on their own awareness lane (`gev:awareness-subject-selected`, consumed by the readout card and the Contacts panel) and for the life of the voice tools never wrote the shared slot, so `scope:'selected'` silently answered `in_view` with a contact plainly selected on screen. They now call `selectTrackedSubjectContext` / `refreshTrackedSubjectContext` / `clearTrackedSubjectContext` (`src/data/contextStore.js`) on select / poll / deselect. **Invariants: (a)** that write path must NOT dispatch `gev:entity-selected` β€” tracking layers already own an event lane and a second one makes two surfaces fight over one subject; **(b)** exactly one record per tracking layer, because a frozen snapshot of a moved contact must never reach the visible-entity scan; **(c)** precedence is recency, not layer β€” one slot, last selection wins, so clicking an overlay entity supersedes a tracked plane as the voice subject while the plane stays tracked. Satellites refresh on the 1 s propagation beat, not per frame. - **Context-mode vocabulary is symmetric** (2026-08-21). `set_context_mode` accepts `contacts`; the internal id is `flights`. Every model-readable field (`mode`, `entering`, `priorMode`, nested `context`/`contextRollback`, and the transition diagnostic text) is reported in the accepted vocabulary, with the internal id preserved as `Internal`; an absent secondary mode stays `null` rather than claiming to be `off`. Mapping lives in `src/contextModePolicy.js` so UI text and voice payloads cannot drift. Reporting the internal id made the model read `mode:'flights'` as "Contacts is off" and refuse to answer from the `contactsWindow` counts in the same payload. - **Analyst β†’ track handoff carries a key, not just a label** (2026-08-21). `analyst_query` items include `icao24`/`mmsi` alongside the display `id`, and contact lookup uses the shared tiered ranking in `src/data/contactMatch.js`: hex exact β†’ callsign exact β†’ registration exact β†’ callsign prefix β†’ registration prefix β†’ callsign substring β†’ registration substring. The tiers keep an exact callsign ahead of a colliding registration regardless of feed order, and registrations compare separator-insensitively (`G-ABCD`/`GABCD`, `05-8152`/`058152`, `N123AB`/`N-123AB`). - - **A typed command supersedes the turn it interrupts** (2026-08-21). `sendTextCommand` defers its `response.create` behind an active response instead of colliding with it, marks that response superseded so a late function call from it is refused rather than dispatched, and drops the old turn's queued follow-up. A refused call is still ANSWERED β€” a terminal `{ok:false, superseded:true}` `function_call_output` β€” because an unanswered `function_call` strands a pending call and deadlocks the model; the refusal creates no response of its own. A burst of typed commands coalesces into one response while keeping both conversation items. + - **A typed command supersedes the turn it interrupts** (2026-08-21; hardened 2026-10-17). `sendTextCommand` defers its `response.create` behind an active response instead of colliding with it, marks that response superseded so a late function call from it is refused rather than dispatched, and drops the old turn's queued follow-up. A refused call is still ANSWERED β€” a terminal `{ok:false, superseded:true}` `function_call_output` β€” because an unanswered `function_call` strands a pending call and deadlocks the model; the refusal creates no response of its own. A burst of typed commands coalesces into one response while keeping both conversation items. If the local channel rejects the user item, that command requests no response and adds no deferred answer; earlier accepted deferred input remains queued. Interruption and pointer preparation still happen before the send attempt. - **Subject reconciliation across satellite catalog rebuilds** (2026-08-21). A dense↔core toggle or TLE refresh clears and repopulates the catalog, so the published subject is re-resolved against the new satrec; a subject that did NOT survive releases the slot. The per-frame refresh cannot do this itself β€” `_getTrackedFramePosition` returns early once the satellite has no catalog entry β€” so the reconcile runs at rebuild completion. An empty catalog is a rebuild in flight, not a disappearance. - **One aircraft-proximity snapshot** (2026-08-22; hardened 2026-09-30). `collectAircraftProximityWindow` (`src/data/militaryAwareness.js`) computes the Contacts panel cohort once. The voice analyst reads the exact retained snapshot and evaluation timestamp instead of re-scanning live layers, so the panel readout and spoken count for one centre cannot cross snapshots. **Invariant: do not re-derive a proximity count anywhere else.** Each layer retains 20,000 rows and samples one row beyond the cap; a saturated cohort reports `complete:false`, `truncated:true`, and is spoken as a lower bound. They diverged before because the panel read live billboard positions while the analyst used last-fix coordinates over a 2,000-record slice β€” 111 on screen, 15 spoken. Explicit regions, arbitrary points, and a radius different from the panel radius deliberately keep the general record engine. Entity-centred results carry `window: {engine:'contacts-window', centeredOn, radiusKm, flights, military, aircraft, complete, evaluatedAt}`. - **Centre precedence for nearby asks**: explicit place in the question > Contacts subject (a selected non-contact entity never silently becomes the centre) > an entity the user names > the current view, said aloud. Contacts active with no subject uses the view rather than reading an empty panel. diff --git a/package.json b/package.json index 99e35c0c5..a6396db73 100644 --- a/package.json +++ b/package.json @@ -277,6 +277,7 @@ "./tools/mcp": "./src/tools/mcp/index.js", "./tools/panel": "./src/tools/panel.js", "./tools/services": "./src/tools/services.js", - "./view": "./src/view/index.js" + "./view": "./src/view/index.js", + "./agent/world-packet": "./src/agent/worldPacket.js" } } diff --git a/scripts/format-scope.json b/scripts/format-scope.json index b353f7067..543b0f9a4 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -557,6 +557,8 @@ "src/data/cctvNswSource.test.mjs", "src/data/cctvTxdotSource.test.mjs", "src/layers/cctv/source.test.mjs", + "src/layers/cctv/frames.test.mjs", + "src/layers/cctv/rendering.test.mjs", "src/data/alprCameras.js", "src/data/alprCameras.test.mjs", "src/data/dataCredits.test.mjs", @@ -637,6 +639,7 @@ "src/voice/control.js", "src/voice/realtimeBackend.js", "src/voice/realtimeBackend.test.mjs", + "src/agent/worldPacket.test.mjs", "src/voice/layerSummary.js", "build/application-html.js", "src/app/sourceSlot.js", @@ -692,6 +695,7 @@ "src/ui/shareRestoration.js", "src/ui/visualSettings.js", "src/ui/panelChrome.js", + "src/ui/panelChrome.radioOwnership.test.mjs", "src/testSupport/readShellSource.mjs", "src/ui/stateOwners.test.mjs", "src/layers/flights/records.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 627c9307f..ca5dc0c7f 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1633,7 +1633,15 @@ "src/i18n/locale.js", "src/i18n/locales/es.json", "src/i18n/locales/ru.json", - "src/ui/statusText.js" + "src/ui/statusText.js", + "src/voice/providerCommands.js", + "src/voice/providerCapabilities.js", + "src/voice/geminiSession.js", + "src/voice/geminiAudio.js", + "src/voice/geminiCaptureWorklet.js", + "src/sources/httpBody.js", + "src/sources/requestDeadline.js", + "src/i18n/marker.js" ], "external": [] }, @@ -2221,6 +2229,7 @@ "src/voice/commands.js", "src/voice/control.js", "src/voice/gevActions.js", + "src/agent/worldPacket.js", "src/voice/speech.js", "src/ui/layerNames.js", "src/voice/earcon.js", @@ -2496,7 +2505,12 @@ "src/layers/airQuality/index.js", "src/layers/airQuality/model.js", "src/layers/airQuality/source.js", - "src/ui/statusText.js" + "src/ui/statusText.js", + "src/voice/providerCommands.js", + "src/voice/providerCapabilities.js", + "src/voice/geminiSession.js", + "src/voice/geminiAudio.js", + "src/voice/geminiCaptureWorklet.js" ], "external": [ "hls.js", @@ -3384,7 +3398,8 @@ "src/i18n/locale.js", "src/i18n/locales/es.json", "src/i18n/locales/ru.json", - "src/ui/statusText.js" + "src/ui/statusText.js", + "src/voice/providerCapabilities.js" ], "external": [] }, @@ -3951,5 +3966,18 @@ "src/i18n/marker.js" ], "external": [] + }, + "agent-world-packet": { + "exports": ["./agent/world-packet"], + "modules": [ + "src/agent/worldPacket.js", + "src/data/layerSnapshot.js", + "src/data/feedState.js", + "src/contextModePolicy.js", + "src/loadingFeedback.js", + "src/data/installationFeedback.js", + "src/i18n/marker.js" + ], + "external": ["cesium"] } } diff --git a/scripts/precompute-cctv-heights.mjs b/scripts/precompute-cctv-heights.mjs index 648768341..2d55f4168 100644 --- a/scripts/precompute-cctv-heights.mjs +++ b/scripts/precompute-cctv-heights.mjs @@ -326,8 +326,10 @@ async function main() { 'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json', ); let sidecar = await loadSidecar(output); + // A cold catalog waits for its slowest pack (Tarktee gets 30 s), so allow + // more than that. const response = await fetch(`${base}/api/cctv/sources`, { - signal: AbortSignal.timeout(30_000), + signal: AbortSignal.timeout(60_000), }); if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`); const catalog = await response.json(); diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 08995603f..2c121e0d6 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -139,6 +139,56 @@ function record(name, ok, detail) { const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// Each page.evaluate is bounded by puppeteer's `protocolTimeout` (default +// 180s). Waits never come close: they poll from Node (below). The headroom is +// for the few long evaluates, such as enabling the full ~3,700-camera catalog +// on a real GPU, and matches the other qa-*.mjs harnesses. +const PROTOCOL_TIMEOUT_MS = 300_000; + +/** + * Polls an in-page predicate from Node, one short page.evaluate per tick, + * until it holds or the budget runs out. Used instead of page.waitForFunction + * for two reasons: its default rAF polling stops in a minimised or hidden + * headful window (timers keep running, rAF does not), and its result is one + * CDP call that protocolTimeout cuts off, while the headful drain budget + * reaches ~50 minutes. A predicate that throws fails the run instead of + * silently spending the budget. + * @param {import('puppeteer').Page} page + * @param {Function} predicate - In-page predicate. + * @param {object} opts + * @param {number} opts.timeoutMs - Overall budget. + * @param {number} [opts.intervalMs=250] - Delay between checks. + * @param {unknown[]} [opts.args=[]] - Arguments passed to the predicate. + * The predicate must return a boolean: its result crosses CDP by value. + * @returns {Promise} True if the predicate held within the budget. + */ +async function waitForPredicate( + page, + predicate, + { timeoutMs, intervalMs = 250, args = [] }, +) { + // performance.now() is monotonic and doesn't advance while the machine + // sleeps, so neither a clock change nor a laptop sleep eats the budget. + const deadline = performance.now() + timeoutMs; + for (;;) { + try { + if (await page.evaluate(predicate, ...args)) return true; + } catch (err) { + // A navigation or Vite reload replaces the execution context: poll again. + if ( + !/Execution context was destroyed|Cannot find context/.test( + err?.message ?? '', + ) + ) { + throw err; + } + } + const remaining = deadline - performance.now(); + if (remaining <= 0) return false; + await sleep(Math.min(intervalMs, remaining)); + } +} + // SwiftShader can spend tens of seconds servicing the hover drillPick emitted // by page.mouse.move(), before the LEFT_DOWN under test even runs. Position the // pointer while ADJUST is disabled, then re-enable it and explicitly render a @@ -155,29 +205,58 @@ async function positionPointerForGizmoDrag(page, point, frameCount = 3) { gev.dataManager.layers .get('cctv') .module.setParams({ calibrationMode: true }); - const scene = gev.viewer.scene; - let rendered = 0; - return new Promise((resolve) => { - let settled = false; - const finish = (complete) => { - if (settled) return; - settled = true; - clearTimeout(timer); - remove(); - resolve(complete); - }; - const remove = scene.postRender.addEventListener(() => { - rendered += 1; - if (rendered >= count) { - finish(true); - return; - } + return (await window.__qaRenderFrames(count, 5000)) >= count; + }, frameCount); +} + +/** + * Saves a screenshot for human review. These are never a gate, so a capture + * that fails (for example when the renderer stalls in a hidden headful window) + * is logged instead of ending the run. + * @param {import('puppeteer').Page} page + * @param {string} file + * @param {string} label + */ +async function saveReviewShot(page, file, label) { + try { + await page.screenshot({ path: file }); + console.log(` screenshot (${label}) β†’ ${path.relative(REPO_ROOT, file)}`); + } catch (err) { + console.warn( + ` screenshot skipped (${label}): ${path.relative(REPO_ROOT, file)}: ${err?.message ?? err}`, + ); + } +} + +/** + * Installs window.__qaRenderFrames(count, capMs) in the page: requests renders + * until `count` frames have passed scene.postRender or capMs runs out, and + * resolves with the number of frames rendered. The cap keeps a stalled + * renderer (headful window asleep or hidden) from hanging the CDP call. + * @param {import('puppeteer').Page} page + */ +function installRenderFramesHelper(page) { + return page.evaluate(() => { + window.__qaRenderFrames = (count, capMs) => + new Promise((resolve) => { + const scene = window.__godsEyeView.viewer.scene; + let frames = 0; + const finish = () => { + clearTimeout(timer); + remove(); + resolve(frames); + }; + const remove = scene.postRender.addEventListener(() => { + if (++frames >= count) { + finish(); + return; + } + scene.requestRender(); + }); + const timer = setTimeout(finish, capMs); scene.requestRender(); }); - const timer = setTimeout(() => finish(false), 5000); - scene.requestRender(); - }); - }, frameCount); + }); } // In-page helper (stringified and re-defined inside each evaluate that needs @@ -280,23 +359,21 @@ const BEARING_SHIFT_SRC = ` * @returns {Promise} True if tilesLoaded was observed within the timeout. */ function waitForTilesLoaded(page, timeoutMs = 15000) { - return page - .waitForFunction( - () => { - const scene = window.__godsEyeView.viewer.scene; - const prims = scene.primitives; - for (let i = 0; i < prims.length; i++) { - const p = prims.get(i); - if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { - return p.tilesLoaded === true; - } + return waitForPredicate( + page, + () => { + const scene = window.__godsEyeView.viewer.scene; + const prims = scene.primitives; + for (let i = 0; i < prims.length; i++) { + const p = prims.get(i); + if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { + return p.tilesLoaded === true; } - return true; // no tileset in the scene β€” nothing to wait for - }, - { timeout: timeoutMs }, - ) - .then(() => true) - .catch(() => false); + } + return true; // no tileset in the scene β€” nothing to wait for + }, + { timeoutMs }, + ); } /** @@ -364,6 +441,10 @@ async function main() { const chromeExecutable = findChromeExecutable(); const browser = await puppeteer.launch({ headless: HEADFUL ? false : 'new', + protocolTimeout: PROTOCOL_TIMEOUT_MS, + // Talk CDP over a pipe, so no debugging port stays open on localhost for + // the length of a headful run. + pipe: true, ...(chromeExecutable ? { executablePath: chromeExecutable } : {}), args: [ '--no-sandbox', @@ -397,13 +478,18 @@ async function main() { }); console.log('Loading app...'); await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); - await page.waitForFunction( + const appReady = await waitForPredicate( + page, () => - window.__godsEyeView && - window.__godsEyeView.viewer && - window.__godsEyeView.dataManager, - { timeout: 60000 }, + Boolean( + window.__godsEyeView && + window.__godsEyeView.viewer && + window.__godsEyeView.dataManager, + ), + { timeoutMs: 60000 }, ); + if (!appReady) throw new Error('app did not initialise within 60s'); + await installRenderFramesHelper(page); // Let the initial fly-to Austin and first tiles settle. await sleep(4000); @@ -475,18 +561,15 @@ async function main() { console.log( `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`, ); - const drained = await page - .waitForFunction( - () => { - const mod = - window.__godsEyeView.dataManager.layers.get('cctv').module; - const ui = mod.getUIState(); - return ui.loading && ui.loading.active === false; - }, - { timeout: drainBudgetMs }, - ) - .then(() => true) - .catch(() => false); + const drained = await waitForPredicate( + page, + () => { + const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; + const ui = mod.getUIState(); + return Boolean(ui.loading) && ui.loading.active === false; + }, + { timeoutMs: drainBudgetMs, intervalMs: 1000 }, + ); record( `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`, drained ? true : HEADFUL ? false : null, @@ -579,14 +662,11 @@ async function main() { const tilesSeenLoaded = await waitForTilesLoaded(page, 45000); let sampleFloorOk = false; if (tilesSeenLoaded) { - sampleFloorOk = await page - .waitForFunction( - (base) => window.__qaCounters.sampleHeight - base >= 1, - { timeout: 30000 }, - c0.sampleHeight, - ) - .then(() => true) - .catch(() => false); + sampleFloorOk = await waitForPredicate( + page, + (base) => window.__qaCounters.sampleHeight - base >= 1, + { timeoutMs: 30000, args: [c0.sampleHeight] }, + ); } const cFloor = await readCounters(); record( @@ -1358,18 +1438,7 @@ async function main() { }, }); window.__qaCctvSiblingOwner = owner; - await new Promise((resolve) => { - let frames = 0; - const remove = scene.postRender.addEventListener(() => { - if (++frames < 3) { - scene.requestRender(); - return; - } - remove(); - resolve(); - }); - scene.requestRender(); - }); + const framesRendered = await window.__qaRenderFrames(3, 10000); const rect = scene.canvas.getBoundingClientRect(); const picked = scene.pick(canvasPoint); const pickedId = picked?.id?.id ?? picked?.id ?? null; @@ -1382,26 +1451,33 @@ async function main() { typeof pickedId === 'string' ? pickedId : (pickedId?.id ?? null), ownerId: owner.id, ownsExactObject: picked?.id === owner, + framesRendered, projectionDelta: Math.hypot( projected.x - canvasPoint.x, projected.y - canvasPoint.y, ), }; }, emptyClickPoint); + const siblingOwnsPick = + siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && + siblingTarget.ownsExactObject; + // No frame at all within the 10s cap points at a stalled renderer rather + // than at picking, though the harness can't rule out both. Fail it on the + // real-GPU sign-off path; leave it inconclusive under headless GL. A slow + // SwiftShader frame (1-2 of 3) still counts as a real miss. + const siblingRendererStalled = + siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered === 0; record( 'sibling test object owns its canvas pick', + !siblingTarget + ? null + : siblingOwnsPick || + (siblingRendererStalled && !HEADFUL ? null : false), siblingTarget - ? siblingTarget.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - : null, - siblingTarget - ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΞ”=${siblingTarget.projectionDelta}` + ? `${siblingRendererStalled ? 'renderer stalled: ' : ''}frames=${siblingTarget.framesRendered}/3 picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΞ”=${siblingTarget.projectionDelta}` : 'no previously verified empty canvas ray was available', ); - if ( - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - ) { + if (siblingOwnsPick) { await page.mouse.click(siblingTarget.x, siblingTarget.y); await sleep(250); } @@ -1426,8 +1502,7 @@ async function main() { }); record( 'sibling canvas click passes through without CCTV selection or deselection', - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject + siblingOwnsPick ? siblingClick.activeId === clickEvidenceSetup.activeId && siblingClick.transitions === siblingClick.baselineTransitions && siblingClick.focusEvents === 0 && @@ -1626,10 +1701,7 @@ async function main() { ); const viewshedShot = path.join(SHOTS_DIR, 'cctv-v3-viewshed.png'); - await page.screenshot({ path: viewshedShot }); - console.log( - ` screenshot (visual review only) β†’ ${path.relative(REPO_ROOT, viewshedShot)}`, - ); + await saveReviewShot(page, viewshedShot, 'visual review only'); // ========================================================================= // Group 7: calibration gizmo (v3 design Β§3c β€” ADJUST mode + drag) @@ -1963,10 +2035,7 @@ async function main() { // ----------------------------------------------------------------------- await sleep(1000); const shot = path.join(SHOTS_DIR, 'cctv-v2-active.png'); - await page.screenshot({ path: shot }); - console.log( - ` screenshot (visual review only, not a gate) β†’ ${path.relative(REPO_ROOT, shot)}`, - ); + await saveReviewShot(page, shot, 'visual review only, not a gate'); // No console errors during the whole exercise. record( diff --git a/scripts/qa-panel-resize.mjs b/scripts/qa-panel-resize.mjs index d4c5b9287..c3669feb6 100644 --- a/scripts/qa-panel-resize.mjs +++ b/scripts/qa-panel-resize.mjs @@ -298,11 +298,12 @@ async function main() { ); console.log('PASS: header drag lifts CCTV out of the rail'); - // Park near the top: the voice dock would cover the south corners once - // the loop below has grown the window. + // Park above and right of the voice dock and its raised provider chooser: + // growing every edge must leave the south handles clear of both controls. { + const panel = await readPanel(); const header = await headerPoint(); - await drag(header, 0, 120 - header.y); + await drag(header, 900 - panel.left, 120 - header.y); } const limits = { diff --git a/server/providers/cctv.js b/server/providers/cctv.js index b24f38fa1..e549753e8 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -143,7 +143,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { if (url.pathname === '/sources') { const body = { - trimmedPacks: getCctvSources.trimmedPacks?.() ?? [], + trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [], sources: sources.map((source) => ({ id: source.id, pack: source.pack, diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 0e69ac3c9..13b817a83 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -204,16 +204,19 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { /** @type {Promise>|null} In-flight refresh, shared by concurrent * callers so a post-TTL burst launches ONE refetch, not one per request. */ let _cctvSourceInflight = null; - /** @type {Array<{pack: string, available: number, served: number}>} Packs - * that serve fewer cameras than they offer. */ - let _cctvTrimmedPacks = []; + /** @type {WeakMap, Array<{pack: string, available: number, served: number}>>} + * Packs that serve fewer cameras than they offer, keyed by the source list + * they describe, so a request served the old list during a background + * refresh never pairs it with the new list's trim notes. */ + const _cctvTrimmedPacks = new WeakMap(); /** * Assemble and cache the merged CCTV source list. * * Merges every source pack (live open-data packs, local file, env * variable), deduplicates by ID, shares the catalog cap fairly across - * packs, and caches for CCTV_SOURCE_CACHE_MS. + * packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is + * returned while a background refresh replaces it; only a first load waits. * * @returns {Promise>} Deduplicated, capped source list. */ @@ -228,11 +231,31 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { // Single-flight: a burst of requests arriving past the TTL shares ONE refresh // instead of each launching the full multi-provider refetch. The `.finally` // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { - _cctvSourceInflight = null; - }); - return _cctvSourceInflight; + if (!_cctvSourceInflight) { + _cctvSourceInflight = refreshCctvSources().finally(() => { + _cctvSourceInflight = null; + }); + // A background refresh has no caller waiting on it: report its failure + // here, once per refresh rather than once per request it serves. Keep + // the old catalog for another TTL, as an empty refresh does, so a refresh + // that keeps failing is retried once per TTL, not back to back. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + _cctvSourceCacheAt = Date.now(); + console.warn( + '[CCTV] background catalog refresh failed; serving', + _cctvSourceCache.length, + 'cameras from the last catalog:', + error?.message || error, + ); + }); + } + } + // With a catalog in hand, answer with it while the refresh runs: the + // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up + // to its 30 s timeout), and every /api/cctv request would otherwise wait + // for it. Only the first load, with nothing to serve yet, waits. + return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight; } /** @@ -322,7 +345,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { } if (capped.length > 0 || _cctvSourceCache.length === 0) { _cctvSourceCache = capped; - _cctvTrimmedPacks = trimmedPacks; + _cctvTrimmedPacks.set(capped, trimmedPacks); } else { // Every source came back empty (all live packs timed out / upstream outage) // but a good catalog is already cached β€” serve it stale rather than blanking @@ -336,7 +359,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { return _cctvSourceCache; } - /** Packs serving fewer cameras than they offer, as of the last refresh. */ - getCctvSources.trimmedPacks = () => _cctvTrimmedPacks; + /** Packs serving fewer cameras than they offer in `sources` (by default the + * current catalog). Pass the list a request was served to keep the two in + * step while a background refresh swaps the catalog. */ + getCctvSources.trimmedPacks = (sources = _cctvSourceCache) => + _cctvTrimmedPacks.get(sources) ?? []; return getCctvSources; } diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 8922e8938..ee50a6b69 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -503,6 +503,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; * pending forever β€” a hung fetch aborts, the loader returns [], and * serve-stale/other packs take over. */ export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000; +/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across + * ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian + * pack on about a third of loads. Both must arrive for any camera to load. */ +export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000; /** Individual CCTV image fetches must settle before the active 10-second * client refresh cadence. A bounded miss can fall through to Street View or * the synthetic frame instead of leaving the browser preview pending. */ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 4964f2aa0..4ed18417c 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -76,6 +76,7 @@ import { TORONTO_GROUND_ELEVATION_M, TORONTO_MAX_CATALOG_BYTES, CCTV_SOURCE_FETCH_TIMEOUT_MS, + TARKTEE_FETCH_TIMEOUT_MS, DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, @@ -1214,11 +1215,11 @@ export async function loadTarkteeSourcesFromDatex() { const [locResp, imgResp] = await Promise.all([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), fetch(TARKTEE_IMAGES_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); if (!locResp.ok) { diff --git a/server/providers/common/request.js b/server/providers/common/request.js index 93bbbaa85..3a45bdd30 100644 --- a/server/providers/common/request.js +++ b/server/providers/common/request.js @@ -14,16 +14,34 @@ async function readRequestBodyCapped(req, maxBytes) { return Buffer.concat(chunks); } -function readRequestBody(req, maxBytes = 1024 * 1024) { +/** Optional cancellation discards buffered data while the request drains. */ +function readRequestBody(req, maxBytes = 1024 * 1024, signal) { return new Promise((resolve, reject) => { let total = 0; let bodyTooLarge = false; + let settled = false; const chunks = []; - req.on('data', (chunk) => { + const stopReading = () => { + if (!signal) return; + req.off('data', data); + req.off('end', end); + signal.removeEventListener('abort', abort); + }; + const fail = (error) => { + if (signal) { + if (settled) return; + settled = true; + chunks.length = 0; + signal.removeEventListener('abort', abort); + } + reject(error); + }; + const abort = () => fail(signal.reason); + const data = (chunk) => { // Keep draining after the cap without retaining more bytes. Destroying // the request also destroys its socket, so the handler cannot deliver // the fixed error response it constructs for BODY_TOO_LARGE. - if (bodyTooLarge) return; + if (bodyTooLarge || settled) return; total += chunk.length; if (total > maxBytes) { // Same marker readRequestBodyCapped sets, so a caller can tell an @@ -32,15 +50,42 @@ function readRequestBody(req, maxBytes = 1024 * 1024) { chunks.length = 0; const error = new Error(`Request body exceeds ${maxBytes} bytes`); error.code = 'BODY_TOO_LARGE'; - reject(error); + fail(error); return; } chunks.push(chunk); - }); - req.on('end', () => { + }; + const end = () => { + if (settled) { + stopReading(); + return; + } if (!bodyTooLarge) resolve(Buffer.concat(chunks).toString('utf8')); - }); - req.on('error', reject); + if (signal) { + settled = true; + chunks.length = 0; + stopReading(); + } + }; + const close = () => { + fail( + new DOMException('Request closed before its body ended', 'AbortError'), + ); + stopReading(); + req.off('error', fail); + req.off('close', close); + }; + if (signal) { + // Abort/overflow release retained bytes but keep draining. Retain an + // error handler until close, including a late error after end/abort. + req.on('close', close); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + } + req.on('data', data); + req.on('end', end); + req.on('error', fail); + if (signal && req.destroyed) close(); }); } diff --git a/server/providers/common/request.test.mjs b/server/providers/common/request.test.mjs new file mode 100644 index 000000000..0360e6a04 --- /dev/null +++ b/server/providers/common/request.test.mjs @@ -0,0 +1,137 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter, getEventListeners } from 'node:events'; +import { readRequestBody } from './request.js'; + +function fixture() { + const req = new EventEmitter(); + req.pause = () => assert.fail('The reader must keep draining'); + req.destroy = () => + assert.fail('The reader must preserve the response socket'); + const owner = new AbortController(); + return { req, owner }; +} + +function finish(req, owner) { + assert.doesNotThrow(() => req.emit('error', new Error('late stream error'))); + req.emit('end'); + assert.doesNotThrow(() => req.emit('error', new Error('error before close'))); + req.emit('close'); + for (const event of ['data', 'end', 'error', 'close']) + assert.equal(req.listenerCount(event), 0, event); + assert.equal(getEventListeners(owner.signal, 'abort').length, 0); +} + +test('default and cancellable readers preserve byte caps and UTF-8 across chunks', async () => { + for (const useSignal of [false, true]) { + for (const text of ['', 'hello', 'Γ©']) { + const { req, owner } = fixture(); + const bytes = Buffer.from(text); + const body = readRequestBody( + req, + bytes.length, + useSignal ? owner.signal : undefined, + ); + for (const byte of bytes) req.emit('data', Buffer.from([byte])); + req.emit('end'); + assert.equal(await body, text); + if (useSignal) finish(req, owner); + } + } +}); + +test('default overflow rejects before end and drains without pausing or destroying', async () => { + const { req } = fixture(); + const rejection = assert.rejects(readRequestBody(req, 2), { + code: 'BODY_TOO_LARGE', + }); + req.emit('data', Buffer.from('Γ©')); + req.emit('data', Buffer.from('!')); + await rejection; + req.emit('data', Buffer.alloc(4096)); + assert.doesNotThrow(() => req.emit('error', new Error('late'))); + req.emit('end'); +}); + +test('partial abort settles before end, discards late data and guards errors until close', async () => { + const { req, owner } = fixture(); + const reason = new Error('request deadline'); + const rejection = assert.rejects( + readRequestBody(req, 1024, owner.signal), + (error) => error === reason, + ); + req.emit('data', Buffer.from('partial')); + owner.abort(reason); + await rejection; + assert.equal(getEventListeners(owner.signal, 'abort').length, 0); + req.emit('data', Buffer.alloc(4096)); + finish(req, owner); +}); + +test('already-aborted and already-closed requests settle and release their listeners', async () => { + for (const closed of [false, true]) { + const { req, owner } = fixture(); + if (closed) req.destroyed = true; + else owner.abort(); + await assert.rejects(readRequestBody(req, 1024, owner.signal), { + name: 'AbortError', + }); + if (!closed) finish(req, owner); + for (const event of ['data', 'end', 'error', 'close']) + assert.equal(req.listenerCount(event), 0); + assert.equal(getEventListeners(owner.signal, 'abort').length, 0); + } +}); + +test('overflow keeps its first failure through later abort and stream errors', async () => { + const { req, owner } = fixture(); + const rejection = assert.rejects(readRequestBody(req, 2, owner.signal), { + code: 'BODY_TOO_LARGE', + }); + req.emit('data', Buffer.from('Γ©!')); + await rejection; + owner.abort(new Error('later deadline')); + req.emit('data', Buffer.alloc(4096)); + finish(req, owner); +}); + +test('early close and request error reject once; successful end wins over later abort', async () => { + for (const event of ['close', 'error', 'end']) { + const { req, owner } = fixture(); + const reason = new Error('request failed'); + const body = readRequestBody(req, 32, owner.signal); + const observed = + event === 'end' + ? body.then((value) => assert.equal(value, 'ok')) + : assert.rejects( + body, + event === 'close' + ? { name: 'AbortError' } + : (error) => error === reason, + ); + req.emit('data', Buffer.from('ok')); + req.emit(event, reason); + owner.abort(new Error('later abort')); + await observed; + if (event !== 'close') finish(req, owner); + assert.equal(getEventListeners(owner.signal, 'abort').length, 0); + } +}); + +test('a prior data observer can abort synchronously without retaining or resolving that chunk', async () => { + const { req, owner } = fixture(); + const reason = new Error('observer stopped request'); + const external = () => owner.abort(reason); + req.on('data', external); + const rejection = assert.rejects( + readRequestBody(req, 1024, owner.signal), + (error) => error === reason, + ); + req.emit('data', Buffer.from('must be discarded')); + await rejection; + req.emit('end'); + req.emit('close'); + assert.deepEqual(req.listeners('data'), [external]); + assert.equal(getEventListeners(owner.signal, 'abort').length, 0); + req.off('data', external); +}); diff --git a/server/providers/gemini.js b/server/providers/gemini.js new file mode 100644 index 000000000..b3fe29e80 --- /dev/null +++ b/server/providers/gemini.js @@ -0,0 +1,22 @@ +import { createGeminiTokenHandler } from './gemini/realtime.js'; + +/** Mint constrained Live API credentials; the permanent key stays on Node. */ +function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) { + function install(middlewares) { + middlewares.use( + '/api/gemini/token', + createGeminiTokenHandler({ ...realtime, annotationGuidance }), + ); + } + return { + name: 'gemini-live-proxy', + configureServer(server) { + install(server.middlewares); + }, + configurePreviewServer(server) { + install(server.middlewares); + }, + }; +} + +export { geminiLiveProxy }; diff --git a/server/providers/gemini/config.js b/server/providers/gemini/config.js new file mode 100644 index 000000000..4fa438d15 --- /dev/null +++ b/server/providers/gemini/config.js @@ -0,0 +1,53 @@ +import { realtimeInstructions } from '../openai/instructions.js'; +import { GEV_REALTIME_TOOLS } from '../openai/tools.js'; + +export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live'; +export const GEMINI_TOKEN_ENDPOINT = + 'https://generativelanguage.googleapis.com/v1beta/auth_tokens'; +export const GEMINI_LIVE_WEBSOCKET_URL = + 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained'; + +export function normalizeGeminiModel(value) { + const model = String(value ?? GEMINI_LIVE_MODEL_DEFAULT) + .trim() + .replace(/^models\//, ''); + return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null; +} + +/** REST BidiGenerateContentSetup fields, excluding its model resource name. */ +export function createGeminiLiveConfig( + annotationGuidance, + inputMode = 'open-mic', + tools = GEV_REALTIME_TOOLS, +) { + if (!['open-mic', 'push-to-talk'].includes(inputMode)) + throw new Error('Invalid Gemini input mode'); + const config = { + generationConfig: { responseModalities: ['AUDIO'] }, + systemInstruction: { + parts: [{ text: realtimeInstructions(annotationGuidance) }], + }, + tools: [ + { + functionDeclarations: tools.map( + ({ name, description, parameters }) => ({ + name, + description, + // GEV's canonical schemas are JSON Schema, not Google's Schema. + parametersJsonSchema: structuredClone(parameters), + // App actions must finish before the model confirms their result. + behavior: 'BLOCKING', + }), + ), + }, + ], + inputAudioTranscription: {}, + outputAudioTranscription: {}, + contextWindowCompression: { slidingWindow: {} }, + }; + if (inputMode === 'push-to-talk') + config.realtimeInputConfig = { + automaticActivityDetection: { disabled: true }, + }; + return config; +} diff --git a/server/providers/gemini/realtime.js b/server/providers/gemini/realtime.js new file mode 100644 index 000000000..dbbdba345 --- /dev/null +++ b/server/providers/gemini/realtime.js @@ -0,0 +1,208 @@ +import { waitForSignal } from '../../../src/sources/requestDeadline.js'; +import { clientKey, makeCostRateLimiter } from '../common/rate-limit.js'; +import { readResponseJsonCapped } from '../../../src/sources/httpBody.js'; +import { + GEMINI_TOKEN_ENDPOINT, + GEMINI_LIVE_WEBSOCKET_URL, + GEMINI_LIVE_MODEL_DEFAULT, + createGeminiLiveConfig, + normalizeGeminiModel, +} from './config.js'; +import { + isGeminiRequestOriginAllowed, + readGeminiTokenRequest, +} from './request.js'; + +const UPSTREAM_RESPONSE_BYTES = 32 * 1024; +const MINT_FAILURE = 'Gemini voice could not connect. Try again.'; + +export function createGeminiTokenHandler({ + annotationGuidance, + tools, + fetchImpl = (...args) => fetch(...args), + resolveApiKey = () => process.env.GEMINI_API_KEY, + resolveModel = () => + process.env.GEMINI_LIVE_MODEL || GEMINI_LIVE_MODEL_DEFAULT, + resolveHost = () => process.env.HOST, + resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS, + resolveRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN ?? '30', + now = Date.now, + timeoutMs = 15_000, +} = {}) { + let limiter; + return async (req, res) => { + if (req.aborted || res.destroyed || res.writableEnded) return; + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('X-Content-Type-Options', 'nosniff'); + const reply = (status, errorOrPayload) => { + if (res.destroyed || res.writableEnded) return; + res.statusCode = status; + if (status >= 400) res.setHeader('Connection', 'close'); + res.end( + JSON.stringify( + typeof errorOrPayload === 'string' + ? { error: errorOrPayload } + : errorOrPayload, + ), + ); + }; + if (req.method !== 'POST') { + res.setHeader('Allow', 'POST'); + return reply(405, 'Method not allowed'); + } + let apiKey; + let model; + try { + if ( + !isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts()) + ) + return reply(403, 'Gemini voice requires a same-origin app request'); + apiKey = String(resolveApiKey() || '').trim(); + if (!apiKey) + return reply(503, 'Add a Gemini API key in Provider Settings'); + model = normalizeGeminiModel(resolveModel()); + if (!model) + return reply(503, 'The Gemini Live model configuration is invalid'); + if (limiter === undefined) + limiter = makeCostRateLimiter(resolveRateLimit(), 30); + if (limiter && !limiter(clientKey(req))) { + res.setHeader('Retry-After', '60'); + return reply( + 429, + 'Gemini voice is starting too often. Try again shortly.', + ); + } + } catch { + // Connect does not observe the handler's returned promise. A synchronous + // configuration resolver must fail with a bounded, fixed response. + return reply(503, MINT_FAILURE); + } + + const lifetime = new AbortController(); + let disconnected = false; + const disconnect = () => { + disconnected = true; + lifetime.abort(new DOMException('Request closed', 'AbortError')); + }; + const responseClosed = () => { + if (!res.writableEnded) disconnect(); + }; + req.once('aborted', disconnect); + res.once('close', responseClosed); + const deadline = setTimeout(() => { + lifetime.abort( + new DOMException('Token request timed out', 'TimeoutError'), + ); + }, timeoutMs); + deadline.unref?.(); + let acquiredResponse; + try { + const inputMode = await readGeminiTokenRequest(req, lifetime.signal); + lifetime.signal.throwIfAborted(); + const config = createGeminiLiveConfig( + annotationGuidance, + inputMode, + tools, + ); + const issuedAt = now(); + // REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK + // input shape, converted before transport. With no fieldMask the complete + // setup is locked to this server-owned configuration. + const request = { + uses: 1, + expireTime: new Date(issuedAt + 30 * 60_000).toISOString(), + newSessionExpireTime: new Date(issuedAt + 60_000).toISOString(), + bidiGenerateContentSetup: { model: `models/${model}`, ...config }, + }; + const acquisition = Promise.resolve( + fetchImpl(GEMINI_TOKEN_ENDPOINT, { + method: 'POST', + redirect: 'error', + signal: lifetime.signal, + headers: { + 'x-goog-api-key': apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(request), + }), + ); + // A fetch implementation can ignore abort. Bound this caller's wait and + // discard a response that arrives after timeout/disconnection. + void acquisition.then( + (late) => { + acquiredResponse = late; + if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {}); + }, + () => {}, + ); + const response = await waitForSignal(acquisition, lifetime.signal); + lifetime.signal.throwIfAborted(); + if ( + response.redirected || + (response.url && response.url !== GEMINI_TOKEN_ENDPOINT) + ) { + void response.body?.cancel().catch(() => {}); + throw new Error('Unexpected token response destination'); + } + if (!response.ok) { + // Never forward upstream bodies, headers or exception messages. + void response.body?.cancel().catch(() => {}); + if (response.status === 429) { + res.setHeader('Retry-After', '60'); + return reply(429, 'Gemini quota is unavailable. Try again later.'); + } + if (response.status === 401 || response.status === 403) + return reply( + 503, + 'Gemini access was refused. Check the API key and model access.', + ); + if (response.status === 400 || response.status === 404) + return reply( + 502, + 'Gemini Live is unavailable for the configured model.', + ); + return reply(502, MINT_FAILURE); + } + const data = await readResponseJsonCapped( + response, + UPSTREAM_RESPONSE_BYTES, + lifetime.signal, + ); + lifetime.signal.throwIfAborted(); + if ( + !data || + Array.isArray(data) || + typeof data !== 'object' || + typeof data.name !== 'string' || + data.name.length > 8192 || + !/^auth_tokens\/[A-Za-z0-9._~+/=-]+$/.test(data.name) || + data.name.includes(apiKey) + ) + throw new Error('Invalid ephemeral token'); + reply(200, { + token: data.name, + model, + config, + websocketUrl: GEMINI_LIVE_WEBSOCKET_URL, + }); + } catch (error) { + if (disconnected || req.aborted) return; + if (error?.code === 'BODY_TOO_LARGE') + return reply(413, 'Token request is too large'); + if (error?.code === 'BAD_CONTENT_TYPE') + return reply(415, 'Use an empty JSON token request'); + if (error?.code === 'BAD_BODY') + return reply(400, 'Use an empty JSON token request'); + reply(lifetime.signal.aborted ? 504 : 502, MINT_FAILURE); + } finally { + // Abort can happen after acquisition resolves but before this handler + // resumes and passes ownership to the body reader. + if (lifetime.signal.aborted) + void acquiredResponse?.body?.cancel().catch(() => {}); + clearTimeout(deadline); + req.off('aborted', disconnect); + res.off('close', responseClosed); + } + }; +} diff --git a/server/providers/gemini/request.js b/server/providers/gemini/request.js new file mode 100644 index 000000000..3dde288d6 --- /dev/null +++ b/server/providers/gemini/request.js @@ -0,0 +1,79 @@ +import { readRequestBody } from '../common/request.js'; +import { admitSameSiteRequest } from '../../../src/localRequestGate.mjs'; +import { + isAllowedHost, + resolveAllowedHosts, +} from '../../../build/allowedHosts.js'; + +/** Use the app's Host and request policies, with a required browser Origin. */ +export function isGeminiRequestOriginAllowed( + req, + configuredHost = '', + configuredAllowedHosts, +) { + const host = req.headers?.host; + const origin = req.headers?.origin; + if ( + typeof host !== 'string' || + typeof origin !== 'string' || + !host || + !origin || + /[\s/?#@\\]/.test(host) + ) + return false; + const protocol = req.socket?.encrypted ? 'https:' : 'http:'; + if ( + !admitSameSiteRequest({ + hostHeader: host, + protocol, + origin, + secFetchSite: req.headers?.['sec-fetch-site'], + proxyHeaders: req.headers || {}, + }).ok + ) + return false; + // The shared gate compares a present Origin exactly with this authority; + // this route also requires the Origin and an allowed Host. + const boundHost = String(configuredHost || '') + .trim() + .toLowerCase(); + return isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [ + boundHost, + ]); +} + +/** Admit only the existing input gesture; callers cannot supply model/setup. */ +export async function readGeminiTokenRequest(req, signal, maxBytes = 1024) { + const declared = Number(req.headers?.['content-length']); + const fail = (code) => + Object.assign(new Error('Invalid token request'), { code }); + const body = await readRequestBody( + req, + maxBytes, + Number.isFinite(declared) && declared > maxBytes + ? AbortSignal.abort(fail('BODY_TOO_LARGE')) + : signal, + ); + if (!body.trim()) return 'open-mic'; + if ( + !/^application\/json(?:\s*;|$)/i.test(req.headers?.['content-type'] || '') + ) + throw fail('BAD_CONTENT_TYPE'); + let parsed; + try { + parsed = JSON.parse(body); + } catch { + throw fail('BAD_BODY'); + } + if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') + throw fail('BAD_BODY'); + const keys = Object.keys(parsed); + if (!keys.length) return 'open-mic'; + if ( + keys.length === 1 && + keys[0] === 'inputMode' && + parsed.inputMode === 'push-to-talk' + ) + return 'push-to-talk'; + throw fail('BAD_BODY'); +} diff --git a/server/providers/local.js b/server/providers/local.js index 854b77f33..5d39c3a01 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -19,6 +19,7 @@ import { adsbLolProxy } from './aircraft/adsb-lol.js'; import { aisLiveProxy } from './vessels/ais-live.js'; import { trackBackfillProxies } from './aircraft/tracks.js'; import { openAiRealtimeProxy } from './openai.js'; +import { geminiLiveProxy } from './gemini.js'; import { googlePlacesContextProxy } from './places.js'; import { mapillaryProxy } from './mapillary.js'; import { keySetupEndpoint } from '../standalone/key-setup.js'; @@ -30,9 +31,9 @@ import { windProxy } from './wind.js'; /** * Construct the local provider plugins in their established order. - * `realtime` configures the voice session token endpoint. + * `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live. */ -function localProviderPlugins({ realtime } = {}) { +function localProviderPlugins({ realtime, gemini } = {}) { return [ openSkyProxy(), celestrakProxy(), @@ -55,6 +56,7 @@ function localProviderPlugins({ realtime } = {}) { aisLiveProxy(), trackBackfillProxies(), openAiRealtimeProxy({ realtime }), + ...(gemini ? [geminiLiveProxy({ realtime: gemini })] : []), googlePlacesContextProxy(), windProxy(), weatherProxy(), diff --git a/server/standalone/vite.config.js b/server/standalone/vite.config.js index 03a935092..ffe5124ce 100644 --- a/server/standalone/vite.config.js +++ b/server/standalone/vite.config.js @@ -16,9 +16,10 @@ export default defineConfig(({ command, mode }) => { for (const [key, value] of Object.entries(loaded)) { if (process.env[key] === undefined) process.env[key] = value; } + const tools = standaloneVoiceTools(); return createBrowserViteConfig({ plugins: [ - ...localProviderPlugins({ realtime: { tools: standaloneVoiceTools() } }), + ...localProviderPlugins({ realtime: { tools }, gemini: { tools } }), localMcpPlugin(), apiNotFoundPlugin(), unwatchedRuntimeDirsPlugin(), diff --git a/src/agent/worldPacket.js b/src/agent/worldPacket.js new file mode 100644 index 000000000..4f8dad144 --- /dev/null +++ b/src/agent/worldPacket.js @@ -0,0 +1,170 @@ +import * as Cesium from 'cesium'; +import { + feedProvenanceEnvelope, + layerSnapshot, + layerSnapshots, +} from '../data/layerSnapshot.js'; +import { contextModeWord } from '../contextModePolicy.js'; + +export const TRACKABLE_FAMILIES = Object.freeze([ + { layerId: 'flights', kind: 'aircraft' }, + { layerId: 'military', kind: 'aircraft' }, + { layerId: 'ais-live-vessels', kind: 'vessel' }, + { layerId: 'satellites', kind: 'satellite' }, +]); + +const CONTEXT_MODE_RESULT_FIELDS = Object.freeze([ + { field: 'mode', emptyAs: 'off' }, + { field: 'entering', emptyAs: null }, + { field: 'priorMode', emptyAs: null }, +]); + +const NESTED_CONTEXT_RESULT_FIELDS = Object.freeze([ + 'context', + 'contextRollback', +]); + +/** + * Translate context-mode payloads into the public tool vocabulary while + * retaining the internal id beside each translated field. + */ +export function withContextModeVocabulary(state) { + if (!state || typeof state !== 'object') return state; + let out = state; + const mutable = () => { + if (out === state) out = { ...state }; + return out; + }; + for (const { field, emptyAs } of CONTEXT_MODE_RESULT_FIELDS) { + if (!(field in state)) continue; + const internal = state[field] ?? null; + const target = mutable(); + target[field] = contextModeWord(internal, { emptyAs }); + target[`${field}Internal`] = internal; + } + for (const field of NESTED_CONTEXT_RESULT_FIELDS) { + const nested = state[field]; + if (!nested || typeof nested !== 'object') continue; + const translated = withContextModeVocabulary(nested); + if (translated !== nested) mutable()[field] = translated; + } + return out; +} + +/** Read tracked entities from their canonical layer owners. */ +export function collectTrackedEntities(dataManager) { + const tracked = []; + for (const family of TRACKABLE_FAMILIES) { + const module = dataManager?.layers?.get(family.layerId)?.module; + if (!module) continue; + try { + const info = + family.kind === 'vessel' + ? module.getSelectedInfo?.() + : module.getTrackedInfo?.(); + if (info) + tracked.push({ + kind: family.kind, + layerId: family.layerId, + ...info, + }); + } catch { + // A not-yet-ready layer contributes no tracked observation. + } + } + return tracked; +} + +/** Read the active Contacts window from its owning awareness layer. */ +export function activeContactsWindow(dataManager) { + try { + const layer = dataManager?.layers?.get('military-awareness')?.module; + return ( + layer?.contactsWindowFromSnapshot?.(layer.getContextSnapshot?.()) ?? null + ); + } catch { + return null; + } +} + +/** + * Build the model-free fast World Packet from existing application owners. + * + * This performs no network requests and intentionally mirrors the historical + * get_current_view_state payload so voice can delegate without behavior drift. + */ +export function getWorldPacket(options = {}) { + const { + viewer, + styleManager, + dataManager, + sceneDirector = null, + now: requestedNow, + } = options ?? {}; + // A default parameter only covers undefined; null or NaN would age every + // feed against the epoch and report stale data as "just now". + const now = Number.isFinite(requestedNow) ? requestedNow : Date.now(); + if (!viewer?.camera?.positionWC) + throw new TypeError('World Packet requires a viewer camera'); + if (!dataManager || typeof dataManager.getAll !== 'function') + throw new TypeError('World Packet requires a data manager'); + + const cartographic = Cesium.Cartographic.fromCartesian( + viewer.camera.positionWC, + ); + // Read the roster once. Voice must not rewalk disabled layers: preserving the + // current enabled-only view keeps the model's context budget bounded. + const rows = dataManager.getAll(); + const enabledRows = rows.filter((layer) => layer.enabled); + const contactsWindow = activeContactsWindow(dataManager); + const contextState = + typeof styleManager?.getContextModeState === 'function' + ? withContextModeVocabulary(styleManager.getContextModeState()) + : null; + + return { + ok: true, + action: 'get_current_view_state', + camera: { + latitude: Cesium.Math.toDegrees(cartographic.latitude), + longitude: Cesium.Math.toDegrees(cartographic.longitude), + heightM: cartographic.height, + }, + style: styleManager?.activeStyle || 'normal', + context: contextState + ? { + ...contextState, + ...(contactsWindow ? { contactsWindow } : {}), + } + : null, + cockpit: + typeof styleManager?.getCockpitState === 'function' + ? styleManager.getCockpitState() + : null, + controls: + typeof styleManager?.getControlState === 'function' + ? styleManager.getControlState() + : null, + scenePlayback: sceneDirector?.getPlaybackStatus?.() || null, + tracked: collectTrackedEntities(dataManager), + disabledLayerCount: rows.length - enabledRows.length, + // Preserve get_current_view_state's existing compact enabled-only shape. + layers: enabledRows.map((layer) => { + const snapshot = layerSnapshot(layer, { now }); + return { + id: layer.id, + name: layer.name, + enabled: layer.enabled, + count: layer.stats?.count || 0, + error: layer.stats?.error || null, + feedState: snapshot.feedState, + source: snapshot.source, + lastUpdate: snapshot.lastUpdate, + }; + }), + feedProvenance: feedProvenanceEnvelope( + layerSnapshots(rows, { now }).filter((layer) => layer.enabled), + { now }, + ), + }; +} diff --git a/src/agent/worldPacket.test.mjs b/src/agent/worldPacket.test.mjs new file mode 100644 index 000000000..063e3652c --- /dev/null +++ b/src/agent/worldPacket.test.mjs @@ -0,0 +1,335 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as Cesium from 'cesium'; + +import { getWorldPacket } from './worldPacket.js'; +import { createGevActionRunner } from '../voice/gevActions.js'; + +function withoutVoicePresentation(result) { + const packet = { ...result }; + delete packet.say; + delete packet.display; + delete packet.schedule; + return packet; +} + +function harness() { + const position = Cesium.Cartesian3.fromDegrees(-97.74, 30.26, 500); + const viewer = { + trackedEntity: undefined, + clock: { onTick: { addEventListener: () => () => {} } }, + scene: { + canvas: { + clientWidth: 1200, + clientHeight: 800, + addEventListener() {}, + removeEventListener() {}, + }, + globe: { getHeight: () => 0 }, + tweens: [], + }, + camera: { + positionWC: position, + positionCartographic: Cesium.Cartographic.fromCartesian(position), + heading: 0, + pitch: Cesium.Math.toRadians(-45), + moveEnd: { addEventListener: () => () => {} }, + cancelFlight() {}, + lookAtTransform() {}, + }, + }; + const rows = [ + { + id: 'traffic', + name: 'Traffic', + enabled: true, + stats: { + count: 3, + source: 'fixture', + lastUpdate: null, + }, + }, + ]; + const dataManager = { + layers: new Map([['traffic', { module: null }]]), + getAll: () => rows, + }; + const styleManager = { + activeStyle: 'normal', + getContextModeState: () => ({ mode: 'flights' }), + getCockpitState: () => null, + getControlState: () => ({ hud: { visible: 'auto' } }), + }; + return { viewer, dataManager, styleManager }; +} + +test('World Packet preserves get_current_view_state parity', async () => { + const { viewer, dataManager, styleManager } = harness(); + const packet = getWorldPacket({ viewer, dataManager, styleManager }); + const runner = createGevActionRunner({ viewer, dataManager, styleManager }); + const voice = await runner('get_current_view_state'); + + assert.deepEqual(withoutVoicePresentation(voice), packet); + assert.equal(packet.context.mode, 'contacts'); + assert.equal(packet.context.modeInternal, 'flights'); + assert.equal(packet.layers[0].feedState, 'nominal'); + assert.equal(packet.feedProvenance.overall, 'nominal'); +}); + +test('World Packet keeps stale/unavailable feed semantics distinct from empty', () => { + const { viewer, dataManager, styleManager } = harness(); + dataManager.getAll = () => [ + { + id: 'traffic', + name: 'Traffic', + enabled: true, + stats: { + count: 0, + source: 'fixture', + error: 'fixture unavailable', + lastUpdate: null, + }, + }, + ]; + + const packet = getWorldPacket({ viewer, dataManager, styleManager }); + assert.equal(packet.layers[0].count, 0); + assert.notEqual(packet.layers[0].feedState, 'nominal'); + assert.notEqual(packet.feedProvenance.overall, 'nominal'); + assert.match(packet.feedProvenance.note, /fixture unavailable/i); +}); + +// The 2026-10 main voice contract omits disabled layers from the model payload. +test('World Packet and voice preserve enabled-only payload and count disabled layers', async () => { + const { viewer, dataManager, styleManager } = harness(); + const rows = dataManager.getAll(); + rows.push({ + id: 'earthquakes', + name: 'Earthquakes', + enabled: false, + stats: { count: 99, source: 'fixture', error: 'stale' }, + }); + let reads = 0; + dataManager.getAll = () => { + reads += 1; + return rows; + }; + const packet = getWorldPacket({ viewer, dataManager, styleManager }); + assert.equal(reads, 1, 'collect the roster once rather than repeatedly'); + assert.equal(packet.disabledLayerCount, 1); + assert.deepEqual( + packet.layers.map((layer) => layer.id), + ['traffic'], + ); + assert.equal(packet.feedProvenance.overall, 'nominal'); + + const runner = createGevActionRunner({ viewer, dataManager, styleManager }); + const voice = await runner('get_current_view_state'); + assert.deepEqual(withoutVoicePresentation(voice), packet); +}); + +const PINNED_NOW = Date.UTC(2026, 9, 17, 12, 0, 0); + +// Exercises every packet field: context vocabulary and Contacts window, +// cockpit, controls, scene playback, tracked families (one not ready), +// enabled and disabled layers, and an aged and an unavailable feed. +function pinnedHarness() { + const { viewer } = harness(); + const rows = [ + { + id: 'flights', + name: 'Flights', + enabled: true, + stats: { + count: 12, + source: 'fixture-adsb', + lastUpdate: PINNED_NOW - 90_000, + }, + }, + { + id: 'traffic', + name: 'Traffic', + enabled: true, + stats: { + count: 0, + source: 'fixture', + error: 'fixture unavailable', + lastUpdate: null, + }, + }, + { + id: 'earthquakes', + name: 'Earthquakes', + enabled: false, + stats: { count: 99, source: 'fixture', error: 'stale' }, + }, + ]; + const module = (methods) => ({ module: methods }); + const dataManager = { + layers: new Map([ + [ + 'flights', + module({ + getTrackedInfo: () => ({ icao24: 'abc123', callsign: 'TEST1' }), + }), + ], + [ + 'ais-live-vessels', + module({ getSelectedInfo: () => ({ mmsi: '366000001' }) }), + ], + [ + 'satellites', + module({ + getTrackedInfo: () => { + throw new Error('not ready'); + }, + }), + ], + [ + 'military-awareness', + module({ + getContextSnapshot: () => ({ snapshot: true }), + contactsWindowFromSnapshot: (snapshot) => + snapshot ? { windowMin: 30, count: 4 } : null, + }), + ], + ]), + getAll: () => rows, + }; + const styleManager = { + activeStyle: 'night-vision', + getContextModeState: () => ({ + mode: 'flights', + entering: null, + priorMode: null, + }), + getCockpitState: () => ({ active: false }), + getControlState: () => ({ hud: { visible: 'auto' } }), + }; + const sceneDirector = { + getPlaybackStatus: () => ({ playing: false, sceneId: 'fixture' }), + }; + return { viewer, dataManager, styleManager, sceneDirector }; +} + +// Captured from main's get_current_view_state before the extraction +// (bilawalsidhu/gods-eye-view@3d8e1ef0) with Date.now pinned to PINNED_NOW, +// minus the speech envelope. Key order is part of the contract. +const PRE_EXTRACTION_PAYLOAD = { + ok: true, + action: 'get_current_view_state', + camera: { + latitude: 30.259999999999994, + longitude: -97.74, + heightM: 499.99999999901877, + }, + style: 'night-vision', + context: { + mode: 'contacts', + entering: null, + priorMode: null, + modeInternal: 'flights', + enteringInternal: null, + priorModeInternal: null, + contactsWindow: { windowMin: 30, count: 4 }, + }, + cockpit: { active: false }, + controls: { hud: { visible: 'auto' } }, + scenePlayback: { playing: false, sceneId: 'fixture' }, + tracked: [ + { + kind: 'aircraft', + layerId: 'flights', + icao24: 'abc123', + callsign: 'TEST1', + }, + { kind: 'vessel', layerId: 'ais-live-vessels', mmsi: '366000001' }, + ], + disabledLayerCount: 1, + layers: [ + { + id: 'flights', + name: 'Flights', + enabled: true, + count: 12, + error: null, + feedState: 'nominal', + source: 'fixture-adsb', + lastUpdate: 1792238310000, + }, + { + id: 'traffic', + name: 'Traffic', + enabled: true, + count: 0, + error: 'fixture unavailable', + feedState: 'unavailable', + source: 'fixture', + lastUpdate: 0, + }, + ], + feedProvenance: { + overall: 'unavailable', + layers: [ + { + id: 'flights', + name: 'Flights', + enabled: true, + feedState: 'nominal', + source: 'fixture-adsb', + count: 12, + lastUpdate: 1792238310000, + ageSec: 90, + ageLabel: '1m ago', + error: null, + }, + { + id: 'traffic', + name: 'Traffic', + enabled: true, + feedState: 'unavailable', + source: 'fixture', + count: 0, + lastUpdate: 0, + ageSec: null, + ageLabel: null, + error: 'fixture unavailable', + }, + ], + note: 'Traffic is UNAVAILABLE (fixture, fixture unavailable). Narrate that provenance with any count; do not describe this as live.', + }, +}; + +function assertPinnedPayload(actual) { + assert.deepEqual(actual, PRE_EXTRACTION_PAYLOAD); + // deepEqual ignores key order; the serialized form does not. + assert.equal(JSON.stringify(actual), JSON.stringify(PRE_EXTRACTION_PAYLOAD)); +} + +test('World Packet matches the pre-extraction get_current_view_state payload', () => { + const packet = getWorldPacket({ ...pinnedHarness(), now: PINNED_NOW }); + assertPinnedPayload(packet); +}); + +test('voice get_current_view_state still returns the pre-extraction payload', async (t) => { + t.mock.method(Date, 'now', () => PINNED_NOW); + const runner = createGevActionRunner(pinnedHarness()); + const voice = await runner('get_current_view_state'); + assertPinnedPayload(withoutVoicePresentation(voice)); +}); + +test('World Packet ages feeds against the clock when now is not a finite number', (t) => { + t.mock.method(Date, 'now', () => PINNED_NOW); + for (const now of [null, Number.NaN, '2026-10-17']) { + const packet = getWorldPacket({ ...pinnedHarness(), now }); + const flights = packet.feedProvenance.layers.find( + (layer) => layer.id === 'flights', + ); + assert.equal(flights.ageSec, 90, `now: ${String(now)}`); + assert.equal(flights.ageLabel, '1m ago', `now: ${String(now)}`); + } +}); + +test('World Packet rejects null options with the camera requirement', () => { + assert.throws(() => getWorldPacket(null), /requires a viewer camera/); +}); diff --git a/src/data/cctv.test.mjs b/src/data/cctv.test.mjs index ca6a3c11a..dafaeb5f6 100644 --- a/src/data/cctv.test.mjs +++ b/src/data/cctv.test.mjs @@ -77,6 +77,11 @@ import { CCTV_FOCUS_REQUEST_EVENT, activateCctvCameraFromWorldClick, } from '../cctvFocusRequest.js'; +import { + ACTIVE_CAMERA_COLOR, + IDLE_CAMERA_COLOR, + LIVE_CAMERA_COLOR, +} from '../layers/cctv/policy.js'; const CCTV_PRESENTATION_SOURCE = fs.readFileSync( path.join( @@ -360,6 +365,37 @@ test('default COVERAGE refresh materializes the active and visible camera frustu } }); +test('camera icons are violet for video feeds, cyan for stills, amber when active', () => { + const record = (id, feedType) => ({ + camera: { ...UNCLAMPED_CAMERA, id, feedType, groundElevationM: 0 }, + billboard: {}, + coverageEntities: [], + projection: null, + }); + const idleVideo = record('idle-hls', 'hls'); + const idleStill = record('idle-image', 'image'); + const activeVideo = record('active-hls', 'hls'); + const viewer = { entities: { add: (entity) => entity } }; + + try { + // Disabled keeps the active video camera from opening a real projection; + // icon colors are assigned either way. + _setCctvCoverageStateForTest({ + viewer, + records: [idleVideo, idleStill, activeVideo], + activeCameraId: activeVideo.camera.id, + enabled: false, + coverageMode: 'off', + }); + refreshCoverageStyles(); + assert.equal(idleVideo.billboard.color, LIVE_CAMERA_COLOR); + assert.equal(idleStill.billboard.color, IDLE_CAMERA_COLOR); + assert.equal(activeVideo.billboard.color, ACTIVE_CAMERA_COLOR); + } finally { + _setCctvCoverageStateForTest({ enabled: false }); + } +}); + test('geometry drain pacing yields to tracked and cockpit camera ownership', () => { assert.deepEqual(cctvGeometryDrainPacing(), { batchSize: 4, delayMs: 120 }); assert.deepEqual( diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 9d93778db..7ef4b1f7b 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -6,6 +6,7 @@ import { } from '../../server/providers/cctv/cap.js'; import { CCTV_MAX_SOURCES_CEILING, + CCTV_SOURCE_CACHE_MS, DEFAULT_CCTV_MAX_SOURCES, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -160,3 +161,224 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.equal(kept.available, 5); assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); + +/** A CCTV_SOURCES_JSON value with `count` cameras named `-`. */ +const envSources = (prefix, count = 3) => + JSON.stringify( + Array.from({ length: count }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + +/** Let a background refresh that needs no timers run to completion. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +/** Restore process.env to a snapshot taken with `{ ...process.env }`. */ +const restoreEnv = (saved) => { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); +}; + +test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { + const saved = { ...process.env }; + try { + // Live packs run alongside the env pack, so one slow upstream can hold + // the refresh open. + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'log', () => {}); + const hung = []; + let hang = false; + t.mock.method(globalThis, 'fetch', () => + hang + ? new Promise((_, reject) => { + hung.push(() => reject(new Error('upstream gave up'))); + }) + : Promise.reject(new Error('offline')), + ); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // Past the TTL, with every live pack's upstream hanging. + now += CCTV_SOURCE_CACHE_MS + 1; + hang = true; + process.env.CCTV_SOURCES_JSON = envSources('new'); + let timer; + const served = await Promise.race([ + getSources(), + new Promise((resolve) => { + timer = setTimeout(() => resolve('waited'), 200); + }), + ]); + clearTimeout(timer); + assert.notEqual( + served, + 'waited', + 'a stale request waited for the background refresh', + ); + assert.deepEqual( + served.map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // Requests during the refresh share it rather than starting another. + const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); + await pause(); + const inFlight = hung.length; + assert.ok(inFlight > 0, 'the refresh reached the hanging upstreams'); + await getSources(); + await getSources(); + await pause(); + assert.equal(hung.length, inFlight, 'only one refresh ran'); + + // When the slow pack gives up, the refreshed catalog replaces the old one. + hang = false; + for (const giveUp of hung) giveUp(); + for (let i = 0; i < 50; i += 1) { + const ids = (await getSources()).map((s) => s.id); + if (ids[0] === 'new-0') break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + } finally { + restoreEnv(saved); + } +}); + +test('a failed background refresh keeps the catalog, warns once, and waits a TTL to retry', async (t) => { + const saved = { ...process.env }; + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + process.env.CCTV_PREFER_AUSTIN = '0'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old', 9); + process.env.CCTV_MAX_SOURCES = '8'; + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + // Nine cameras over a cap of eight make every refresh warn; throwing from + // that warning is how this test makes a refresh reject. + let failRefresh = false; + let failedRefreshes = 0; + const warnings = []; + t.mock.method(console, 'warn', (...args) => { + const text = args.join(' '); + if (failRefresh && text.includes('exceeds cap')) { + failedRefreshes += 1; + throw new Error('refresh broke'); + } + warnings.push(text); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + assert.equal(old.length, 8); + + // Past the TTL the refresh throws: the old catalog is still served. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = true; + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1); + assert.equal( + warnings.filter((w) => w.includes('background catalog refresh failed')) + .length, + 1, + ); + + // The failure is not retried on every request until the next TTL. + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1, 'a failed refresh was retried at once'); + + // After it, a refresh that works replaces the catalog. With no live packs + // the refresh finishes before the request that started it returns. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = false; + process.env.CCTV_SOURCES_JSON = envSources('new'); + await getSources(); + await settle(); + const fresh = await getSources(); + assert.deepEqual( + fresh.map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + // Trim notes stay with the list they describe, so a request served the + // old list during the refresh never reports the new list's notes. + assert.deepEqual( + getSources.trimmedPacks(old).map((p) => [p.pack, p.available, p.served]), + [['env', 9, 8]], + ); + assert.deepEqual(getSources.trimmedPacks(fresh), []); + assert.deepEqual(getSources.trimmedPacks(), []); + assert.deepEqual(unhandled, []); + } finally { + process.off('unhandledRejection', onUnhandled); + restoreEnv(saved); + } +}); + +test('an empty background refresh keeps the catalog and waits a TTL before fetching again', async (t) => { + const saved = { ...process.env }; + try { + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'log', () => {}); + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + let fetches = 0; + t.mock.method(globalThis, 'fetch', () => { + fetches += 1; + return Promise.reject(new Error('offline')); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + + // Past the TTL every pack comes back empty, as in an upstream outage. + now += CCTV_SOURCE_CACHE_MS + 1; + process.env.CCTV_SOURCES_JSON = '[]'; + assert.equal(await getSources(), old); + for (let i = 0; i < 50; i += 1) { + if (warnings.some((w) => w.includes('source refresh returned empty'))) + break; + await settle(); + } + assert.ok( + warnings.some((w) => w.includes('serving 3 stale cameras')), + 'the empty refresh kept the old catalog', + ); + + // Until the next TTL, requests are served without touching any upstream. + const fetchesAfterRefresh = fetches; + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal( + fetches, + fetchesAfterRefresh, + 'an empty refresh reran at once', + ); + } finally { + restoreEnv(saved); + } +}); diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 7c0ebaba1..b46f9eb06 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -2,6 +2,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { loadTallinnSourcesFromCatalog, + loadTarkteeSourcesFromDatex, parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; @@ -84,3 +85,26 @@ test('Tarktee DATEX image parser rejects non-official hosts', () => { const images = parseTarkteeDatexImages(xml); assert.equal(images.size, 0); }); + +test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', async (t) => { + const timeouts = []; + const timeout = AbortSignal.timeout; + t.mock.method(AbortSignal, 'timeout', (ms) => { + timeouts.push(ms); + return timeout.call(AbortSignal, ms); + }); + t.mock.method(console, 'log', () => {}); + t.mock.method( + globalThis, + 'fetch', + async (url) => + new Response( + String(url).includes('roadCameraLocations') + ? SAMPLE_LOCATIONS_XML + : SAMPLE_IMAGES_XML, + ), + ); + const cameras = await loadTarkteeSourcesFromDatex(); + assert.deepEqual(timeouts, [30_000, 30_000]); + assert.equal(cameras.length, 1); +}); diff --git a/src/geminiComposition.test.mjs b/src/geminiComposition.test.mjs new file mode 100644 index 000000000..1dcd7f1c4 --- /dev/null +++ b/src/geminiComposition.test.mjs @@ -0,0 +1,115 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { localProviderPlugins } from '../server/providers/local.js'; +import { standaloneVoiceTools } from '../server/standalone/voiceTools.js'; +import { + KEY_SETUP_KEYS, + keySetupStatus, + validateKeySetupUpdates, + upsertDotenvValues, +} from './keySetupCore.mjs'; + +test('local hosts opt into Gemini explicitly and retain the default plugin order', (t) => { + t.mock.method(globalThis, 'fetch', () => { + throw Error('construction must not fetch'); + }); + const defaults = localProviderPlugins().map((plugin) => plugin.name); + assert.ok(!defaults.includes('gemini-live-proxy')); + const enabled = localProviderPlugins({ gemini: {} }).map( + (plugin) => plugin.name, + ); + assert.equal( + enabled.filter((name) => name === 'gemini-live-proxy').length, + 1, + ); + assert.deepEqual( + enabled.filter((name) => name !== 'gemini-live-proxy'), + defaults, + ); +}); + +for (const hook of ['configureServer', 'configurePreviewServer']) { + test(`opted-in ${hook} locks the standalone catalog into the Gemini token`, async () => { + const tools = standaloneVoiceTools(); + const calls = []; + const plugin = localProviderPlugins({ + gemini: { + tools, + resolveApiKey: () => 'fixture-key', + resolveModel: () => 'gemini-3.8-live', + resolveHost: () => '', + resolveAllowedHosts: () => '', + resolveRateLimit: () => '0', + fetchImpl: async (url, options) => { + calls.push({ url, options }); + return Response.json({ name: 'auth_tokens/fixture' }); + }, + }, + }).find((entry) => entry.name === 'gemini-live-proxy'); + let handler; + plugin[hook]({ + middlewares: { + use(route, value) { + assert.equal(route, '/api/gemini/token'); + handler = value; + }, + }, + }); + const req = new PassThrough(); + req.method = 'POST'; + req.headers = { + host: 'localhost:5173', + origin: 'http://localhost:5173', + 'content-type': 'application/json', + }; + req.socket = { remoteAddress: '127.0.0.1' }; + const res = new EventEmitter(); + res.setHeader = () => {}; + res.end = (body) => { + res.body = JSON.parse(body); + res.writableEnded = true; + }; + req.end('{}'); + await handler(req, res); + assert.equal(res.statusCode, 200); + assert.equal(calls.length, 1); + const locked = JSON.parse(calls[0].options.body).bidiGenerateContentSetup + .tools[0].functionDeclarations; + assert.deepEqual( + locked.map(({ name }) => name), + tools.map(({ name }) => name), + ); + assert.deepEqual( + locked.map(({ parametersJsonSchema }) => parametersJsonSchema), + tools.map(({ parameters }) => parameters), + ); + assert.ok(locked.every(({ behavior }) => behavior === 'BLOCKING')); + assert.deepEqual(res.body.config.tools[0].functionDeclarations, locked); + assert.ok(!JSON.stringify(res.body).includes('fixture-key')); + }); +} + +test('Gemini key setup uses the existing server-only status and save contract', () => { + const entry = KEY_SETUP_KEYS.find(({ id }) => id === 'gemini'); + assert.deepEqual(entry.envVars, ['GEMINI_API_KEY']); + assert.ok(!entry.clientExposed); + assert.equal( + keySetupStatus({}).keys.find(({ id }) => id === 'gemini').set, + false, + ); + const env = { GEMINI_API_KEY: 'fixture-private-key' }; + const status = keySetupStatus(env); + assert.equal(status.keys.find(({ id }) => id === 'gemini').set, true); + assert.ok(!JSON.stringify(status).includes(env.GEMINI_API_KEY)); + assert.equal(validateKeySetupUpdates(env).ok, true); + assert.match( + upsertDotenvValues('GEMINI_API_KEY=old\n', env), + /^GEMINI_API_KEY=fixture-private-key\n$/, + ); + assert.equal( + validateKeySetupUpdates({ VITE_GEMINI_API_KEY: 'fixture-private-key' }).ok, + false, + ); +}); diff --git a/src/geminiServer.test.mjs b/src/geminiServer.test.mjs new file mode 100644 index 000000000..d17f2960d --- /dev/null +++ b/src/geminiServer.test.mjs @@ -0,0 +1,1022 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { createServer, preview } from 'vite'; +import { hostCheckPlugin, resolveAllowedHosts } from '../build/allowedHosts.js'; +import { standaloneVoiceTools } from '../server/standalone/voiceTools.js'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { EventEmitter, once } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { geminiLiveProxy } from '../server/providers/gemini.js'; +import { createGeminiTokenHandler } from '../server/providers/gemini/realtime.js'; +import { + GEMINI_TOKEN_ENDPOINT, + GEMINI_LIVE_WEBSOCKET_URL, + createGeminiLiveConfig, +} from '../server/providers/gemini/config.js'; +import { GEV_REALTIME_TOOLS } from '../server/providers/openai/tools.js'; +import { PROXY_SIGNALS } from './localRequestGate.mjs'; + +const API_KEY = 'private-gemini-test-key-never-return'; +const TOKEN = 'auth_tokens/short-lived-test'; +const ISSUED_AT = Date.parse('2026-09-29T12:00:00Z'); + +async function fixture(t, options = {}) { + const calls = []; + const handler = createGeminiTokenHandler({ + resolveApiKey: () => API_KEY, + resolveModel: () => 'gemini-3.8-live', + resolveHost: () => '', + resolveAllowedHosts: () => '', + resolveRateLimit: () => '0', + now: () => ISSUED_AT, + fetchImpl: async (...args) => { + calls.push(args); + return Response.json({ + name: TOKEN, + privateDebug: API_KEY, + upstreamUrl: GEMINI_TOKEN_ENDPOINT, + }); + }, + ...options, + }); + const server = http.createServer((req, res) => void handler(req, res)); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const origin = `http://127.0.0.1:${server.address().port}`; + t.after(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }); + const send = (overrides = {}) => { + const { headers, ...rest } = overrides; + return fetch(origin, { + method: 'POST', + body: '{}', + ...rest, + headers: { + Origin: origin, + 'Content-Type': 'application/json', + ...headers, + }, + }); + }; + const sendRaw = (overrides = {}) => + new Promise((resolve, reject) => { + const { headers, body = '{}', ...rest } = overrides; + const req = http.request( + origin, + { + method: 'POST', + ...rest, + headers: { + Origin: origin, + 'Content-Type': 'application/json', + ...headers, + }, + }, + (res) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => + resolve( + new Response(Buffer.concat(chunks), { + status: res.statusCode, + headers: res.headers, + }), + ), + ); + res.on('error', reject); + }, + ); + req.on('error', reject); + req.end(body); + }); + return { calls, origin, send, sendRaw }; +} + +test('dev and preview install the same bounded Gemini token endpoint', () => { + for (const hook of ['configureServer', 'configurePreviewServer']) { + const installed = []; + geminiLiveProxy()[hook]({ + middlewares: { use: (...args) => installed.push(args) }, + }); + assert.equal(installed.length, 1); + assert.equal(installed[0][0], '/api/gemini/token'); + assert.equal(typeof installed[0][1], 'function'); + } +}); + +test('mounted token handler completes synchronous resolver failures without exposing details', async (t) => { + for (const resolver of [ + 'resolveHost', + 'resolveAllowedHosts', + 'resolveApiKey', + 'resolveModel', + 'resolveRateLimit', + null, + ]) { + await t.test(resolver || 'healthy control', async () => { + const req = Object.assign(new PassThrough(), { + method: 'POST', + headers: { + host: '127.0.0.1', + origin: 'http://127.0.0.1', + 'content-type': 'application/json', + }, + socket: { remoteAddress: '127.0.0.1' }, + }); + const writes = []; + const headers = new Map(); + const res = Object.assign(new EventEmitter(), { + setHeader: (name, value) => headers.set(name.toLowerCase(), value), + end(body) { + this.writableEnded = true; + writes.push(JSON.parse(body)); + }, + }); + let calls = 0; + let handler; + geminiLiveProxy({ + realtime: { + resolveHost: () => '', + resolveAllowedHosts: () => '', + resolveApiKey: () => API_KEY, + resolveModel: () => 'gemini-3.8-live', + resolveRateLimit: () => '0', + fetchImpl: async () => { + calls++; + return Response.json({ name: TOKEN }); + }, + ...(resolver && { + [resolver]: () => { + throw new Error(`Synthetic resolver failure: ${API_KEY}`); + }, + }), + }, + }).configureServer({ + middlewares: { + use(_path, installed) { + handler = installed; + }, + }, + }); + const pending = handler(req, res); + req.end('{}'); + await assert.doesNotReject(pending); + assert.equal(writes.length, 1); + assert.equal(res.statusCode, resolver ? 503 : 200); + assert.equal(calls, resolver ? 0 : 1); + assert.equal(headers.get('cache-control'), 'no-store'); + if (resolver) { + assert.deepEqual(writes[0], { + error: 'Gemini voice could not connect. Try again.', + }); + assert.equal(headers.get('connection'), 'close'); + } else assert.equal(writes[0].token, TOKEN); + assert.equal(req.listenerCount('aborted'), 0); + assert.equal(req.listenerCount('data'), 0); + assert.equal(req.listenerCount('end'), 0); + assert.equal(res.listenerCount('close'), 0); + req.destroy(); + }); + } +}); + +test('mint locks canonical action schemas and server instructions with no permanent credential in browser data', async (t) => { + const f = await fixture(t, { + annotationGuidance: 'Keep existing annotations.', + }); + const response = await f.send(); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.equal(response.headers.get('x-content-type-options'), 'nosniff'); + const data = await response.json(); + assert.deepEqual(Object.keys(data).sort(), [ + 'config', + 'model', + 'token', + 'websocketUrl', + ]); + assert.equal(data.token, TOKEN); + assert.equal(data.model, 'gemini-3.8-live'); + assert.equal(data.websocketUrl, GEMINI_LIVE_WEBSOCKET_URL); + assert.equal(new URL(data.websocketUrl).search, ''); + assert.equal(JSON.stringify(data).includes(API_KEY), false); + assert.deepEqual(data.config.generationConfig, { + responseModalities: ['AUDIO'], + }); + assert.match( + data.config.systemInstruction.parts[0].text, + /Keep existing annotations\./, + ); + assert.deepEqual(data.config.inputAudioTranscription, {}); + assert.deepEqual(data.config.outputAudioTranscription, {}); + assert.equal( + data.config.realtimeInputConfig?.automaticActivityDetection?.disabled, + undefined, + ); + const functions = data.config.tools[0].functionDeclarations; + assert.deepEqual( + functions.map((tool) => tool.name), + GEV_REALTIME_TOOLS.map((tool) => tool.name), + ); + for (let i = 0; i < functions.length; i++) { + assert.equal(functions[i].behavior, 'BLOCKING'); + assert.deepEqual( + functions[i].parametersJsonSchema, + GEV_REALTIME_TOOLS[i].parameters, + ); + assert.equal(functions[i].parameters, undefined); + } + const [url, init] = f.calls[0]; + assert.equal(url, GEMINI_TOKEN_ENDPOINT); + assert.equal(init.redirect, 'error'); + assert.equal(init.headers['x-goog-api-key'], API_KEY); + const request = JSON.parse(init.body); + assert.equal(request.uses, 1); + assert.equal(Date.parse(request.expireTime) - ISSUED_AT, 30 * 60_000); + assert.equal(Date.parse(request.newSessionExpireTime) - ISSUED_AT, 60_000); + assert.equal(request.fieldMask, undefined); + assert.equal(request.liveConnectConstraints, undefined); + assert.deepEqual(request.bidiGenerateContentSetup, { + model: 'models/gemini-3.8-live', + ...data.config, + }); +}); + +test('missing key and malformed model fail without an upstream request', async (t) => { + const missing = await fixture(t, { resolveApiKey: () => '' }); + assert.equal((await missing.send()).status, 503); + assert.equal(missing.calls.length, 0); + for (const model of [ + '', + 'models/../../secrets', + 'https://attacker.example/model', + 'gemini-live?key=private', + 'x'.repeat(150), + ]) { + const f = await fixture(t, { resolveModel: () => model }); + const response = await f.send(); + assert.equal(response.status, 503); + assert.equal(f.calls.length, 0); + assert.equal((await response.text()).includes(model || API_KEY), false); + } + const valid = await fixture(t, { + resolveModel: () => + ' models/gemini-2.5-flash-native-audio-preview-12-2025 ', + }); + assert.equal( + (await (await valid.send()).json()).model, + 'gemini-2.5-flash-native-audio-preview-12-2025', + ); +}); + +test('only the PTT gesture selects manual activity in the complete locked setup', async (t) => { + const f = await fixture(t); + const ordinary = await (await f.send()).json(); + const response = await f.send({ + body: JSON.stringify({ inputMode: 'push-to-talk' }), + }); + assert.equal(response.status, 200); + const manual = await response.json(); + assert.deepEqual(manual.config.realtimeInputConfig, { + automaticActivityDetection: { disabled: true }, + }); + const { realtimeInputConfig, ...shared } = manual.config; + assert.equal(JSON.stringify(shared), JSON.stringify(ordinary.config)); + assert.equal( + JSON.stringify(createGeminiLiveConfig(undefined, 'open-mic')), + JSON.stringify(ordinary.config), + ); + assert.throws( + () => createGeminiLiveConfig(undefined, 'unknown'), + /input mode/, + ); + const locked = JSON.parse(f.calls[1][1].body); + assert.deepEqual(locked.bidiGenerateContentSetup, { + model: 'models/gemini-3.8-live', + ...manual.config, + }); + assert.equal(locked.fieldMask, undefined); + assert.equal(locked.uses, 1); + assert.equal(Date.parse(locked.newSessionExpireTime) - ISSUED_AT, 60_000); + assert.equal(Date.parse(locked.expireTime) - ISSUED_AT, 30 * 60_000); + assert.equal(JSON.stringify(manual).includes(API_KEY), false); +}); + +test('other methods, missing/cross origins, hostile hosts and cross-site metadata cannot mint', async (t) => { + const f = await fixture(t); + for (const method of ['GET', 'OPTIONS', 'PUT', 'DELETE']) { + const response = await f.send({ method, body: undefined }); + assert.equal(response.status, 405); + assert.equal(response.headers.get('allow'), 'POST'); + } + for (const headers of [ + { Origin: '' }, + { Origin: 'null' }, + { Origin: 'https://attacker.example' }, + { Origin: `${f.origin}/path` }, + { Origin: `${f.origin}?key=secret` }, + { Origin: f.origin.replace('http:', 'https:') }, + { Host: 'attacker.example', Origin: 'http://attacker.example' }, + { Host: '127.0.0.1@attacker.example', Origin: 'http://attacker.example' }, + { 'Sec-Fetch-Site': 'cross-site' }, + ]) { + assert.equal((await f.sendRaw({ headers })).status, 403); + } + assert.equal(f.calls.length, 0); + assert.equal((await f.send()).status, 200); +}); + +test('same-origin local and explicitly bound hosts work without trusting forwarded authority', async (t) => { + const f = await fixture(t, { resolveHost: () => 'demo.example' }); + assert.equal( + ( + await f.sendRaw({ + headers: { + Host: 'demo.example:4173', + Origin: 'http://demo.example:4173', + }, + }) + ).status, + 200, + ); + assert.equal( + ( + await f.sendRaw({ + headers: { Host: 'gev.local:4173', Origin: 'http://gev.local:4173' }, + }) + ).status, + 403, + ); + assert.equal( + ( + await f.send({ + headers: { + Origin: 'https://demo.example', + 'X-Forwarded-Host': 'demo.example', + 'X-Forwarded-Proto': 'https', + }, + }) + ).status, + 403, + ); +}); + +test('forwarded requests cannot mint or consume the real socket peer rate cap', async (t) => { + const f = await fixture(t, { resolveRateLimit: () => '1' }); + assert.equal( + (await f.send({ headers: { 'X-Forwarded-For': '10.1.1.1' } })).status, + 403, + ); + assert.equal( + (await f.send({ headers: { 'X-Forwarded-For': '10.1.1.2' } })).status, + 403, + ); + assert.equal((await f.send()).status, 200); + const response = await f.send(); + assert.equal(response.status, 429); + assert.equal(response.headers.get('retry-after'), '60'); + assert.equal(f.calls.length, 1); +}); + +test('all shared proxy signals and same-site cross-origin metadata fail before minting', async (t) => { + const f = await fixture(t, { resolveRateLimit: () => '1' }); + for (const header of PROXY_SIGNALS) { + assert.equal( + (await f.sendRaw({ headers: { [header]: 'untrusted' } })).status, + 403, + header, + ); + } + assert.equal( + (await f.sendRaw({ headers: { 'Sec-Fetch-Site': 'same-site' } })).status, + 403, + ); + assert.equal(f.calls.length, 0); + assert.equal( + (await f.sendRaw({ headers: { 'Sec-Fetch-Site': 'same-origin' } })).status, + 200, + ); + assert.equal((await f.send()).status, 429); + assert.equal(f.calls.length, 1); +}); + +test('LAN binding, exact configured hosts, IPs, and localhost subdomains follow the app host policy', async (t) => { + const f = await fixture(t, { + resolveHost: () => '0.0.0.0', + resolveAllowedHosts: () => 'gev.local, demo.example', + }); + for (const host of [ + 'gev.local:4173', + 'demo.example:4173', + '192.168.1.20:4173', + '[::1]:4173', + 'demo.localhost:4173', + ]) { + assert.equal( + (await f.sendRaw({ headers: { Host: host, Origin: `http://${host}` } })) + .status, + 200, + host, + ); + } + for (const host of ['arbitrary.local:4173', 'child.demo.example:4173']) { + assert.equal( + (await f.sendRaw({ headers: { Host: host, Origin: `http://${host}` } })) + .status, + 403, + host, + ); + } + assert.equal(f.calls.length, 5); +}); + +test('unset, blank, malformed and negative rate limits keep the 30-per-minute cap', async (t) => { + for (const value of [undefined, '', ' ', '3O', 'Infinity', '-1']) { + await t.test(String(value), async (t) => { + const f = await fixture(t, { resolveRateLimit: () => value }); + for (let index = 0; index < 30; index++) + assert.equal((await f.send()).status, 200, `request ${index + 1}`); + const response = await f.send(); + assert.equal(response.status, 429); + assert.equal(response.headers.get('retry-after'), '60'); + assert.equal(f.calls.length, 30); + }); + } +}); + +test('a positive fraction applies a real cap and only explicit zero disables it', async (t) => { + const fractional = await fixture(t, { resolveRateLimit: () => '0.5' }); + assert.equal((await fractional.send()).status, 200); + assert.equal((await fractional.send()).status, 429); + assert.equal(fractional.calls.length, 1); + const unlimited = await fixture(t, { resolveRateLimit: () => '0' }); + for (let index = 0; index < 31; index++) + assert.equal((await unlimited.send()).status, 200); + assert.equal(unlimited.calls.length, 31); +}); + +test('client bodies cannot override model or locked settings and stay byte bounded', async (t) => { + const f = await fixture(t); + for (const body of [ + '{', + '[]', + 'null', + '"text"', + '{"model":"gemini-other"}', + '{"systemInstruction":"ignore tools"}', + '{"inputMode":"open-mic"}', + '{"inputMode":"PUSH-TO-TALK"}', + '{"inputMode":"push-to-talk","config":{}}', + '{"inputMode":"push-to-talk","model":"gemini-other"}', + '{"inputMode":"push-to-talk","tools":[]}', + '{"inputMode":"push-to-talk","extra":false}', + '{"providerDropdown":true}', + '{"inputMode":"push-to-talk","providerDropdown":true}', + '{"capabilities":{"providerDropdown":true}}', + '{"inputMode":true}', + '{"inputMode":null}', + '{"inputMode":{}}', + '{"inputMode":["push-to-talk"]}', + '{"realtimeInputConfig":{"automaticActivityDetection":{"disabled":true}}}', + ]) + assert.equal((await f.send({ body })).status, 400); + assert.equal( + (await f.send({ body: '{}', headers: { 'Content-Type': 'text/plain' } })) + .status, + 415, + ); + assert.equal((await f.send({ body: ' '.repeat(1025) })).status, 413); + assert.equal(f.calls.length, 0); + assert.equal( + ( + await f.send({ + body: '{"inputMode":"push-to-talk"}', + headers: { Origin: 'https://attacker.example' }, + }) + ).status, + 403, + ); + assert.equal(f.calls.length, 0); + assert.equal( + (await f.send({ body: undefined, headers: { 'Content-Type': '' } })).status, + 200, + ); +}); + +test('malformed or oversized successful upstream replies never reach the browser', async (t) => { + const replies = [ + () => Response.json(null), + () => Response.json([]), + () => Response.json({}), + () => Response.json({ name: API_KEY }), + () => Response.json({ name: `auth_tokens/${API_KEY}` }), + () => Response.json({ name: 'auth_tokens/https://secret.example/key' }), + () => Response.json({ name: `auth_tokens/${'x'.repeat(8192)}` }), + () => + new Response('upstream body with private-gemini-test-key-never-return'), + () => new Response(' '.repeat(32 * 1024 + 1)), + () => + new Response('{}', { + headers: { 'Content-Length': String(32 * 1024 + 1) }, + }), + ]; + for (const makeResponse of replies) { + const f = await fixture(t, { fetchImpl: async () => makeResponse() }); + const response = await f.send(); + assert.equal(response.status, 502); + const body = await response.text(); + assert.equal(body.includes(API_KEY), false); + assert.equal(body.includes('upstream'), false); + assert.equal(body.includes('secret.example'), false); + } +}); + +test('upstream refusals and network exceptions are normalized and never log secrets', async (t) => { + const leaked = `${API_KEY} https://generativelanguage.googleapis.com/v1beta/auth_tokens?key=${API_KEY}`; + const logs = []; + const originalWarn = console.warn; + console.warn = (...args) => logs.push(args.join(' ')); + t.after(() => { + console.warn = originalWarn; + }); + for (const [status, expected] of [ + [400, 502], + [401, 503], + [403, 503], + [404, 502], + [429, 429], + [500, 502], + [302, 502], + ]) { + const f = await fixture(t, { + fetchImpl: async () => new Response(leaked, { status }), + }); + const response = await f.send(); + assert.equal(response.status, expected); + assert.equal((await response.text()).includes(API_KEY), false); + } + const failed = await fixture(t, { + fetchImpl: async () => { + throw new Error(leaked); + }, + }); + assert.equal((await failed.send()).status, 502); + const redirected = await fixture(t, { + fetchImpl: async () => { + const response = Response.json({ name: TOKEN }); + Object.defineProperty(response, 'url', { + value: 'https://attacker.example/token', + }); + return response; + }, + }); + assert.equal((await redirected.send()).status, 502); + assert.equal(logs.join(' ').includes(API_KEY), false); +}); + +test('a stalled mint is aborted at the deadline without forwarding error details', async (t) => { + let cancelled = false; + const f = await fixture(t, { + timeoutMs: 30, + fetchImpl: (_url, { signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + cancelled = true; + reject(signal.reason); + }, + { once: true }, + ); + }), + }); + const response = await f.send(); + assert.equal(response.status, 504); + assert.equal(cancelled, true); + assert.deepEqual(await response.json(), { + error: 'Gemini voice could not connect. Try again.', + }); +}); + +test('request disconnect cancels its pending mint and cannot publish a late token', async (t) => { + let started; + let cancelled; + let release; + const ready = new Promise((resolve) => { + started = resolve; + }); + const aborted = new Promise((resolve) => { + cancelled = resolve; + }); + const f = await fixture(t, { + fetchImpl: (_url, { signal }) => { + signal.addEventListener('abort', cancelled, { once: true }); + started(); + return new Promise((resolve) => { + release = resolve; + }); + }, + }); + const req = http.request(f.origin, { + method: 'POST', + headers: { Origin: f.origin }, + }); + req.on('error', () => {}); + req.end(); + await ready; + req.destroy(); + await aborted; + release(Response.json({ name: TOKEN })); + await new Promise((resolve) => setImmediate(resolve)); +}); + +test('disconnect at mint resolution settles without publishing and discards the response body', async () => { + for (const timing of ['before-resolution', 'after-resolution', 'connected']) { + const req = Object.assign(new PassThrough(), { + method: 'POST', + headers: { + host: '127.0.0.1', + origin: 'http://127.0.0.1', + 'content-type': 'application/json', + }, + socket: { remoteAddress: '127.0.0.1' }, + }); + const writes = []; + const res = Object.assign(new EventEmitter(), { + setHeader() {}, + end(body) { + this.writableEnded = true; + writes.push(body); + }, + }); + let started; + let release; + let cancelled = 0; + const ready = new Promise((resolve) => { + started = resolve; + }); + const handler = createGeminiTokenHandler({ + resolveApiKey: () => API_KEY, + resolveModel: () => 'gemini-3.8-live', + resolveHost: () => '', + resolveAllowedHosts: () => '', + resolveRateLimit: () => '0', + fetchImpl: () => + new Promise((resolve) => { + release = resolve; + started(); + }), + }); + const pending = handler(req, res); + req.end('{}'); + await ready; + const disconnect = () => { + req.aborted = true; + req.emit('aborted'); + }; + if (timing === 'before-resolution') { + disconnect(); + await pending; + } + release( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode(JSON.stringify({ name: TOKEN })), + ); + if (timing === 'connected') controller.close(); + }, + cancel() { + cancelled++; + }, + }), + ), + ); + if (timing === 'after-resolution') queueMicrotask(disconnect); + await pending; + await new Promise((resolve) => setImmediate(resolve)); + if (timing === 'connected') { + assert.equal(res.statusCode, 200); + assert.equal(writes.length, 1); + assert.equal(JSON.parse(writes[0]).token, TOKEN); + assert.equal(cancelled, 0); + } else { + assert.equal(writes.length, 0, timing); + assert.equal(cancelled, 1, timing); + } + assert.equal(req.listenerCount('aborted'), 0, timing); + assert.equal(res.listenerCount('close'), 0, timing); + } +}); + +test('timeout also bounds a stalled upstream response body', async (t) => { + let cancelled = false; + const f = await fixture(t, { + timeoutMs: 30, + fetchImpl: async () => + new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + ), + }); + assert.equal((await f.send()).status, 504); + assert.equal(cancelled, true); +}); + +test('chunked client uploads are capped without starting a mint', async (t) => { + const f = await fixture(t); + const response = await f.sendRaw({ + body: ' '.repeat(1025), + headers: { 'Transfer-Encoding': 'chunked' }, + }); + assert.equal(response.status, 413); + assert.equal(f.calls.length, 0); +}); + +test('a partial token request cannot hold the handler beyond its deadline', async (t) => { + const f = await fixture(t, { timeoutMs: 30 }); + const response = await new Promise((resolve, reject) => { + const req = http.request( + f.origin, + { + method: 'POST', + headers: { + Origin: f.origin, + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked', + }, + }, + resolve, + ); + req.on('error', reject); + t.after(() => req.destroy()); + req.write('{'); + }); + assert.equal(response.statusCode, 504); + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + await once(response, 'end'); + assert.deepEqual(JSON.parse(Buffer.concat(chunks)), { + error: 'Gemini voice could not connect. Try again.', + }); + assert.equal(f.calls.length, 0); +}); + +test( + 'deadline responds before an abort-ignoring mint settles and cancels its late body', + { timeout: 2000 }, + async (t) => { + let release; + let signal; + let cancelled = false; + const f = await fixture(t, { + timeoutMs: 30, + fetchImpl: (_url, options) => { + signal = options.signal; + return new Promise((resolve) => { + release = resolve; + }); + }, + }); + const response = await f.send(); + assert.equal(response.status, 504); + assert.equal(signal.aborted, true); + assert.deepEqual(await response.json(), { + error: 'Gemini voice could not connect. Try again.', + }); + release( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode(JSON.stringify({ name: TOKEN })), + ); + }, + cancel() { + cancelled = true; + }, + }), + ), + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(cancelled, true); + }, +); + +test('wrong-destination and redirected successful responses discard their bodies', async (t) => { + for (const property of ['url', 'redirected']) { + let cancelled = false; + const f = await fixture(t, { + fetchImpl: async () => { + const response = new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + ); + Object.defineProperty(response, property, { + value: property === 'url' ? 'https://attacker.example/token' : true, + }); + return response; + }, + }); + const response = await f.send(); + assert.equal(response.status, 502); + assert.deepEqual(await response.json(), { + error: 'Gemini voice could not connect. Try again.', + }); + assert.equal(cancelled, true); + } +}); + +test( + 'a chunked overflow delivers fixed 413 while its upload is still open', + { timeout: 2000 }, + async (t) => { + const f = await fixture(t); + const response = await new Promise((resolve, reject) => { + const req = http.request( + f.origin, + { + method: 'POST', + headers: { + Origin: f.origin, + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked', + }, + }, + resolve, + ); + req.on('error', reject); + t.after(() => req.destroy()); + req.write(' '.repeat(1024)); + req.write(' '); + }); + assert.equal(response.statusCode, 413); + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + await once(response, 'end'); + assert.deepEqual(JSON.parse(Buffer.concat(chunks)), { + error: 'Token request is too large', + }); + assert.equal(f.calls.length, 0); + assert.equal((await f.send({ body: ' '.repeat(1024) })).status, 200); + }, +); + +test('standalone mints lock the full voice query catalog in both credential and browser setup', async (t) => { + const tools = standaloneVoiceTools(); + const declarations = tools.map(({ name, description, parameters }) => ({ + name, + description, + parametersJsonSchema: parameters, + behavior: 'BLOCKING', + })); + for (const name of ['search_places', 'get_weather', 'get_wind', 'plan_route']) + assert.ok( + tools.some((tool) => tool.name === name), + name, + ); + assert.ok(tools.length > GEV_REALTIME_TOOLS.length); + for (const inputMode of ['open-mic', 'push-to-talk']) { + const f = await fixture(t, { tools }); + const response = await f.send({ + body: JSON.stringify(inputMode === 'open-mic' ? {} : { inputMode }), + }); + assert.equal(response.status, 200); + const data = await response.json(); + assert.deepEqual(data.config.tools, [ + { functionDeclarations: declarations }, + ]); + const request = JSON.parse(f.calls[0][1].body); + assert.deepEqual(request.bidiGenerateContentSetup, { + model: 'models/gemini-3.8-live', + ...data.config, + }); + assert.equal(request.fieldMask, undefined); + assert.equal(JSON.stringify(data).includes(API_KEY), false); + } +}); + +test('real Vite dev and preview reject hostile hosts before the Gemini middleware', async (t) => { + for (const mode of ['dev', 'preview']) { + await t.test(mode, async (t) => { + const root = await mkdtemp(path.join(tmpdir(), 'gev-gemini-vite-test-')); + await mkdir(path.join(root, 'dist')); + await writeFile(path.join(root, 'index.html'), '

isolated fixture

'); + await writeFile( + path.join(root, 'dist/index.html'), + '

isolated fixture

', + ); + let credentialReads = 0; + let upstreamCalls = 0; + const allowedHosts = resolveAllowedHosts('gev.local'); + const config = { + configFile: false, + envFile: false, + root, + logLevel: 'silent', + publicDir: false, + optimizeDeps: { noDiscovery: true, include: [] }, + // Deliberately put the provider first: enforce: pre must order the gate. + plugins: [ + geminiLiveProxy({ + realtime: { + resolveApiKey: () => { + credentialReads++; + return ''; + }, + resolveHost: () => '127.0.0.1', + resolveAllowedHosts: () => 'gev.local', + fetchImpl: async () => { + upstreamCalls++; + throw new Error('No upstream calls are permitted'); + }, + }, + }), + hostCheckPlugin(), + ], + server: { + host: '127.0.0.1', + port: 0, + allowedHosts, + hmr: false, + watch: null, + }, + preview: { host: '127.0.0.1', port: 0, allowedHosts }, + }; + let server; + try { + server = + mode === 'dev' ? await createServer(config) : await preview(config); + if (mode === 'dev') await server.listen(); + const origin = `http://127.0.0.1:${server.httpServer.address().port}`; + const request = (host, extraHeaders = {}) => + new Promise((resolve, reject) => { + const req = http.request( + `${origin}/api/gemini/token`, + { + method: 'POST', + headers: { + Host: host, + Origin: `http://${host}`, + ...extraHeaders, + }, + }, + (res) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => + resolve({ + status: res.statusCode, + body: Buffer.concat(chunks).toString(), + }), + ); + res.on('error', reject); + }, + ); + req.on('error', reject); + req.end(); + }); + for (const host of ['attacker.example:4173', 'arbitrary.local:4173']) { + const response = await request(host); + assert.equal(response.status, 403); + assert.match(response.body, /This host is not allowed/); + } + assert.equal(credentialReads, 0); + const sameSite = await request('gev.local:4173', { + 'Sec-Fetch-Site': 'same-site', + }); + assert.equal(sameSite.status, 403); + assert.equal(credentialReads, 0); + for (const host of [ + 'gev.local:4173', + '192.168.1.20:4173', + 'demo.localhost:4173', + ]) { + const response = await request(host); + assert.equal(response.status, 503); + assert.match(response.body, /Add a Gemini API key/); + } + assert.equal(credentialReads, 3); + assert.equal(upstreamCalls, 0); + } finally { + try { + if (server) await server.close(); + } finally { + await rm(root, { recursive: true, force: true }); + } + } + }); + } +}); diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index f93f42070..973239213 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -2890,5 +2890,56 @@ "USB work abandoned after a stalled transfer": "Trabajo USB cancelado tras una transferencia bloqueada", "SDR decoder did not start; receiver stopped": "El decodificador SDR no arrancΓ³; receptor detenido", "SDR decoder stopped responding; receiver stopped": "El decodificador SDR dejΓ³ de responder; receptor detenido", - "SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedΓ³ atrΓ‘s; receptor detenido" + "SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedΓ³ atrΓ‘s; receptor detenido", + "Gemini returned an oversized audio chunk": "Gemini devolviΓ³ un fragmento de audio demasiado grande", + "Gemini returned an unsupported audio format": "Gemini devolviΓ³ un formato de audio no compatible", + "Gemini returned an invalid audio sample rate": "Gemini devolviΓ³ una frecuencia de muestreo de audio no vΓ‘lida", + "Gemini returned incomplete PCM audio": "Gemini devolviΓ³ audio PCM incompleto", + "This browser does not support Gemini microphone audio": "Este navegador no admite audio del micrΓ³fono para Gemini", + "Microphone audio processing failed": "FallΓ³ el procesamiento del audio del micrΓ³fono", + "Gemini audio playback fell behind. Reconnect to continue.": "La reproducciΓ³n de audio de Gemini se ha retrasado. Vuelve a conectar para continuar.", + "Gemini token response did not include an ephemeral credential": "La respuesta de Gemini no incluyΓ³ una credencial temporal", + "Invalid Gemini connection endpoint": "El destino de conexiΓ³n de Gemini no es vΓ‘lido", + "Voice command context is too large": "El contexto del comando de voz es demasiado grande", + "Gemini reliable transport exceeded its queue limit. Start voice again.": "La cola de envΓ­o fiable de Gemini superΓ³ su lΓ­mite. Vuelve a iniciar la voz.", + "Gemini reliable transport did not drain. Start voice again.": "La cola de envΓ­o fiable de Gemini no se vaciΓ³. Vuelve a iniciar la voz.", + "Gemini token request failed (HTTP {status})": "FallΓ³ la solicitud de credencial de Gemini (HTTP {status})", + "Gemini token response did not include a valid model": "La respuesta de Gemini no incluyΓ³ un modelo vΓ‘lido", + "Gemini token response did not include session settings": "La respuesta de Gemini no incluyΓ³ la configuraciΓ³n de la sesiΓ³n", + "Gemini session settings do not match the input mode": "La configuraciΓ³n de la sesiΓ³n de Gemini no coincide con el modo de entrada", + "Gemini could not submit this input. Start voice again.": "Gemini no pudo enviar esta entrada. Vuelve a iniciar la voz.", + "Connecting to Gemini": "Conectando con Gemini", + "Gemini returned an oversized message": "Gemini devolviΓ³ un mensaje demasiado grande", + "Gemini Live connection failed. Check network access, then start voice again.": "FallΓ³ la conexiΓ³n con Gemini Live. Comprueba el acceso a la red y vuelve a iniciar la voz.", + "Gemini Live connection ended. Start voice again for a new conversation.": "La conexiΓ³n con Gemini Live terminΓ³. Vuelve a iniciar la voz para una nueva conversaciΓ³n.", + "Gemini Live could not start. Start voice again to request a fresh session.": "Gemini Live no pudo iniciarse. Vuelve a iniciar la voz para solicitar una nueva sesiΓ³n.", + "Start voice again for a fresh token and a new conversation.": "Vuelve a iniciar la voz para obtener una credencial nueva y comenzar otra conversaciΓ³n.", + "Start voice again to request a fresh session credential.": "Vuelve a iniciar la voz para solicitar una nueva credencial de sesiΓ³n.", + "Microphone permission was denied. Allow microphone access and try again.": "Se denegΓ³ el permiso del micrΓ³fono. Permite el acceso al micrΓ³fono e intΓ©ntalo de nuevo.", + "Gemini connection timed out. Try again.": "Se agotΓ³ el tiempo de conexiΓ³n con Gemini. IntΓ©ntalo de nuevo.", + "Gemini voice could not start": "No se pudo iniciar la voz de Gemini", + "Gemini did not finish the interrupted reply. Start voice again.": "Gemini no terminΓ³ la respuesta interrumpida. Vuelve a iniciar la voz.", + "Gemini session expiring; finishing the current reply": "La sesiΓ³n de Gemini va a caducar; terminando la respuesta actual", + "Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini rechazΓ³ la sesiΓ³n de voz. Vuelve a iniciar la voz para solicitar una nueva sesiΓ³n.", + "Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "Vuelve a iniciar la voz para obtener una nueva credencial de sesiΓ³n. Si se repite, comprueba el modelo de Gemini y el acceso a la API.", + "Gemini action history is full. Start voice again for a fresh session.": "El historial de acciones de Gemini estΓ‘ lleno. Vuelve a iniciar la voz para una nueva sesiΓ³n.", + "Gemini requested too many simultaneous actions": "Gemini solicitΓ³ demasiadas acciones simultΓ‘neas", + "Voice text command is too long": "El comando de voz escrito es demasiado largo", + "Gemini voice is not connected": "La voz de Gemini no estΓ‘ conectada", + "Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "La entrada interrumpida de Gemini superΓ³ el lΓ­mite de bΓΊfer o de espera. Vuelve a iniciar la voz.", + "Gemini Live connection is expiring. Start voice again for a new conversation.": "La conexiΓ³n con Gemini Live va a caducar. Vuelve a iniciar la voz para una nueva conversaciΓ³n.", + "Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "Las sesiones de voz de Gemini tienen un lΓ­mite de tiempo. Cuando caduque la sesiΓ³n, vuelve a iniciar la voz para una nueva conversaciΓ³n.", + "Gemini Live voice control": "Control por voz con Gemini Live", + "Voice provider": "Proveedor de voz", + "Changing provider reconnects voice if it is on.": "Cambiar de proveedor vuelve a conectar la voz si estΓ‘ activada.", + "Gemini voice could not connect. Try again.": "No se pudo conectar la voz de Gemini. IntΓ©ntalo de nuevo.", + "Gemini voice requires a same-origin app request": "La voz de Gemini requiere una solicitud del mismo origen que la aplicaciΓ³n", + "Add a Gemini API key in Provider Settings": "AΓ±ade una clave de API de Gemini en los ajustes de proveedores", + "The Gemini Live model configuration is invalid": "La configuraciΓ³n del modelo de Gemini Live no es vΓ‘lida", + "Gemini voice is starting too often. Try again shortly.": "La voz de Gemini se estΓ‘ iniciando con demasiada frecuencia. Vuelve a intentarlo en breve.", + "Gemini quota is unavailable. Try again later.": "La cuota de Gemini no estΓ‘ disponible. IntΓ©ntalo mΓ‘s tarde.", + "Gemini access was refused. Check the API key and model access.": "Se denegΓ³ el acceso a Gemini. Comprueba la clave de API y el acceso al modelo.", + "Gemini Live is unavailable for the configured model.": "Gemini Live no estΓ‘ disponible para el modelo configurado.", + "Token request is too large": "La solicitud de credencial es demasiado grande", + "Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacΓ­o" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index 6a58ab901..f3a1d9537 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -2957,5 +2957,56 @@ "USB work abandoned after a stalled transfer": "Π Π°Π±ΠΎΡ‚Π° с USB ΠΏΡ€Π΅Ρ€Π²Π°Π½Π°: ΠΏΠ΅Ρ€Π΅Π΄Π°Ρ‡Π° Π΄Π°Π½Π½Ρ‹Ρ… зависла", "SDR decoder did not start; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ запустился; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", "SDR decoder stopped responding; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR пСрСстал ΠΎΡ‚Π²Π΅Ρ‡Π°Ρ‚ΡŒ; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", - "SDR decoder fell behind; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ успСваСт; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн" + "SDR decoder fell behind; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ успСваСт; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", + "Gemini returned an oversized audio chunk": "Gemini Π²Π΅Ρ€Π½ΡƒΠ» слишком большой Ρ„Ρ€Π°Π³ΠΌΠ΅Π½Ρ‚ Π°ΡƒΠ΄ΠΈΠΎ", + "Gemini returned an unsupported audio format": "Gemini Π²Π΅Ρ€Π½ΡƒΠ» Π½Π΅ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΈΠ²Π°Π΅ΠΌΡ‹ΠΉ Ρ„ΠΎΡ€ΠΌΠ°Ρ‚ Π°ΡƒΠ΄ΠΈΠΎ", + "Gemini returned an invalid audio sample rate": "Gemini Π²Π΅Ρ€Π½ΡƒΠ» Π½Π΅Π΄ΠΎΠΏΡƒΡΡ‚ΠΈΠΌΡƒΡŽ частоту дискрСтизации Π°ΡƒΠ΄ΠΈΠΎ", + "Gemini returned incomplete PCM audio": "Gemini Π²Π΅Ρ€Π½ΡƒΠ» Π½Π΅ΠΏΠΎΠ»Π½Ρ‹Π΅ Π°ΡƒΠ΄ΠΈΠΎΠ΄Π°Π½Π½Ρ‹Π΅ PCM", + "This browser does not support Gemini microphone audio": "Π­Ρ‚ΠΎΡ‚ Π±Ρ€Π°ΡƒΠ·Π΅Ρ€ Π½Π΅ ΠΏΠΎΠ΄Π΄Π΅Ρ€ΠΆΠΈΠ²Π°Π΅Ρ‚ Π·Π²ΡƒΠΊ с ΠΌΠΈΠΊΡ€ΠΎΡ„ΠΎΠ½Π° для Gemini", + "Microphone audio processing failed": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ ΠΎΠ±Ρ€Π°Π±ΠΎΡ‚Π°Ρ‚ΡŒ Π·Π²ΡƒΠΊ с ΠΌΠΈΠΊΡ€ΠΎΡ„ΠΎΠ½Π°", + "Gemini audio playback fell behind. Reconnect to continue.": "ВоспроизвСдСниС Π·Π²ΡƒΠΊΠ° Gemini отстаёт. ΠŸΠΎΠ΄ΠΊΠ»ΡŽΡ‡ΠΈΡ‚Π΅ΡΡŒ Π·Π°Π½ΠΎΠ²ΠΎ, Ρ‡Ρ‚ΠΎΠ±Ρ‹ ΠΏΡ€ΠΎΠ΄ΠΎΠ»ΠΆΠΈΡ‚ΡŒ.", + "Gemini token response did not include an ephemeral credential": "Π’ ΠΎΡ‚Π²Π΅Ρ‚Π΅ Gemini Π½Π΅Ρ‚ Π²Ρ€Π΅ΠΌΠ΅Π½Π½Ρ‹Ρ… ΡƒΡ‡Ρ‘Ρ‚Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ…", + "Invalid Gemini connection endpoint": "НСдопустимый адрСс ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ ΠΊ Gemini", + "Voice command context is too large": "ΠšΠΎΠ½Ρ‚Π΅ΠΊΡΡ‚ голосовой ΠΊΠΎΠΌΠ°Π½Π΄Ρ‹ слишком Π²Π΅Π»ΠΈΠΊ", + "Gemini reliable transport exceeded its queue limit. Start voice again.": "ΠžΡ‡Π΅Ρ€Π΅Π΄ΡŒ Π½Π°Π΄Ρ‘ΠΆΠ½ΠΎΠΉ ΠΏΠ΅Ρ€Π΅Π΄Π°Ρ‡ΠΈ Gemini прСвысила Π»ΠΈΠΌΠΈΡ‚. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Gemini reliable transport did not drain. Start voice again.": "ΠžΡ‡Π΅Ρ€Π΅Π΄ΡŒ Π½Π°Π΄Ρ‘ΠΆΠ½ΠΎΠΉ ΠΏΠ΅Ρ€Π΅Π΄Π°Ρ‡ΠΈ Gemini Π½Π΅ освободилась. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Gemini token request failed (HTTP {status})": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ Π·Π°ΠΏΡ€ΠΎΡΠΈΡ‚ΡŒ Ρ‚ΠΎΠΊΠ΅Π½ Gemini (HTTP {status})", + "Gemini token response did not include a valid model": "Π’ ΠΎΡ‚Π²Π΅Ρ‚Π΅ Gemini Π½Π΅Ρ‚ допустимой ΠΌΠΎΠ΄Π΅Π»ΠΈ", + "Gemini token response did not include session settings": "Π’ ΠΎΡ‚Π²Π΅Ρ‚Π΅ Gemini Π½Π΅Ρ‚ настроСк сСанса", + "Gemini session settings do not match the input mode": "Настройки сСанса Gemini Π½Π΅ ΡΠΎΠΎΡ‚Π²Π΅Ρ‚ΡΡ‚Π²ΡƒΡŽΡ‚ Ρ€Π΅ΠΆΠΈΠΌΡƒ Π²Π²ΠΎΠ΄Π°", + "Gemini could not submit this input. Start voice again.": "Gemini Π½Π΅ ΡƒΠ΄Π°Π»ΠΎΡΡŒ ΠΎΡ‚ΠΏΡ€Π°Π²ΠΈΡ‚ΡŒ эти Π΄Π°Π½Π½Ρ‹Π΅. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Connecting to Gemini": "ΠŸΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΠ΅ ΠΊ Gemini", + "Gemini returned an oversized message": "Gemini Π²Π΅Ρ€Π½ΡƒΠ» слишком большоС сообщСниС", + "Gemini Live connection failed. Check network access, then start voice again.": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡ΠΈΡ‚ΡŒΡΡ ΠΊ Gemini Live. ΠŸΡ€ΠΎΠ²Π΅Ρ€ΡŒΡ‚Π΅ доступ ΠΊ сСти ΠΈ запуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Gemini Live connection ended. Start voice again for a new conversation.": "Π‘ΠΎΠ΅Π΄ΠΈΠ½Π΅Π½ΠΈΠ΅ с Gemini Live Π·Π°Π²Π΅Ρ€ΡˆΠ΅Π½ΠΎ. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ для Π½ΠΎΠ²ΠΎΠ³ΠΎ Ρ€Π°Π·Π³ΠΎΠ²ΠΎΡ€Π°.", + "Gemini Live could not start. Start voice again to request a fresh session.": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ Π·Π°ΠΏΡƒΡΡ‚ΠΈΡ‚ΡŒ Gemini Live. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ, Ρ‡Ρ‚ΠΎΠ±Ρ‹ Π·Π°ΠΏΡ€ΠΎΡΠΈΡ‚ΡŒ Π½ΠΎΠ²Ρ‹ΠΉ сСанс.", + "Start voice again for a fresh token and a new conversation.": "ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ, Ρ‡Ρ‚ΠΎΠ±Ρ‹ ΠΏΠΎΠ»ΡƒΡ‡ΠΈΡ‚ΡŒ Π½ΠΎΠ²Ρ‹ΠΉ Ρ‚ΠΎΠΊΠ΅Π½ ΠΈ Π½Π°Ρ‡Π°Ρ‚ΡŒ Π½ΠΎΠ²Ρ‹ΠΉ Ρ€Π°Π·Π³ΠΎΠ²ΠΎΡ€.", + "Start voice again to request a fresh session credential.": "ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ, Ρ‡Ρ‚ΠΎΠ±Ρ‹ Π·Π°ΠΏΡ€ΠΎΡΠΈΡ‚ΡŒ Π½ΠΎΠ²Ρ‹Π΅ ΡƒΡ‡Ρ‘Ρ‚Π½Ρ‹Π΅ Π΄Π°Π½Π½Ρ‹Π΅ сСанса.", + "Microphone permission was denied. Allow microphone access and try again.": "Доступ ΠΊ ΠΌΠΈΠΊΡ€ΠΎΡ„ΠΎΠ½Ρƒ Π·Π°ΠΏΡ€Π΅Ρ‰Ρ‘Π½. Π Π°Π·Ρ€Π΅ΡˆΠΈΡ‚Π΅ доступ ΠΈ ΠΏΠΎΠ²Ρ‚ΠΎΡ€ΠΈΡ‚Π΅ ΠΏΠΎΠΏΡ‹Ρ‚ΠΊΡƒ.", + "Gemini connection timed out. Try again.": "ВрСмя оТидания ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ ΠΊ Gemini истСкло. ΠŸΠΎΠ²Ρ‚ΠΎΡ€ΠΈΡ‚Π΅ ΠΏΠΎΠΏΡ‹Ρ‚ΠΊΡƒ.", + "Gemini voice could not start": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ Π·Π°ΠΏΡƒΡΡ‚ΠΈΡ‚ΡŒ голосовой Ρ€Π΅ΠΆΠΈΠΌ Gemini", + "Gemini did not finish the interrupted reply. Start voice again.": "Gemini Π½Π΅ Π·Π°Π²Π΅Ρ€ΡˆΠΈΠ» ΠΏΡ€Π΅Ρ€Π²Π°Π½Π½Ρ‹ΠΉ ΠΎΡ‚Π²Π΅Ρ‚. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Gemini session expiring; finishing the current reply": "БСанс Gemini истСкаСт; Π·Π°Π²Π΅Ρ€ΡˆΠ°Π΅Ρ‚ΡΡ Ρ‚Π΅ΠΊΡƒΡ‰ΠΈΠΉ ΠΎΡ‚Π²Π΅Ρ‚", + "Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini ΠΎΡ‚ΠΊΠ»ΠΎΠ½ΠΈΠ» голосовой сСанс. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ, Ρ‡Ρ‚ΠΎΠ±Ρ‹ Π·Π°ΠΏΡ€ΠΎΡΠΈΡ‚ΡŒ Π½ΠΎΠ²Ρ‹ΠΉ сСанс.", + "Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ для получСния Π½ΠΎΠ²Ρ‹Ρ… ΡƒΡ‡Ρ‘Ρ‚Π½Ρ‹Ρ… Π΄Π°Π½Π½Ρ‹Ρ…. Если ошибка повторится, ΠΏΡ€ΠΎΠ²Π΅Ρ€ΡŒΡ‚Π΅ модСль Gemini ΠΈ доступ ΠΊ API.", + "Gemini action history is full. Start voice again for a fresh session.": "Π˜ΡΡ‚ΠΎΡ€ΠΈΡ дСйствий Gemini Π·Π°ΠΏΠΎΠ»Π½Π΅Π½Π°. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ для Π½ΠΎΠ²ΠΎΠ³ΠΎ сСанса.", + "Gemini requested too many simultaneous actions": "Gemini запросил слишком ΠΌΠ½ΠΎΠ³ΠΎ ΠΎΠ΄Π½ΠΎΠ²Ρ€Π΅ΠΌΠ΅Π½Π½Ρ‹Ρ… дСйствий", + "Voice text command is too long": "ВСкст голосовой ΠΊΠΎΠΌΠ°Π½Π΄Ρ‹ слишком Π΄Π»ΠΈΠ½Π½Ρ‹ΠΉ", + "Gemini voice is not connected": "Голосовой Ρ€Π΅ΠΆΠΈΠΌ Gemini Π½Π΅ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½", + "Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "ΠŸΡ€Π΅Ρ€Π²Π°Π½Π½Ρ‹ΠΉ Π²Π²ΠΎΠ΄ Gemini прСвысил Π»ΠΈΠΌΠΈΡ‚ Π±ΡƒΡ„Π΅Ρ€Π° ΠΈΠ»ΠΈ оТидания. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ.", + "Gemini Live connection is expiring. Start voice again for a new conversation.": "Π‘ΠΎΠ΅Π΄ΠΈΠ½Π΅Π½ΠΈΠ΅ с Gemini Live истСкаСт. ЗапуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ для Π½ΠΎΠ²ΠΎΠ³ΠΎ Ρ€Π°Π·Π³ΠΎΠ²ΠΎΡ€Π°.", + "Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "ВрСмя голосовых сСансов Gemini ΠΎΠ³Ρ€Π°Π½ΠΈΡ‡Π΅Π½ΠΎ. Когда сСанс истСчёт, запуститС голосовой Ρ€Π΅ΠΆΠΈΠΌ Π·Π°Π½ΠΎΠ²ΠΎ для Π½ΠΎΠ²ΠΎΠ³ΠΎ Ρ€Π°Π·Π³ΠΎΠ²ΠΎΡ€Π°.", + "Gemini Live voice control": "ГолосовоС ΡƒΠΏΡ€Π°Π²Π»Π΅Π½ΠΈΠ΅ Ρ‡Π΅Ρ€Π΅Π· Gemini Live", + "Voice provider": "ΠŸΠΎΡΡ‚Π°Π²Ρ‰ΠΈΠΊ голосового сСрвиса", + "Changing provider reconnects voice if it is on.": "Π‘ΠΌΠ΅Π½Π° поставщика ΠΏΠ΅Ρ€Π΅ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π°Π΅Ρ‚ голосовой Ρ€Π΅ΠΆΠΈΠΌ, Ссли ΠΎΠ½ Π²ΠΊΠ»ΡŽΡ‡Ρ‘Π½.", + "Gemini voice could not connect. Try again.": "НС ΡƒΠ΄Π°Π»ΠΎΡΡŒ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡ΠΈΡ‚ΡŒ голосовой Ρ€Π΅ΠΆΠΈΠΌ Gemini. ΠŸΠΎΠ²Ρ‚ΠΎΡ€ΠΈΡ‚Π΅ ΠΏΠΎΠΏΡ‹Ρ‚ΠΊΡƒ.", + "Gemini voice requires a same-origin app request": "Голосовой Ρ€Π΅ΠΆΠΈΠΌ Gemini Ρ‚Ρ€Π΅Π±ΡƒΠ΅Ρ‚ запрос ΠΈΠ· Ρ‚ΠΎΠ³ΠΎ ΠΆΠ΅ источника, Ρ‡Ρ‚ΠΎ ΠΈ ΠΏΡ€ΠΈΠ»ΠΎΠΆΠ΅Π½ΠΈΠ΅", + "Add a Gemini API key in Provider Settings": "Π”ΠΎΠ±Π°Π²ΡŒΡ‚Π΅ ΠΊΠ»ΡŽΡ‡ API Gemini Π² настройках поставщиков", + "The Gemini Live model configuration is invalid": "НСдопустимая конфигурация ΠΌΠΎΠ΄Π΅Π»ΠΈ Gemini Live", + "Gemini voice is starting too often. Try again shortly.": "Голосовой Ρ€Π΅ΠΆΠΈΠΌ Gemini запускаСтся слишком часто. ΠŸΠΎΠ²Ρ‚ΠΎΡ€ΠΈΡ‚Π΅ ΠΏΠΎΠΏΡ‹Ρ‚ΠΊΡƒ Ρ‡ΡƒΡ‚ΡŒ ΠΏΠΎΠ·ΠΆΠ΅.", + "Gemini quota is unavailable. Try again later.": "ΠšΠ²ΠΎΡ‚Π° Gemini нСдоступна. ΠŸΠΎΠ²Ρ‚ΠΎΡ€ΠΈΡ‚Π΅ ΠΏΠΎΠΏΡ‹Ρ‚ΠΊΡƒ ΠΏΠΎΠ·ΠΆΠ΅.", + "Gemini access was refused. Check the API key and model access.": "Доступ ΠΊ Gemini Π·Π°ΠΏΡ€Π΅Ρ‰Ρ‘Π½. ΠŸΡ€ΠΎΠ²Π΅Ρ€ΡŒΡ‚Π΅ ΠΊΠ»ΡŽΡ‡ API ΠΈ доступ ΠΊ ΠΌΠΎΠ΄Π΅Π»ΠΈ.", + "Gemini Live is unavailable for the configured model.": "Gemini Live нСдоступСн для Π²Ρ‹Π±Ρ€Π°Π½Π½ΠΎΠΉ ΠΌΠΎΠ΄Π΅Π»ΠΈ.", + "Token request is too large": "Запрос Ρ‚ΠΎΠΊΠ΅Π½Π° слишком Π²Π΅Π»ΠΈΠΊ", + "Use an empty JSON token request": "Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΡƒΠΉΡ‚Π΅ запрос Ρ‚ΠΎΠΊΠ΅Π½Π° с пустым ΠΎΠ±ΡŠΠ΅ΠΊΡ‚ΠΎΠΌ JSON" } diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index 10bbe32a5..d8ccc048d 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -19,7 +19,7 @@ import { hasProxySignals } from './localRequestGate.mjs'; /** Longest accepted key/token value. Real provider keys are all far shorter. */ export const KEY_SETUP_VALUE_LIMIT = 512; -/** Most env vars accepted in one save. The registry defines ten. */ +/** Most env vars accepted in one save. */ export const KEY_SETUP_UPDATE_LIMIT = 16; /** Header line written above keys the panel appends to a .env file. */ @@ -65,6 +65,14 @@ export const KEY_SETUP_KEYS = Object.freeze([ envVars: Object.freeze(['OPENAI_API_KEY']), tier: 'metered', }), + Object.freeze({ + id: 'gemini', + title: 'GEMINI', // i18n-ignore: provider product name + unlocks: msg('Gemini Live voice control'), + getUrl: 'https://aistudio.google.com/apikey', + envVars: Object.freeze(['GEMINI_API_KEY']), + tier: 'metered', + }), Object.freeze({ id: 'aisstream', title: 'AISSTREAM', // i18n-ignore: provider product name diff --git a/src/layers/cctv/frames.js b/src/layers/cctv/frames.js index d137b74b2..807770c10 100644 --- a/src/layers/cctv/frames.js +++ b/src/layers/cctv/frames.js @@ -3,6 +3,7 @@ import { FRAME_SIGNATURE_H, PROJECTION_CANVAS_WIDTH, PROJECTION_CANVAS_HEIGHT, + PROJECTION_MATTE_COLOR, PROJECTION_TEXTURE_SWAP_MS, ACTIVE_FRAME_REFRESH_MS, FRAME_ENDPOINT, @@ -12,6 +13,37 @@ import { PLACEHOLDER_REPAINT_MS, } from './policy.js'; +/** + * Contain-fits a source frame into the projection canvas: scaled to the + * largest size that fits, centered, with the margins left for the matte fill. + * Falls back to the full canvas when either dimension is missing or zero. + * + * @param {number} sourceW - Source width in pixels. + * @param {number} sourceH - Source height in pixels. + * @returns {{x:number,y:number,w:number,h:number}} Draw rectangle in canvas pixels. + */ +export function fitToProjection(sourceW, sourceH) { + if (!sourceW || !sourceH) + return { + x: 0, + y: 0, + w: PROJECTION_CANVAS_WIDTH, + h: PROJECTION_CANVAS_HEIGHT, + }; + const scale = Math.min( + PROJECTION_CANVAS_WIDTH / sourceW, + PROJECTION_CANVAS_HEIGHT / sourceH, + ); + const w = Math.round(sourceW * scale); + const h = Math.round(sourceH * scale); + return { + x: Math.round((PROJECTION_CANVAS_WIDTH - w) / 2), + y: Math.round((PROJECTION_CANVAS_HEIGHT - h) / 2), + w, + h, + }; +} + export function createFrames({ state: layerState, services, parts, source }) { /** * FNV-1a over the RGB channels of a downsampled frame. Pure (takes the raw @@ -182,7 +214,7 @@ export function createFrames({ state: layerState, services, parts, source }) { ctx.clearRect(0, 0, w, h); const g = ctx.createLinearGradient(0, 0, w, h); g.addColorStop(0, '#05111a'); - g.addColorStop(1, '#01070c'); + g.addColorStop(1, PROJECTION_MATTE_COLOR); ctx.fillStyle = g; ctx.fillRect(0, 0, w, h); @@ -320,7 +352,14 @@ export function createFrames({ state: layerState, services, parts, source }) { // base color (planeMaterial color = WHITE, alpha .95) for the frame or // two Cesium needs to rebind, which IS the periodic white flash from the // owner field tests (2026-07-04 and 2026-07-30). - const signature = projectionFrameSignature(runtime); + const fit = fitToProjection( + runtime.image.naturalWidth, + runtime.image.naturalHeight, + ); + // Pixels are hashed at a fixed size, so the fit carries the aspect. + const pixels = projectionFrameSignature(runtime); + const signature = + pixels === null ? null : `${pixels}:${fit.w}x${fit.h}`; runtime.drawnImageStamp = runtime.imageStamp; if (signature !== null && signature === runtime.lastFrameSignature) { // Identical pixels β€” leave the canvas, and therefore the bound @@ -328,19 +367,14 @@ export function createFrames({ state: layerState, services, parts, source }) { return; } runtime.lastFrameSignature = signature; - runtime.ctx.clearRect( - 0, - 0, - PROJECTION_CANVAS_WIDTH, - PROJECTION_CANVAS_HEIGHT, - ); - runtime.ctx.drawImage( - runtime.image, + runtime.ctx.fillStyle = PROJECTION_MATTE_COLOR; + runtime.ctx.fillRect( 0, 0, PROJECTION_CANVAS_WIDTH, PROJECTION_CANVAS_HEIGHT, ); + runtime.ctx.drawImage(runtime.image, fit.x, fit.y, fit.w, fit.h); runtime.canvasStamp = (runtime.canvasStamp || 0) + 1; runtime.lastPlaceholderPaintAt = 0; } diff --git a/src/layers/cctv/frames.test.mjs b/src/layers/cctv/frames.test.mjs new file mode 100644 index 000000000..0d03e9457 --- /dev/null +++ b/src/layers/cctv/frames.test.mjs @@ -0,0 +1,141 @@ +// src/layers/cctv/frames.test.mjs +// Still-frame fit (#1201): stills draw at their source aspect, centered on the +// 1920x1080 projection canvas, instead of being stretched to fill it. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createFrames, fitToProjection } from './frames.js'; +import { + FRAME_SIGNATURE_W, + FRAME_SIGNATURE_H, + PROJECTION_CANVAS_WIDTH, + PROJECTION_CANVAS_HEIGHT, + PROJECTION_MATTE_COLOR, +} from './policy.js'; + +// The expected rectangles below are written for a 1920x1080 canvas. +assert.equal(PROJECTION_CANVAS_WIDTH, 1920); +assert.equal(PROJECTION_CANVAS_HEIGHT, 1080); + +const FULL_CANVAS = { + x: 0, + y: 0, + w: PROJECTION_CANVAS_WIDTH, + h: PROJECTION_CANVAS_HEIGHT, +}; + +test('a 4:3 still is pillarboxed at full height', () => { + assert.deepEqual(fitToProjection(640, 480), { + x: 240, + y: 0, + w: 1440, + h: 1080, + }); +}); + +test('a source wider than 16:9 is letterboxed at full width', () => { + assert.deepEqual(fitToProjection(3840, 1080), { + x: 0, + y: 270, + w: 1920, + h: 540, + }); +}); + +test('a 16:9 still fills the canvas', () => { + assert.deepEqual(fitToProjection(1920, 1080), FULL_CANVAS); +}); + +test('a missing or zero dimension falls back to the full canvas', () => { + assert.deepEqual(fitToProjection(0, 480), FULL_CANVAS); + assert.deepEqual(fitToProjection(640, 0), FULL_CANVAS); + assert.deepEqual(fitToProjection(640, undefined), FULL_CANVAS); + assert.deepEqual(fitToProjection(undefined, undefined), FULL_CANVAS); +}); + +// Draw path: the fit reaches drawImage, and the unchanged-frame skip keys on the +// fitted size as well as the pixels (the 64x36 hash alone drops aspect). +function stillHarness() { + const calls = []; + const ctx = { + fillStyle: null, + fillRect: (...rect) => calls.push(['fillRect', ctx.fillStyle, ...rect]), + clearRect: () => {}, + drawImage: (_image, ...rect) => calls.push(['drawImage', ...rect]), + }; + // Every frame downsamples to the same pixels, so only the size differs. + const pixels = new Uint8ClampedArray( + FRAME_SIGNATURE_W * FRAME_SIGNATURE_H * 4, + ).fill(7); + const runtime = { + mode: 'image', + ctx, + image: { naturalWidth: 0, naturalHeight: 0 }, + imageReady: true, + // In flight, so refreshProjectionImage returns before touching image.src. + imageLoading: true, + imageStamp: 0, + drawnImageStamp: -1, + lastFrameSignature: null, + canvasStamp: 0, + signatureCtx: { + clearRect: () => {}, + drawImage: () => {}, + getImageData: () => ({ data: pixels }), + }, + }; + const record = { camera: { id: 'still' }, projection: runtime }; + const { drawProjectionFrame } = createFrames({ + state: { _healthById: new Map(), _activeCameraId: null }, + services: {}, + parts: {}, + source: {}, + }); + const load = (width, height) => { + runtime.image = { naturalWidth: width, naturalHeight: height }; + runtime.imageStamp += 1; + calls.length = 0; + drawProjectionFrame(record); + return calls.slice(); + }; + return { runtime, load }; +} + +test('a still is drawn into its fitted rectangle over a full matte', () => { + const { runtime, load } = stillHarness(); + assert.deepEqual(load(640, 480), [ + [ + 'fillRect', + PROJECTION_MATTE_COLOR, + 0, + 0, + PROJECTION_CANVAS_WIDTH, + PROJECTION_CANVAS_HEIGHT, + ], + ['drawImage', 240, 0, 1440, 1080], + ]); + assert.equal(runtime.canvasStamp, 1); +}); + +test('same pixels at the same size skip the redraw', () => { + const { runtime, load } = stillHarness(); + load(640, 480); + assert.deepEqual(load(640, 480), []); + assert.equal(runtime.canvasStamp, 1); +}); + +test('same pixels at a new aspect redraw at the new fit', () => { + const { runtime, load } = stillHarness(); + load(640, 480); + assert.deepEqual(load(1920, 1080), [ + [ + 'fillRect', + PROJECTION_MATTE_COLOR, + 0, + 0, + PROJECTION_CANVAS_WIDTH, + PROJECTION_CANVAS_HEIGHT, + ], + ['drawImage', 0, 0, 1920, 1080], + ]); + assert.equal(runtime.canvasStamp, 2); +}); diff --git a/src/layers/cctv/policy.js b/src/layers/cctv/policy.js index cf09ebe64..96d7985f1 100644 --- a/src/layers/cctv/policy.js +++ b/src/layers/cctv/policy.js @@ -35,6 +35,9 @@ export const PROJECTION_CANVAS_WIDTH = 1920; export const PROJECTION_CANVAS_HEIGHT = 1080; +/** Matte behind fitted stills; the placeholder gradient's dark end. */ +export const PROJECTION_MATTE_COLOR = '#01070c'; + // Downsample grid for the unchanged-frame signature (drawProjectionFrame). // 64x36 keeps the 16:9 aspect and reads ~9 KB per check versus the 8.3 MB a // full-resolution compare would touch. @@ -410,6 +413,9 @@ export const IDLE_CAMERA_COLOR = export const ACTIVE_CAMERA_COLOR = Cesium.Color.fromCssColorString('#ffd97a').withAlpha(0.95); +export const LIVE_CAMERA_COLOR = + Cesium.Color.fromCssColorString('#b48cff').withAlpha(0.88); + export const IDLE_COVERAGE_COLOR = Cesium.Color.fromCssColorString('#2fe0ff').withAlpha(0.24); diff --git a/src/layers/cctv/rendering.js b/src/layers/cctv/rendering.js index 6601d938b..d10368d40 100644 --- a/src/layers/cctv/rendering.js +++ b/src/layers/cctv/rendering.js @@ -4,6 +4,7 @@ import { isHeadingEstimated } from './headingConfidence.js'; import { ACTIVE_CAMERA_COLOR, IDLE_CAMERA_COLOR, + LIVE_CAMERA_COLOR, ACTIVE_COVERAGE_CENTER, IDLE_COVERAGE_CENTER_MUTED, ACTIVE_COVERAGE_CENTER_DEPTHFAIL, @@ -18,6 +19,15 @@ export function createRendering({ parts, source, }) { + /** Icon color: active wins, then live-capable video feeds, then idle stills. */ + function cameraColorFor(record) { + if (record.camera.id === layerState._activeCameraId) + return ACTIVE_CAMERA_COLOR; + if (parts.model.isVideoFeedType(record.camera.feedType)) + return LIVE_CAMERA_COLOR; + return IDLE_CAMERA_COLOR; + } + const { focusNowMs, getFocusTarget, @@ -58,10 +68,7 @@ export function createRendering({ ), cameraDistanceFor: (position) => Cesium.Cartesian3.distance(camera.positionWC, position), - baseColorFor: (record) => - record.camera.id === layerState._activeCameraId - ? ACTIVE_CAMERA_COLOR - : IDLE_CAMERA_COLOR, + baseColorFor: cameraColorFor, }); layerState._activeFocusStyleCount = result.activeCount; } @@ -208,9 +215,7 @@ export function createRendering({ for (const record of layerState._records) { const isActive = record.camera.id === activeId; if (record.billboard) { - record.billboard.color = isActive - ? ACTIVE_CAMERA_COLOR - : IDLE_CAMERA_COLOR; + record.billboard.color = cameraColorFor(record); record.billboard.scale = isActive ? 1.25 : 1.0; // disableDepthTestDistance stays POSITIVE_INFINITY for every billboard // (set at creation) β€” see the field-test far-zoom submerge fix there. diff --git a/src/layers/cctv/rendering.test.mjs b/src/layers/cctv/rendering.test.mjs new file mode 100644 index 000000000..c4b490dd2 --- /dev/null +++ b/src/layers/cctv/rendering.test.mjs @@ -0,0 +1,73 @@ +// src/layers/cctv/rendering.test.mjs +// Icon colors under focus de-emphasis (#1200): the focus pass fades each icon +// from the same base color the coverage refresh assigns, so a video-feed camera +// stays violet while it dims instead of falling back to the idle cyan. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRendering } from './rendering.js'; +import { + ACTIVE_CAMERA_COLOR, + IDLE_CAMERA_COLOR, + LIVE_CAMERA_COLOR, +} from './policy.js'; +import { isVideoFeedType } from '../../sources/cctvTypes.js'; + +const FOCUS_FACTOR = 0.5; + +function assertFadedFrom(actual, base, label) { + assert.ok(actual, `${label}: color written`); + assert.deepEqual( + [actual.red, actual.green, actual.blue], + [base.red, base.green, base.blue], + `${label}: hue`, + ); + assert.equal(actual.alpha, base.alpha * FOCUS_FACTOR, `${label}: alpha`); +} + +test('focus de-emphasis fades each icon from its own color', () => { + const record = (id, feedType) => ({ + camera: { id, feedType }, + // No position, so the pass skips screen projection; the focus stub below + // decides the factor on its own. + billboard: { show: true }, + }); + const idleVideo = record('idle-hls', 'hls'); + const idleStill = record('idle-image', 'image'); + const activeVideo = record('active-hls', 'hls'); + const state = { + _enabled: true, + _viewer: { scene: {}, camera: {} }, + _records: [idleVideo, idleStill, activeVideo], + _activeCameraId: activeVideo.camera.id, + _activeFocusStyleCount: 0, + _lastFocusStyleAt: -Infinity, + }; + const focus = { + focusNowMs: (nowMs) => nowMs, + getFocusTarget: () => ({}), + focusPassIsNeeded: () => true, + advanceSpriteFocus: () => ({ + factor: FOCUS_FACTOR, + active: true, + transitioning: false, + }), + focusAlphaNeedsWrite: () => true, + }; + const { refreshCctvFocusStyles } = createRendering({ + state, + services: { focus }, + parts: { model: { isVideoFeedType } }, + source: {}, + }); + + refreshCctvFocusStyles(1000); + + assertFadedFrom(idleVideo.billboard.color, LIVE_CAMERA_COLOR, 'idle video'); + assertFadedFrom(idleStill.billboard.color, IDLE_CAMERA_COLOR, 'idle still'); + assertFadedFrom( + activeVideo.billboard.color, + ACTIVE_CAMERA_COLOR, + 'active video', + ); + assert.equal(state._activeFocusStyleCount, 3); +}); diff --git a/src/navigationPolicyFrame.test.mjs b/src/navigationPolicyFrame.test.mjs index a861870cf..5110232f1 100644 --- a/src/navigationPolicyFrame.test.mjs +++ b/src/navigationPolicyFrame.test.mjs @@ -6,6 +6,7 @@ import { readdirSync, readFileSync } from 'node:fs'; import { join, relative } from 'node:path'; import test from 'node:test'; import { fileURLToPath } from 'node:url'; +import { TRACKABLE_FAMILIES } from './agent/worldPacket.js'; import { setLocale } from './i18n/index.js'; import { COCKPIT_REFUSAL_NOUNS, @@ -68,10 +69,8 @@ const NOUN_LITERAL = /\bnoun\s*=\s*'([^']+)'|\bconst noun\s*=\s*[^?;]*\?\s*'([^']+)'\s*:\s*'([^']+)'/g; function trackableKinds() { - const source = readFileSync(join(SRC, 'voice/gevActions.js'), 'utf8'); - const block = source.match(/const TRACKABLE_FAMILIES = \[([\s\S]*?)\];/); - assert.ok(block, 'TRACKABLE_FAMILIES is still in gevActions.js'); - return [...block[1].matchAll(/kind:\s*'([^']+)'/g)].map((m) => m[1]); + // This is shared with voice; test the exported owner, not its file position. + return TRACKABLE_FAMILIES.map(({ kind }) => kind); } // Non-literal arguments: what each one can be. A wrapper's own `noun` (or all diff --git a/src/standalone/tools.js b/src/standalone/tools.js index fb127d598..eec4d0796 100644 --- a/src/standalone/tools.js +++ b/src/standalone/tools.js @@ -17,6 +17,10 @@ export function createStandaloneTools(options) { }, }, ...options, - voice: { toolCatalog: loadToolCatalog, ...options?.voice }, + voice: { + supportedProviders: ['openai', 'gemini'], + toolCatalog: loadToolCatalog, + ...options?.voice, + }, }); } diff --git a/src/testSupport/geminiIntegrationFixture.mjs b/src/testSupport/geminiIntegrationFixture.mjs new file mode 100644 index 000000000..ad84dcce6 --- /dev/null +++ b/src/testSupport/geminiIntegrationFixture.mjs @@ -0,0 +1,364 @@ +import assert from 'node:assert/strict'; +import { createVoiceSession } from '../voice/session.js'; +import { createGevActionRunner } from '../voice/gevActions.js'; +import { createGeminiSession } from '../voice/geminiSession.js'; +import { createNarrationScheduler } from '../voice/narration.js'; + +export const turnOfEventLoop = () => + new Promise((resolve) => setImmediate(resolve)); + +function deferred() { + let resolve; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +function ownedClock() { + let now = 0; + let nextId = 0; + const timers = new Map(); + return { + timers, + now: () => now, + setTimer(fn, delay = 0) { + const id = ++nextId; + timers.set(id, { fn, at: now + Math.max(0, delay) }); + return id; + }, + clearTimer: (id) => timers.delete(id), + advance(ms) { + const until = now + ms; + let callbacks = 0; + for (;;) { + const due = [...timers] + .filter(([, timer]) => timer.at <= until) + .sort((a, b) => a[1].at - b[1].at || a[0] - b[0])[0]; + if (!due) break; + assert.ok( + ++callbacks <= 1000, + 'fixture timer queue must remain bounded', + ); + timers.delete(due[0]); + now = due[1].at; + due[1].fn(); + } + now = until; + }, + }; +} + +/** + * Real common session -> real Gemini adapter -> real action runner. + * Only devices, transport, passive UI and wall-clock facilities are fakes. + * The optional result gate delays return AFTER the actual action commits. + */ +export function nativeIntegrationFixture(t) { + const clock = ownedClock(); + const events = []; + const sockets = []; + const tracks = []; + const audios = []; + const requests = []; + const runnerCalls = []; + const runnerJobs = []; + const resultGates = new Map(); + const acquisitions = { microphone: 0, audio: 0, token: 0, socket: 0 }; + const state = { height: 1000, renders: 0, zooms: [], earconStarts: 0 }; + const position = { x: 6379137, y: 0, z: 0 }; + const viewer = { + clock: { onTick: { addEventListener: () => () => {} } }, + camera: { + positionWC: position, + positionCartographic: { height: state.height }, + moveEnd: { addEventListener() {} }, + cancelFlight() {}, + zoomIn(metres) { + position.x -= metres; + state.height -= metres; + this.positionCartographic.height = state.height; + state.zooms.push({ direction: 'in', metres }); + }, + zoomOut(metres) { + position.x += metres; + state.height += metres; + this.positionCartographic.height = state.height; + state.zooms.push({ direction: 'out', metres }); + }, + }, + scene: { + canvas: { + clientWidth: 800, + clientHeight: 600, + addEventListener() {}, + removeEventListener() {}, + }, + requestRender() { + state.renders++; + }, + }, + }; + const dataManager = { layers: new Map() }; + const actualRunner = createGevActionRunner({ + viewer, + dataManager, + styleManager: {}, + }); + const runner = (name, args, options) => { + runnerCalls.push({ name, args, options }); + const job = (async () => { + // No fake action result: the camera and envelope come from production. + const result = await actualRunner(name, args, options); + const gate = resultGates.get(options.callId); + if (gate) { + gate.entered.resolve(result); + await gate.release.promise; + } + return result; + })(); + runnerJobs.push(job); + return job; + }; + class Socket { + constructor(url) { + acquisitions.socket++; + this.url = url; + this.readyState = 0; + this.bufferedAmount = 0; + this.sent = []; + sockets.push(this); + queueMicrotask(() => { + if (this.closed) return; + this.readyState = 1; + this.onopen?.(); + }); + } + send(raw) { + assert.equal( + this.readyState, + 1, + 'sending requires the owned open socket', + ); + const message = JSON.parse(raw); + this.sent.push(message); + if (message.setup) + queueMicrotask(() => this.message({ setupComplete: {} })); + } + message(message) { + this.onmessage?.({ data: JSON.stringify(message) }); + } + close() { + this.closed = true; + this.readyState = 3; + } + } + const session = createVoiceSession({ + runner, + createAdapter: (hooks) => + createGeminiSession({ + ...hooks, + ui: { root: { remove() {} } }, + dataManager, + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + WebSocketClass: Socket, + getUserMedia: async () => { + acquisitions.microphone++; + const track = { + enabled: true, + readyState: 'live', + stopCalls: 0, + stop() { + this.stopCalls++; + this.readyState = 'ended'; + }, + }; + tracks.push(track); + return { getTracks: () => [track], getAudioTracks: () => [track] }; + }, + fetchImpl: async (url, init) => { + acquisitions.token++; + requests.push({ url, init }); + const manual = JSON.parse(init.body).inputMode === 'push-to-talk'; + return Response.json({ + token: 'auth_tokens/fixture-not-a-credential', + model: 'gemini-3.8-live', + websocketUrl: + 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained', + config: { + generationConfig: { responseModalities: ['AUDIO'] }, + ...(manual + ? { + realtimeInputConfig: { + automaticActivityDetection: { disabled: true }, + }, + } + : {}), + }, + }); + }, + createInput: ({ readStream, operations }) => ({ + ...operations, + setMicrophoneEnabled(enabled) { + for (const track of readStream()?.getAudioTracks() || []) + track.enabled = enabled; + }, + updateVoiceButtonLabel() {}, + setVoiceSpeaker() {}, + startVoiceVisualizer() {}, + startAssistantVoiceVisualizer() {}, + cancelPushToTalkHold() {}, + stopVoiceVisualizer() {}, + detachBindings() {}, + bindPushToTalkShortcut() {}, + resetSession() { + this.pushToTalkMode = + this.pushToTalkKeyHeld = + this.spaceKeyHeld = + false; + }, + }), + createAudio: (callbacks) => { + acquisitions.audio++; + const audio = { + pending: false, + closed: false, + outputStream: {}, + async initialize() {}, + connectMicrophone(stream, enabled, metadata) { + this.stream = stream; + this.setMicrophoneEnabled(enabled, metadata); + }, + setMicrophoneEnabled(enabled, metadata) { + this.enabled = enabled; + this.metadata = metadata; + }, + play() { + this.pending = true; + }, + clearPlayback() { + this.pending = false; + }, + drain() { + this.pending = false; + callbacks.onDrain(); + }, + diagnostics: () => ({ outputChunks: 0 }), + async close() { + this.closed = true; + this.pending = false; + return true; + }, + }; + audios.push(audio); + return audio; + }, + createNarration: (options) => + createNarrationScheduler({ + ...options, + now: clock.now, + setTimer: clock.setTimer, + clearTimer: clock.clearTimer, + }), + createProgressEarcon: () => ({ + available: () => true, + start() { + state.earconStarts++; + }, + stop() {}, + dispose() {}, + }), + captureViewport: async () => null, + }), + }); + session.subscribe((event) => events.push(event)); + const controller = session.adapter.controller; + t.after(async () => { + session.destroy(); + for (const gate of resultGates.values()) gate.release.resolve(); + await Promise.allSettled(runnerJobs); + actualRunner.dispose(); + assert.equal(clock.timers.size, 0, 'fixture must leave no owned timer'); + assert.ok(audios.every((audio) => audio.closed)); + assert.ok(tracks.every((track) => track.readyState === 'ended')); + assert.ok(sockets.every((socket) => socket.closed)); + }); + return { + session, + controller, + state, + events, + sockets, + tracks, + audios, + requests, + acquisitions, + runnerCalls, + clock, + holdResult(callId) { + const gate = { entered: deferred(), release: deferred() }; + resultGates.set(callId, gate); + return { ready: gate.entered.promise, release: gate.release.resolve }; + }, + async start() { + await session.start(); + assert.equal(session.state, 'listening'); + assert.equal(controller.getDiagnostics().connected, true); + return sockets.at(-1); + }, + activity(socket = sockets.at(-1)) { + socket.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + socket.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + }, + tool(callId, socket = sockets.at(-1)) { + socket.message({ + toolCall: { + functionCalls: [ + { + id: callId, + name: 'adjust_camera_zoom', + args: { direction: 'in', amount: 'little' }, + }, + ], + }, + }); + // A deterministic settlement boundary, not a fake action dispatcher. + return controller.toolQueue; + }, + outputs(socket = sockets.at(-1)) { + return socket.sent.flatMap( + (message) => message.toolResponse?.functionResponses || [], + ); + }, + complete(socket = sockets.at(-1)) { + socket.message({ serverContent: { turnComplete: true } }); + }, + }; +} + +export function assertSuccessfulZoom(h, socket, callId, beforeHeight) { + assert.equal(h.state.height, beforeHeight * 0.75); + assert.ok(h.state.renders > 0); + const calls = h.events.filter( + (event) => event.type === 'action-call' && event.callId === callId, + ); + const results = h.events.filter( + (event) => event.type === 'action-result' && event.callId === callId, + ); + assert.equal(calls.length, 1); + assert.equal(results.length, 1); + assert.equal(calls[0].name, 'adjust_camera_zoom'); + assert.deepEqual(calls[0].arguments, { direction: 'in', amount: 'little' }); + assert.equal(results[0].result.ok, true); + assert.equal(results[0].result.action, 'adjust_camera_zoom'); + assert.equal(results[0].result.beforeHeightM, Math.round(beforeHeight)); + assert.equal(results[0].result.afterHeightM, Math.round(h.state.height)); + const outputs = h.outputs(socket).filter((output) => output.id === callId); + assert.equal(outputs.length, 1); + assert.equal(outputs[0].name, 'adjust_camera_zoom'); + assert.equal(outputs[0].response.ok, true); + assert.equal(outputs[0].response.action, 'adjust_camera_zoom'); + assert.equal(outputs[0].response.afterHeightM, Math.round(h.state.height)); +} diff --git a/src/testSupport/geminiNativeHarness.mjs b/src/testSupport/geminiNativeHarness.mjs new file mode 100644 index 000000000..fab535027 --- /dev/null +++ b/src/testSupport/geminiNativeHarness.mjs @@ -0,0 +1,251 @@ +import { GeminiSessionController } from '../voice/geminiSession.js'; +import { createNarrationScheduler } from '../voice/narration.js'; + +const endpoint = + 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained'; +const credential = { + token: 'auth_tokens/test', + model: 'gemini-3.8-live', + websocketUrl: endpoint, + config: { generationConfig: { responseModalities: ['AUDIO'] } }, +}; +const manualCredential = { + ...credential, + config: { + ...credential.config, + realtimeInputConfig: { automaticActivityDetection: { disabled: true } }, + }, +}; +const tick = () => new Promise((resolve) => setImmediate(resolve)); +function deferred() { + let resolve; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} +function harness(overrides = {}) { + const { autoSetup = true, ...controllerOverrides } = overrides; + const events = []; + const sockets = []; + const media = []; + const tracks = []; + const captures = []; + const calls = []; + const requests = []; + const state = { playing: 0, ducked: false, stopped: 0 }; + let clock = 0; + let nextTimer = 0; + const timers = new Map(); + const earcon = { playing: false, starts: 0, disposals: 0 }; + const advance = (ms) => { + const target = clock + ms; + for (;;) { + const due = [...timers] + .filter(([, timer]) => timer.at <= target) + .sort((a, b) => a[1].at - b[1].at)[0]; + if (!due) break; + timers.delete(due[0]); + clock = due[1].at; + due[1].fn(); + } + clock = target; + }; + let radioControl; + const radio = { + setVoiceDucked(value) { + state.ducked = value; + }, + pause() {}, + subscribePlaybackControls(callback) { + radioControl = callback; + return () => { + radioControl = null; + }; + }, + async playForVoice() { + state.playing++; + return true; + }, + stopPlayback() { + state.stopped++; + }, + }; + class Socket { + constructor(url) { + this.url = url; + this.readyState = 0; + this.bufferedAmount = 0; + this.sent = []; + sockets.push(this); + queueMicrotask(() => { + this.readyState = 1; + this.onopen?.(); + }); + } + send(raw) { + const value = JSON.parse(raw); + this.sent.push(value); + if (value.setup && autoSetup) + queueMicrotask(() => this.message({ setupComplete: {} })); + } + message(value) { + this.onmessage?.({ data: JSON.stringify(value) }); + } + error() { + this.onerror?.(); + } + remoteClose() { + this.readyState = 3; + this.onclose?.(); + } + close() { + this.readyState = 3; + this.closed = true; + } + } + const makeStream = () => { + const track = { + enabled: true, + readyState: 'live', + stop() { + this.readyState = 'ended'; + }, + }; + tracks.push(track); + return { getTracks: () => [track], getAudioTracks: () => [track] }; + }; + const c = new GeminiSessionController({ + emit: (event) => events.push(event), + runAction: async (name, args, options) => { + calls.push({ name, args, options }); + return { ok: true, action: name }; + }, + ui: { root: { remove() {} } }, + radioLayer: radio, + now: () => clock, + setTimer: (fn, ms) => { + const id = ++nextTimer; + timers.set(id, { fn, at: clock + ms }); + return id; + }, + clearTimer: (id) => timers.delete(id), + WebSocketClass: Socket, + fetchImpl: async (_url, init) => { + requests.push(init); + return Response.json( + JSON.parse(init.body).inputMode === 'push-to-talk' + ? manualCredential + : credential, + ); + }, + getUserMedia: async () => makeStream(), + createInput: ({ readStream, operations }) => ({ + ...operations, + setMicrophoneEnabled(enabled) { + for (const track of readStream()?.getAudioTracks() || []) + track.enabled = enabled; + }, + updateVoiceButtonLabel() {}, + setVoiceSpeaker() {}, + startVoiceVisualizer() {}, + startAssistantVoiceVisualizer() {}, + cancelPushToTalkHold() {}, + stopVoiceVisualizer() {}, + detachBindings() {}, + bindPushToTalkShortcut() {}, + resetSession() { + this.pushToTalkMode = false; + this.pushToTalkKeyHeld = false; + this.spaceKeyHeld = false; + }, + }), + createNarration: (options) => + createNarrationScheduler({ + ...options, + now: () => clock, + setTimer: (fn, ms) => { + const id = ++nextTimer; + timers.set(id, { fn, at: clock + ms }); + return id; + }, + clearTimer: (id) => timers.delete(id), + }), + createProgressEarcon: ({ isMicrophoneMuted }) => ({ + available: isMicrophoneMuted, + start() { + earcon.playing = true; + earcon.starts++; + }, + stop() { + earcon.playing = false; + }, + dispose() { + this.stop(); + earcon.disposals++; + }, + }), + createAudio: (callbacks) => { + const audio = { + callbacks, + pending: false, + closed: false, + played: [], + async initialize() {}, + connectMicrophone(stream, enabled, metadata) { + media.push({ stream, enabled }); + this.connected = true; + this.setMicrophoneEnabled(enabled, metadata); + }, + setMicrophoneEnabled(value, metadata) { + if (!this.connected) return; + if (value && !this.enabled) { + this.metadata = metadata; + callbacks.onAudioStart(metadata); + } + if (!value && this.enabled) callbacks.onAudioEnd(this.metadata); + this.enabled = value; + }, + play(data, mime) { + this.pending = true; + this.played.push({ data, mime }); + }, + drain() { + this.pending = false; + callbacks.onDrain(); + }, + clearPlayback() { + this.pending = false; + this.cleared = true; + }, + diagnostics() { + return { outputChunks: this.played.length }; + }, + close() { + this.closed = true; + }, + }; + captures.push(audio); + return audio; + }, + ...controllerOverrides, + }); + return { + c, + sockets, + tracks, + makeStream, + events, + media, + captures, + calls, + requests, + state, + advance, + timers, + earcon, + radioControl: (value) => radioControl?.(value), + }; +} + +export { harness, tick, deferred }; diff --git a/src/tooling/viteBuild.test.mjs b/src/tooling/viteBuild.test.mjs index 9e38a0295..e1eb1aecd 100644 --- a/src/tooling/viteBuild.test.mjs +++ b/src/tooling/viteBuild.test.mjs @@ -88,8 +88,19 @@ test('root config retains existing named exports and standalone provider order', for (const [name, value] of Object.entries(providers)) assert.equal(compatibility[name], value, name); const config = standaloneConfig({ mode: 'test' }); + const standaloneProviders = config.plugins + .slice(3, -5) + .map((plugin) => plugin.name); + assert.equal( + standaloneProviders.filter((name) => name === 'gemini-live-proxy').length, + 1, + ); + assert.equal( + standaloneProviders.indexOf('gemini-live-proxy'), + standaloneProviders.indexOf('openai-realtime-proxy') + 1, + ); assert.deepEqual( - config.plugins.slice(3, -5).map((plugin) => plugin.name), + standaloneProviders.filter((name) => name !== 'gemini-live-proxy'), providers.localProviderPlugins().map((plugin) => plugin.name), ); assert.equal(config.plugins.at(-6).name, 'gev-key-setup'); diff --git a/src/ui/panelChrome.js b/src/ui/panelChrome.js index 359ac3792..6fd04554a 100644 --- a/src/ui/panelChrome.js +++ b/src/ui/panelChrome.js @@ -72,7 +72,8 @@ export class PanelChrome { variant: this.hud.getVariant(), }), scheduleCockpitLayout: () => this.cockpitView?.scheduleContextLayout(), - syncPanelCollapseButton: (panel) => this._syncPanelCollapseButton(panel), + syncPanelCollapseButton: (panel) => + this._syncRailPanelCollapseButton(panel), readDisplayScrollTop: () => this._displayPortalScrollRestoreOwner === 'standard' ? this._standardDisplayScrollTop @@ -287,6 +288,19 @@ export class PanelChrome { return this._panelLayout._scheduleLeftPanelLayout(options0); } + _syncRailPanelCollapseButton(panelEl) { + // Adaptive rail layout may directly restore an auto-collapsed Context + // without invoking setPanelCollapsed(). Enforce Radio ownership there too. + if ( + panelEl?.id === 'global-context-panel' && + !panelEl.classList.contains('collapsed') && + this._contextRadioDock?.classList.contains('disclosure-open') + ) { + this._setRadioDisclosure?.(false); + } + this._syncPanelCollapseButton(panelEl); + } + _syncPanelCollapseButton(panelEl) { const isRightRail = [ 'pp-toggles', @@ -501,6 +515,15 @@ export class PanelChrome { syncShare, }); } + // A repeated Context-open request must clear a stale compact Radio mini + // even when no collapsed class changes. + if ( + !nextCollapsed && + panelId === 'global-context-panel' && + this._contextRadioDock?.classList.contains('disclosure-open') + ) { + this._setRadioDisclosure?.(false); + } if ( panelEl.classList.contains('collapsed') === nextCollapsed && !wasAutoCollapsed diff --git a/src/ui/panelChrome.radioOwnership.test.mjs b/src/ui/panelChrome.radioOwnership.test.mjs new file mode 100644 index 000000000..123eda6e6 --- /dev/null +++ b/src/ui/panelChrome.radioOwnership.test.mjs @@ -0,0 +1,200 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { PanelChrome } from './panelChrome.js'; +import { PanelLayoutController } from './panelLayoutController.js'; + +function classList(initial = []) { + const values = new Set(initial); + return { + contains: (value) => values.has(value), + add: (...items) => items.forEach((item) => values.add(item)), + remove: (...items) => items.forEach((item) => values.delete(item)), + toggle(value, force) { + if (force === true) values.add(value); + else if (force === false) values.delete(value); + else if (values.has(value)) values.delete(value); + else values.add(value); + return values.has(value); + }, + }; +} + +test('opening an already-open Context still closes a stale compact Radio disclosure', (t) => { + const priorDocument = globalThis.document; + const contextPanel = { + id: 'global-context-panel', + classList: classList(), + }; + globalThis.document = { + documentElement: { dataset: { uiTheme: 'tactical' } }, + getElementById(id) { + return id === 'global-context-panel' ? contextPanel : null; + }, + }; + t.after(() => { + globalThis.document = priorDocument; + }); + + const radioDock = { classList: classList(['disclosure-open']) }; + const calls = []; + const owner = { + _contextRadioDock: radioDock, + _setRadioDisclosure(open) { + calls.push(open); + radioDock.classList.toggle('disclosure-open', open); + }, + _leftPanelStack: null, + _rightPanelStack: null, + _panelLayout: { + _leftStackPreferredPanelId: null, + _rightStackPreferredPanelId: null, + }, + _syncPanelCollapseButton() {}, + _scheduleLeftPanelLayout() {}, + _scheduleRightPanelLayout() {}, + shareLinkManager: null, + cockpitView: null, + }; + + PanelChrome.prototype.setPanelCollapsed.call( + owner, + 'global-context-panel', + false, + ); + + assert.deepEqual(calls, [false]); + assert.equal(radioDock.classList.contains('disclosure-open'), false); +}); + +test('adaptive rail restore hides compact Radio before revealing Context', (t) => { + const priorRaf = globalThis.requestAnimationFrame; + let nextFrame; + globalThis.requestAnimationFrame = (callback) => { + nextFrame = callback; + return 1; + }; + t.after(() => { + globalThis.requestAnimationFrame = priorRaf; + }); + + // Valid starting state: the tactical rail automatically collapsed Context, + // and the user opened the compact Radio controls from its header. + const contextPanel = { + id: 'global-context-panel', + classList: classList(['collapsed', 'layout-auto-collapsed']), + }; + const radioDock = { classList: classList(['disclosure-open']) }; + const radioMini = { hidden: false }; + const calls = []; + const chrome = { + _contextRadioDock: radioDock, + _setRadioDisclosure(open) { + calls.push(`disclosure:${open}`); + radioDock.classList.toggle('disclosure-open', open); + radioMini.hidden = !open; + }, + _syncPanelCollapseButton(panel) { + calls.push(`button:${panel.id}`); + }, + }; + const layout = { + destroyed: false, + _rightStackLayoutFrame: null, + _rightStackReconsiderAutoCollapse: false, + _rightPanelStack: { + querySelectorAll: (selector) => + selector === '.layout-auto-collapsed' ? [contextPanel] : [], + }, + _syncPanelCollapseButton(panel) { + PanelChrome.prototype._syncRailPanelCollapseButton.call(chrome, panel); + }, + _syncRightPanelAdaptiveLayout() { + calls.push('layout'); + }, + }; + + PanelLayoutController.prototype._scheduleRightPanelLayout.call(layout, { + reconsiderAutoCollapse: true, + }); + assert.equal(contextPanel.classList.contains('collapsed'), true); + assert.equal(radioMini.hidden, false); + nextFrame(); + + assert.equal(contextPanel.classList.contains('collapsed'), false); + assert.equal(contextPanel.classList.contains('layout-auto-collapsed'), false); + assert.equal(radioDock.classList.contains('disclosure-open'), false); + assert.equal(radioMini.hidden, true); + assert.deepEqual(calls, [ + 'disclosure:false', + 'button:global-context-panel', + 'layout', + ]); +}); + +test('real PanelChrome constructor wires rail restore to compact Radio owner', (t) => { + const priorDocument = globalThis.document; + const priorRaf = globalThis.requestAnimationFrame; + let nextFrame; + const contextPanel = { + id: 'global-context-panel', + classList: classList(['collapsed', 'layout-auto-collapsed']), + }; + const rightRail = { + querySelectorAll(selector) { + return selector === '.layout-auto-collapsed' ? [contextPanel] : []; + }, + }; + globalThis.document = { + getElementById(id) { + return id === 'right-context-rail' ? rightRail : null; + }, + }; + globalThis.requestAnimationFrame = (callback) => { + nextFrame = callback; + return 1; + }; + t.after(() => { + globalThis.document = priorDocument; + globalThis.requestAnimationFrame = priorRaf; + }); + + const dock = { classList: classList(['disclosure-open']) }; + const calls = []; + const chrome = new PanelChrome({ + elements: { _contextRadioDock: dock }, + operations: { + _setRadioDisclosure(open) { + calls.push(`disclosure:${open}`); + dock.classList.toggle('disclosure-open', open); + }, + _syncPanelCollapseButton(panel) { + calls.push(`button:${panel.id}`); + }, + }, + readHud: () => ({ visible: true, getVariant: () => 'tactical' }), + readCockpit: () => null, + readShareLinks: () => null, + readInitialShare: () => null, + readScrollRestoreOwner: () => null, + readDisplayScrollTop: () => 0, + }); + chrome._panelLayout._syncRightPanelAdaptiveLayout = () => { + calls.push('layout'); + }; + + chrome._panelLayout._scheduleRightPanelLayout({ + reconsiderAutoCollapse: true, + }); + assert.equal(typeof nextFrame, 'function'); + assert.equal(dock.classList.contains('disclosure-open'), true); + nextFrame(); + + assert.equal(contextPanel.classList.contains('collapsed'), false); + assert.equal(dock.classList.contains('disclosure-open'), false); + assert.deepEqual(calls, [ + 'disclosure:false', + 'button:global-context-panel', + 'layout', + ]); +}); diff --git a/src/ui/styles/voice-cost.css b/src/ui/styles/voice-cost.css index 128811e3b..b43f40168 100644 --- a/src/ui/styles/voice-cost.css +++ b/src/ui/styles/voice-cost.css @@ -140,3 +140,95 @@ html:not([lang='en']) white-space: normal; overflow-wrap: anywhere; } + +#gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: 0px; +} + +/* Provider choice belongs to the existing voice control, not a separate panel. */ +.gev-voice-provider-field { + position: absolute; + bottom: calc(100% + var(--gev-provider-lift, 0px)); + z-index: 9; + left: 0; + right: 0; + display: grid; + gap: 4px; + padding: 8px 6px; + border: 1px solid var(--glass-border); + border-radius: 8px; + background: var(--bg-panel, #09121b); + color: var(--text-secondary); + font-family: var(--font-mono); + font-size: 0.5rem; + opacity: 0; + pointer-events: none; +} +.gev-voice-provider-field[hidden] { + display: none; +} +.gev-voice-provider-field select { + min-width: 0; + width: 100%; + min-height: 30px; + color: var(--text-primary); + background: #09121b; + border: 1px solid var(--glass-border); + border-radius: 4px; + font: inherit; + font-size: 0.65rem; +} +#gev-voice-control:hover .gev-voice-provider-field, +#gev-voice-control:focus-within .gev-voice-provider-field { + opacity: 1; + pointer-events: auto; +} +#command-dock + #gev-voice-control[data-provider-choice='true'] + .gev-voice-help-tray, +#command-dock #gev-voice-control[data-provider-choice='true'] .gev-voice-card, +#command-dock + #gev-voice-control[data-provider-choice='true'] + .gev-voice-error-tray { + bottom: calc(100% + 4.8rem + var(--gev-provider-lift)); +} +#command-dock.dock-has-pinned-tray + #gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: calc(1.7rem + var(--dock-pinned-stack-height)); +} +/* Pinned trays separate the chooser from its hover owner. Keep it reachable + across that gap without requiring keyboard focus to hold it open. */ +#command-dock.dock-has-pinned-tray .gev-voice-provider-field { + opacity: 1; + pointer-events: auto; +} +@media (max-width: 900px) { + /* The attribution clearance also creates a gap for mouse users in a + narrow window, so the raised chooser must not depend on root hover. */ + .gev-voice-provider-field { + opacity: 1; + pointer-events: auto; + } + #command-dock #gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: 3rem; + } + #command-dock.dock-has-pinned-tray + #gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: calc(3.5rem + var(--dock-pinned-stack-height)); + } +} +@media (max-width: 720px) { + #command-dock #gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: calc(2vh + 2.5rem); + } + #command-dock.dock-has-pinned-tray + #gev-voice-control[data-provider-choice='true'] { + --gev-provider-lift: calc(2vh + 3rem + var(--dock-pinned-stack-height)); + } +} +@media (hover: none) { + .gev-voice-provider-field { + opacity: 1; + pointer-events: auto; + } +} diff --git a/src/voice/commands.js b/src/voice/commands.js index cc059852b..c5f92b209 100644 --- a/src/voice/commands.js +++ b/src/voice/commands.js @@ -1,8 +1,16 @@ import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js'; import { createRealtimeSession } from './realtimeSession.js'; +import { createGeminiSession } from './geminiSession.js'; +import { createProviderCommands } from './providerCommands.js'; /** Default composition; callers may supply another session adapter factory. */ -export function createVoiceCommands(options) { +export function createVoiceCommands(options = {}) { + // Generic hosts retain the existing default; standalone opts in explicitly. + if (!options.createSession && options.supportedProviders !== undefined) + return createProviderCommands(options, { + bind: bindVoiceCommands, + factories: { openai: createRealtimeSession, gemini: createGeminiSession }, + }); return bindVoiceCommands({ createSession: createRealtimeSession, ...options, diff --git a/src/voice/control.js b/src/voice/control.js index e729a5cb1..c77197412 100644 --- a/src/voice/control.js +++ b/src/voice/control.js @@ -39,6 +39,10 @@ export function createVoiceControl({ reset = false } = {}) { root.dataset.status = 'idle'; root.dataset.speaker = 'idle'; root.innerHTML = ` +
${t('AI AGENT')}
${voiceStatusLabel('idle')}
@@ -110,6 +114,8 @@ export function createVoiceControl({ reset = false } = {}) { } return { root, + providerField: root.querySelector('#gev-voice-provider-field'), + providerSelect: root.querySelector('#gev-voice-provider'), button: root.querySelector('#gev-voice-button'), buttonLabel: root.querySelector('.gev-mic-label'), status: root.querySelector('#gev-voice-status'), diff --git a/src/voice/geminiAcquisition.test.mjs b/src/voice/geminiAcquisition.test.mjs new file mode 100644 index 000000000..1b80d6cd6 --- /dev/null +++ b/src/voice/geminiAcquisition.test.mjs @@ -0,0 +1,142 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + harness, + deferred, + tick, +} from '../testSupport/geminiNativeHarness.mjs'; + +function unreadResponse(bytes = null) { + const state = { reads: 0, cancelled: 0 }; + const response = new Response( + new ReadableStream( + { + pull(controller) { + state.reads++; + if (bytes) controller.enqueue(bytes); + }, + cancel() { + state.cancelled++; + }, + }, + { highWaterMark: 0 }, + ), + ); + return { response, state }; +} + +test('already aborted token request does not acquire or read a response', async (t) => { + let fetches = 0; + const h = harness({ + fetchImpl: () => { + fetches++; + throw Error('must not fetch'); + }, + }); + t.after(() => h.c.stop()); + const abort = new AbortController(); + abort.abort(); + await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' }); + assert.equal(fetches, 0); +}); + +for (const phase of [ + 'before response', + 'between resolution and continuation', +]) { + test(`aborted token acquisition closes the unread body ${phase}`, async (t) => { + const gate = deferred(); + const h = harness({ fetchImpl: () => gate.promise }); + t.after(() => h.c.stop()); + const abort = new AbortController(); + const { response, state } = unreadResponse(); + const result = assert.rejects(h.c.requestToken(abort.signal), { + name: 'AbortError', + }); + if (phase === 'before response') abort.abort(); + gate.resolve(response); + if (phase !== 'before response') queueMicrotask(() => abort.abort()); + await result; + await tick(); + assert.deepEqual(state, { reads: 0, cancelled: 1 }); + }); +} + +test('token body cap cancels a native response stream', async (t) => { + const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1)); + const h = harness({ fetchImpl: async () => response }); + t.after(() => h.c.stop()); + await assert.rejects( + h.c.requestToken(new AbortController().signal), + /too large|exceed|limit/i, + ); + assert.equal(state.cancelled, 1); + assert.equal(state.reads, 1); +}); + +test('token body read ends promptly on caller abort and releases the reader', async (t) => { + const { response, state } = unreadResponse(); + const h = harness({ fetchImpl: async () => response }); + t.after(() => h.c.stop()); + const abort = new AbortController(); + const result = assert.rejects(h.c.requestToken(abort.signal), { + name: 'AbortError', + }); + await tick(); + assert.equal(state.reads, 1); + abort.abort(); + await result; + assert.equal(state.cancelled, 1); + assert.equal(response.body.locked, false); +}); + +test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => { + const gate = deferred(); + const h = harness({ getUserMedia: () => gate.promise }); + t.after(() => h.c.stop()); + const stream = h.makeStream(); + const start = h.c.start(); + gate.resolve(stream); + queueMicrotask(() => h.c.stop()); + await start; + await tick(); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.requests.length, 0); + assert.equal(h.sockets.length, 0); + assert.equal(h.captures[0].closed, true); + h.c.getUserMedia = async () => h.makeStream(); + await h.c.start(); + assert.equal(h.c.status, 'listening'); + assert.equal(h.tracks[1].readyState, 'live'); + assert.equal(h.captures[1].closed, false); +}); + +test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => { + const gate = deferred(); + const h = harness(); + t.after(() => h.c.stop()); + const createAudio = h.c.createAudio; + let first = true; + h.c.createAudio = (options) => { + const audio = createAudio(options); + if (first) { + first = false; + audio.initialize = () => gate.promise; + audio.connectMicrophone = () => + assert.fail('retired initialization must not connect'); + } + return audio; + }; + const oldStart = h.c.start(); + await tick(); + h.c.stop(); + await h.c.start(); + gate.resolve(); + await oldStart; + await tick(); + assert.equal(h.c.status, 'listening'); + assert.equal(h.captures[0].closed, true); + assert.equal(h.captures[1].closed, false); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.tracks[1].readyState, 'live'); +}); diff --git a/src/voice/geminiAudio.js b/src/voice/geminiAudio.js new file mode 100644 index 000000000..3074923e5 --- /dev/null +++ b/src/voice/geminiAudio.js @@ -0,0 +1,380 @@ +import { msg } from '../i18n/marker.js'; +const MAX_AUDIO_BYTES = 2 * 1024 * 1024; +const MAX_QUEUED_SECONDS = 60; +const MAX_QUEUED_SOURCES = 256; +const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz. +const PTT_ENERGY_FLOOR = 0.002; +const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4; + +/** Admit manual input only after sustained energy; not an utterance detector. */ +function pttCapture(captureId) { + return { + captureId, + admitted: false, + buffers: [], + bytes: 0, + samples: 0, + energy: 0, + aboveFloor: 0, + voicedWindows: 0, + }; +} + +function hasPttEnergy(state, pcm) { + for (let index = 0; index + 1 < pcm.byteLength; index += 2) { + const sample = pcm.getInt16(index, true) / 32768; + state.energy += sample * sample; + if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++; + if (++state.samples !== PTT_WINDOW_SAMPLES) continue; + // Requiring occupancy as well as RMS rejects isolated high-peak key clicks. + const voiced = + state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 && + state.aboveFloor >= PTT_WINDOW_SAMPLES / 8; + state.voicedWindows = voiced ? state.voicedWindows + 1 : 0; + state.samples = state.energy = state.aboveFloor = 0; + if (state.voicedWindows >= 2) return true; + } + return false; +} + +export function pcmBytesToBase64(buffer) { + const bytes = new Uint8Array(buffer); + let binary = ''; + for (let offset = 0; offset < bytes.length; offset += 8192) { + binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192)); + } + return btoa(binary); +} + +export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') { + if ( + typeof data !== 'string' || + data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3) + ) + throw new Error(msg('Gemini returned an oversized audio chunk')); + const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType); + if (!match) + throw new Error(msg('Gemini returned an unsupported audio format')); + const sampleRate = Number(match[1] || 24000); + if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000) + throw new Error(msg('Gemini returned an invalid audio sample rate')); + const binary = atob(data); + if (binary.length > MAX_AUDIO_BYTES) + throw new Error(msg('Gemini returned an oversized audio chunk')); + if (!binary.length || binary.length % 2) + throw new Error(msg('Gemini returned incomplete PCM audio')); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) + bytes[index] = binary.charCodeAt(index); + const view = new DataView(bytes.buffer); + const samples = new Float32Array(bytes.length / 2); + for (let index = 0; index < samples.length; index++) + samples[index] = view.getInt16(index * 2, true) / 32768; + return { samples, sampleRate }; +} + +/** Own capture and playback independently from the Live protocol. */ +export function createGeminiAudio({ + pushToTalk = false, + onAudioStart, + onAudio, + onAudioEnd, + onAudioDiscard, + onDrain, + onError, + AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext, + AudioWorkletNodeClass = globalThis.AudioWorkletNode, +} = {}) { + if (!AudioContextClass || !AudioWorkletNodeClass) + throw new Error( + msg('This browser does not support Gemini microphone audio'), + ); + let captureContext; + let playbackContext; + let output; + let meter; + let resumed; + const closeContext = (context) => { + try { + return Promise.resolve(context?.close()); + } catch (error) { + return Promise.reject(error); + } + }; + try { + // Capture at the device rate; the worklet explicitly resamples to 16 kHz. + captureContext = new AudioContextClass(); + playbackContext = new AudioContextClass({ sampleRate: 24000 }); + // Resume both synchronously within the mic gesture, before any awaits. + const resume = (context) => { + try { + return Promise.resolve(context.resume()); + } catch (error) { + return Promise.reject(error); + } + }; + resumed = Promise.all([resume(captureContext), resume(playbackContext)]); + // Permission may remain pending before initialize observes this failure. + void resumed.catch(() => {}); + output = playbackContext.createGain(); + meter = playbackContext.createMediaStreamDestination(); + output.connect(playbackContext.destination); + output.connect(meter); + } catch (error) { + output?.disconnect(); + meter?.disconnect(); + meter?.stream.getTracks?.().forEach((track) => track.stop()); + void Promise.allSettled([ + closeContext(captureContext), + closeContext(playbackContext), + ]); + throw error; + } + let closed = false; + let closeResult; + let capture = null; + let microphone = null; + let mute = null; + let nextTime = 0; + let inputChunks = 0; + let inputPeak = 0; + let nonSilentInputChunks = 0; + let outputChunks = 0; + let outputSamples = 0; + let completedOutputSamples = 0; + let discardedOutputSamples = 0; + let pendingPtt = null; + let latestPttCaptureId = 0; + let suppressedPttCaptures = 0; + let admittedPttCaptures = 0; + const sources = new Map(); + + const emitAudio = (buffer, metadata) => + onAudio?.( + { data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' }, + metadata, + ); + + function clearPlayback() { + const oldSources = [...sources]; + sources.clear(); + nextTime = playbackContext.currentTime; + for (const [source, sampleCount] of oldSources) { + discardedOutputSamples += sampleCount; + source.onended = null; + try { + source.stop(); + } catch { + /* Already ended. */ + } + source.disconnect(); + } + } + + return { + outputStream: meter.stream, + get pending() { + return sources.size > 0; + }, + async initialize(signal) { + try { + await Promise.all([ + resumed, + captureContext.audioWorklet.addModule( + new URL('./geminiCaptureWorklet.js', import.meta.url), + ), + ]); + signal?.throwIfAborted(); + if (closed) throw new DOMException('Voice stopped', 'AbortError'); + } catch (error) { + await this.close(); + throw error; + } + }, + connectMicrophone(stream, enabled, metadata) { + if (closed) throw new DOMException('Voice stopped', 'AbortError'); + capture = new AudioWorkletNodeClass( + captureContext, + 'gev-gemini-capture', + { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + }, + ); + capture.port.onmessage = ({ data }) => { + if (closed) return; + // Capture ownership is transport metadata, never part of Live audio. + if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0) + return; + const metadata = { captureId: data.captureId }; + try { + if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) { + inputChunks++; + const pcm = new DataView(data.buffer); + let peak = 0; + for (let index = 0; index + 1 < pcm.byteLength; index += 2) + peak = Math.max( + peak, + Math.abs(pcm.getInt16(index, true)) / 32768, + ); + inputPeak = Math.max(inputPeak, peak); + if (peak >= 0.01) nonSilentInputChunks++; + if (!pushToTalk) emitAudio(data.buffer, metadata); + else { + const state = pendingPtt; + if (state?.captureId !== data.captureId) return; + if (state.admitted) emitAudio(data.buffer, metadata); + else { + state.buffers.push(data.buffer); + state.bytes += data.buffer.byteLength; + if (hasPttEnergy(state, pcm)) { + state.admitted = true; + admittedPttCaptures++; + // Context and activityStart are gated together with PCM. + onAudioStart?.(metadata); + for (const buffer of state.buffers) { + if (closed || pendingPtt !== state) break; + emitAudio(buffer, metadata); + } + state.buffers = []; + state.bytes = 0; + } else { + while (state.bytes > PTT_PRE_ROLL_BYTES) { + state.bytes -= state.buffers.shift().byteLength; + } + } + } + } + } else if (data?.type === 'start') { + if (!pushToTalk) onAudioStart?.(metadata); + else if (data.captureId > latestPttCaptureId) { + latestPttCaptureId = data.captureId; + pendingPtt = pttCapture(data.captureId); + } + } else if (data?.type === 'end') { + if (!pushToTalk) onAudioEnd?.(metadata); + else if (pendingPtt?.captureId === data.captureId) { + const admitted = pendingPtt.admitted; + pendingPtt = null; + if (admitted) onAudioEnd?.(metadata); + else { + suppressedPttCaptures++; + onAudioDiscard?.(metadata); + } + } + } + } catch (error) { + onError?.(error); + } + }; + capture.onprocessorerror = () => { + if (!closed) + onError?.(new Error(msg('Microphone audio processing failed'))); + }; + microphone = captureContext.createMediaStreamSource(stream); + mute = captureContext.createGain(); + mute.gain.value = 0; + capture.port.postMessage({ + enabled: Boolean(enabled), + captureId: metadata?.captureId, + }); + microphone.connect(capture); + capture.connect(mute); + mute.connect(captureContext.destination); + }, + setMicrophoneEnabled(enabled, metadata) { + if (closed) return; + capture?.port.postMessage({ + enabled: Boolean(enabled), + captureId: metadata?.captureId, + }); + }, + play(data, mimeType) { + if (closed) return; + // Tiny chunks can exhaust source nodes long before the duration limit. + if (sources.size >= MAX_QUEUED_SOURCES) + throw new Error( + msg('Gemini audio playback fell behind. Reconnect to continue.'), + ); + const { samples, sampleRate } = decodeGeminiPcm(data, mimeType); + const start = Math.max(playbackContext.currentTime + 0.02, nextTime); + if ( + start + samples.length / sampleRate - playbackContext.currentTime > + MAX_QUEUED_SECONDS + ) + throw new Error( + msg('Gemini audio playback fell behind. Reconnect to continue.'), + ); + const buffer = playbackContext.createBuffer( + 1, + samples.length, + sampleRate, + ); + buffer.copyToChannel(samples, 0); + const source = playbackContext.createBufferSource(); + source.buffer = buffer; + source.connect(output); + sources.set(source, samples.length); + source.onended = () => { + if (!sources.delete(source)) return; + completedOutputSamples += samples.length; + source.disconnect(); + if (!closed && !sources.size) onDrain?.(); + }; + nextTime = start + buffer.duration; + outputChunks++; + outputSamples += samples.length; + source.start(start); + }, + clearPlayback, + diagnostics() { + return { + inputChunks, + inputPeak, + nonSilentInputChunks, + outputChunks, + outputSamples, + completedOutputSamples, + discardedOutputSamples, + queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime), + contextState: + captureContext.state === playbackContext.state + ? captureContext.state + : 'suspended', + captureContextState: captureContext.state, + playbackContextState: playbackContext.state, + captureSampleRate: captureContext.sampleRate, + playbackSampleRate: playbackContext.sampleRate, + inputSampleRate: 16000, + suppressedPttCaptures, + admittedPttCaptures, + pttBufferedBytes: pendingPtt?.bytes || 0, + }; + }, + close() { + if (closed) return closeResult; + closed = true; + pendingPtt = null; + clearPlayback(); + if (capture) { + capture.port.onmessage = null; + capture.onprocessorerror = null; + capture.port.close(); + capture.disconnect(); + } + microphone?.disconnect(); + mute?.disconnect(); + output.disconnect(); + meter.disconnect(); + meter.stream.getTracks?.().forEach((track) => track.stop()); + closeResult = Promise.allSettled([ + closeContext(captureContext), + closeContext(playbackContext), + ]).then((results) => + results.every((result) => result.status === 'fulfilled'), + ); + return closeResult; + }, + }; +} diff --git a/src/voice/geminiAudio.test.mjs b/src/voice/geminiAudio.test.mjs new file mode 100644 index 000000000..5ba719b32 --- /dev/null +++ b/src/voice/geminiAudio.test.mjs @@ -0,0 +1,1134 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import vm from 'node:vm'; +import { + createGeminiAudio, + decodeGeminiPcm, + pcmBytesToBase64, +} from './geminiAudio.js'; + +test('PCM decoding preserves little-endian signed samples and rejects malformed audio', () => { + const data = new Uint8Array([0, 128, 0, 0, 255, 127]); + const decoded = decodeGeminiPcm(pcmBytesToBase64(data.buffer)); + assert.equal(decoded.sampleRate, 24000); + assert.deepEqual([...decoded.samples], [-1, 0, 32767 / 32768]); + assert.throws(() => decodeGeminiPcm('AA=='), /incomplete/); + assert.throws(() => decodeGeminiPcm('AAAA', 'audio/mp3'), /unsupported/); + assert.throws( + () => decodeGeminiPcm('AAAA', 'audio/pcm;rate=1'), + /sample rate/, + ); +}); + +test('PCM decoding accepts both rate endpoints and the exact decoded chunk limit', () => { + for (const rate of [8000, 96000]) { + const decoded = decodeGeminiPcm('AAA=', `audio/pcm;rate=${rate}`); + assert.equal(decoded.sampleRate, rate); + assert.deepEqual([...decoded.samples], [0]); + } + const limit = 2 * 1024 * 1024; + const maximum = decodeGeminiPcm( + pcmBytesToBase64(new ArrayBuffer(limit)), + 'audio/pcm;rate=96000', + ); + assert.equal(maximum.samples.length, limit / 2); + for (const size of [limit + 1, limit + 2]) + assert.throws( + () => decodeGeminiPcm(pcmBytesToBase64(new ArrayBuffer(size))), + /oversized audio chunk/, + ); +}); + +function createCaptureHarness(sampleRate = 16000) { + let Processor; + const messages = []; + vm.runInNewContext( + fs.readFileSync( + new URL('./geminiCaptureWorklet.js', import.meta.url), + 'utf8', + ), + { + sampleRate, + AudioWorkletProcessor: class { + constructor() { + this.port = { postMessage: (message) => messages.push(message) }; + } + }, + registerProcessor: (_name, Class) => { + Processor = Class; + }, + }, + ); + return { capture: new Processor(), messages }; +} + +test('worklet orders start, bounded PCM and the final partial chunk before end', () => { + const { capture, messages } = createCaptureHarness(); + capture.process([[new Float32Array([-1, 0, 1])]]); + assert.equal(messages.length, 0); + capture.port.onmessage({ data: { enabled: true, captureId: 1 } }); + capture.process([[new Float32Array([-1, 0, 1])]]); + capture.port.onmessage({ data: { enabled: false, captureId: 1 } }); + assert.deepEqual( + messages.map((message) => message.type), + ['start', 'audio', 'end'], + ); + const view = new DataView(messages[1].buffer); + assert.deepEqual( + [view.getInt16(0, true), view.getInt16(2, true), view.getInt16(4, true)], + [-32768, 0, 32767], + ); + capture.process([[new Float32Array(2048)]]); + assert.equal(messages.length, 3); +}); + +test('worklet serializes rapid re-holds and ignores repeated enable or release', () => { + const { capture, messages } = createCaptureHarness(); + let captureId = 0; + let gate = false; + const enable = (enabled) => { + if (enabled && !gate) captureId++; + gate = enabled; + capture.port.onmessage({ data: { enabled, captureId } }); + }; + enable(false); + enable(true); + enable(true); + capture.process([[new Float32Array(1025).fill(0.5)]]); + enable(false); + enable(false); + capture.process([[new Float32Array(2048).fill(1)]]); + enable(true); + capture.process([[new Float32Array([0.25])]]); + enable(false); + assert.deepEqual( + messages.map((message) => message.type), + ['start', 'audio', 'audio', 'end', 'start', 'audio', 'end'], + ); + assert.deepEqual( + messages + .filter((message) => message.type === 'audio') + .map((message) => message.buffer.byteLength), + [2048, 2, 2], + ); + assert.equal(new DataView(messages[5].buffer).getInt16(0, true), 8192); + assert.deepEqual( + messages.map((message) => message.captureId), + [1, 1, 1, 1, 2, 2, 2], + ); + enable(true); + enable(false); + assert.deepEqual( + messages.slice(-2).map((message) => message.type), + ['start', 'end'], + ); + assert.equal(capture.length, 0); + assert.equal(capture.enabled, false); +}); + +function createPlaybackHarness() { + const sources = []; + const buffers = []; + let drains = 0; + let context; + const contexts = []; + const node = () => ({ + connect() {}, + disconnect() { + this.disconnected = true; + }, + gain: { value: 1 }, + }); + class Context { + constructor(options) { + context = this; + contexts.push(this); + this.options = options; + this.currentTime = 0; + this.sampleRate = options?.sampleRate ?? 48000; + this.state = 'running'; + this.destination = {}; + this.audioWorklet = { addModule: async () => {} }; + } + async resume() {} + async close() { + this.state = 'closed'; + } + createGain() { + return node(); + } + createMediaStreamDestination() { + return { ...node(), stream: {} }; + } + createBuffer(_channels, length, rate) { + const buffer = { duration: length / rate, copyToChannel() {} }; + buffers.push(buffer); + return buffer; + } + createBufferSource() { + const source = { + ...node(), + start(at) { + this.at = at; + }, + stop() { + this.stopped = true; + }, + }; + sources.push(source); + return source; + } + } + const audio = createGeminiAudio({ + AudioContextClass: Context, + AudioWorkletNodeClass: class {}, + onDrain: () => { + drains++; + }, + }); + return { + audio, + sources, + buffers, + contexts, + get drains() { + return drains; + }, + get context() { + return context; + }, + }; +} + +function outputDiagnostics(audio) { + const { outputSamples, completedOutputSamples, discardedOutputSamples } = + audio.diagnostics(); + return { outputSamples, completedOutputSamples, discardedOutputSamples }; +} + +test('pending sources are capped before decoding or allocating and natural completion frees a slot', async (t) => { + const playback = createPlaybackHarness(); + const { audio, sources, buffers } = playback; + t.after(() => audio.close()); + const decode = t.mock.method(globalThis, 'atob', globalThis.atob); + await audio.initialize(); + for (let index = 0; index < 256; index++) + audio.play('AAA=', 'audio/pcm;rate=96000'); + assert.equal(audio.pending, true); + assert.ok(audio.diagnostics().queuedSeconds < 0.023); + assert.equal(decode.mock.calls.length, 256); + assert.equal(buffers.length, 256); + assert.equal(sources.length, 256); + const before = audio.diagnostics(); + assert.throws( + () => audio.play('AAA=', 'audio/pcm;rate=96000'), + /Gemini audio playback fell behind\. Reconnect to continue\./, + ); + assert.equal(decode.mock.calls.length, 256); + assert.equal(buffers.length, 256); + assert.equal(sources.length, 256); + assert.deepEqual(audio.diagnostics(), before); + + const ended = sources[0].onended; + ended(); + assert.equal(sources[0].disconnected, true); + assert.equal(playback.drains, 0); + audio.play('AAA=', 'audio/pcm;rate=96000'); + assert.equal(decode.mock.calls.length, 257); + assert.equal(buffers.length, 257); + assert.equal(sources.length, 257); + ended(); + assert.equal(audio.diagnostics().completedOutputSamples, 1); + assert.throws(() => audio.play('AAA='), /playback fell behind/); + assert.equal(decode.mock.calls.length, 257); +}); + +test('clearing and closing a full queue release every source and ignore late completion', async (t) => { + const playback = createPlaybackHarness(); + const { audio, sources, buffers } = playback; + t.after(() => audio.close()); + const decode = t.mock.method(globalThis, 'atob', globalThis.atob); + await audio.initialize(); + for (let index = 0; index < 256; index++) audio.play('AAA='); + const staleEnded = sources.map((source) => source.onended); + audio.clearPlayback(); + assert.equal(audio.pending, false); + assert.equal(audio.diagnostics().queuedSeconds, 0); + for (const source of sources) { + assert.equal(source.stopped, true); + assert.equal(source.disconnected, true); + assert.equal(source.onended, null); + } + for (let index = 0; index < 256; index++) audio.play('AAA='); + for (const ended of staleEnded) ended(); + assert.equal(audio.pending, true); + assert.throws(() => audio.play('AAA='), /playback fell behind/); + assert.equal(decode.mock.calls.length, 512); + assert.equal(buffers.length, 512); + assert.equal(sources.length, 512); + const lateEnded = sources.slice(256).map((source) => source.onended); + await audio.close(); + for (const source of sources) { + assert.equal(source.stopped, true); + assert.equal(source.disconnected, true); + assert.equal(source.onended, null); + } + for (const ended of [...staleEnded, ...lateEnded]) ended(); + audio.play('not base64'); + assert.equal(decode.mock.calls.length, 512); + assert.equal(buffers.length, 512); + assert.equal(sources.length, 512); + assert.equal(audio.pending, false); + assert.equal(playback.drains, 0); + assert.equal(playback.context.state, 'closed'); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 512, + completedOutputSamples: 0, + discardedOutputSamples: 512, + }); +}); + +test('the duration limit still rejects long chunks below the source cap', async (t) => { + const playback = createPlaybackHarness(); + const { audio, sources, buffers } = playback; + t.after(() => audio.close()); + await audio.initialize(); + const second = pcmBytesToBase64(new ArrayBuffer(16000)); + for (let index = 0; index < 59; index++) + audio.play(second, 'audio/pcm;rate=8000'); + assert.ok(Math.abs(audio.diagnostics().queuedSeconds - 59.02) < 1e-9); + assert.throws( + () => audio.play(second, 'audio/pcm;rate=8000'), + /playback fell behind/, + ); + assert.equal(buffers.length, 59); + assert.equal(sources.length, 59); +}); + +test('audio playback records completed samples only as sources naturally end', async () => { + const playback = createPlaybackHarness(); + const { audio, sources } = playback; + await audio.initialize(); + audio.play('AAA='); + audio.play('AAA='); + assert.ok(sources[1].at > sources[0].at); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 2, + completedOutputSamples: 0, + discardedOutputSamples: 0, + }); + sources[0].onended(); + assert.equal(playback.drains, 0); + assert.equal(audio.diagnostics().completedOutputSamples, 1); + sources[1].onended(); + assert.equal(playback.drains, 1); + assert.equal(audio.pending, false); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 2, + completedOutputSamples: 2, + discardedOutputSamples: 0, + }); + await audio.close(); + assert.equal(playback.context.state, 'closed'); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 2, + completedOutputSamples: 2, + discardedOutputSamples: 0, + }); +}); + +test('interruption records queued samples as discarded and rejects stale ended callbacks', async () => { + const playback = createPlaybackHarness(); + const { audio, sources } = playback; + await audio.initialize(); + audio.play('AAA='); + audio.play(pcmBytesToBase64(new ArrayBuffer(4))); + sources[0].onended(); + const staleEnded = sources[1].onended; + audio.clearPlayback(); + assert.equal(sources[1].stopped, true); + assert.equal(sources[1].onended, null); + assert.equal(audio.pending, false); + assert.equal(audio.diagnostics().queuedSeconds, 0); + staleEnded(); + audio.clearPlayback(); + assert.equal(playback.drains, 0); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 3, + completedOutputSamples: 1, + discardedOutputSamples: 2, + }); + await audio.close(); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 3, + completedOutputSamples: 1, + discardedOutputSamples: 2, + }); +}); + +test('closing queued playback cannot count stopped sources as completed audio', async () => { + const playback = createPlaybackHarness(); + const { audio, sources } = playback; + await audio.initialize(); + audio.play('AAA='); + audio.play(pcmBytesToBase64(new ArrayBuffer(4))); + const staleEnded = sources.map((source) => source.onended); + await audio.close(); + for (const source of sources) { + assert.equal(source.stopped, true); + assert.equal(source.onended, null); + } + for (const ended of staleEnded) ended(); + audio.play('AAA='); + await audio.close(); + assert.equal(audio.pending, false); + assert.equal(playback.drains, 0); + assert.equal(audio.diagnostics().contextState, 'closed'); + assert.equal(audio.diagnostics().queuedSeconds, 0); + assert.deepEqual(outputDiagnostics(audio), { + outputSamples: 3, + completedOutputSamples: 0, + discardedOutputSamples: 3, + }); +}); + +test('capture diagnostics distinguish speech energy from silence without retaining audio', async () => { + let capture; + const emitted = []; + const order = []; + const identities = []; + const gates = []; + const node = () => ({ connect() {}, disconnect() {}, gain: { value: 1 } }); + class Context { + constructor() { + this.currentTime = 0; + this.sampleRate = 16000; + this.state = 'running'; + this.destination = {}; + this.audioWorklet = { addModule: async () => {} }; + } + async resume() {} + async close() { + this.state = 'closed'; + } + createGain() { + return node(); + } + createMediaStreamDestination() { + return { ...node(), stream: { getTracks: () => [] } }; + } + createMediaStreamSource() { + return node(); + } + } + class Worklet { + constructor() { + capture = this; + Object.assign(this, node()); + this.port = { + postMessage(message) { + gates.push(message); + }, + close() {}, + }; + } + } + const audio = createGeminiAudio({ + AudioContextClass: Context, + AudioWorkletNodeClass: Worklet, + onAudioStart: (metadata) => { + identities.push(metadata.captureId); + order.push('start'); + }, + onAudio: (chunk, metadata) => { + identities.push(metadata.captureId); + emitted.push(chunk); + order.push('audio'); + }, + onAudioEnd: (metadata) => { + identities.push(metadata.captureId); + order.push('end'); + }, + }); + await audio.initialize(); + audio.connectMicrophone({}, true, { captureId: 11 }); + const deliver = capture.port.onmessage; + deliver({ data: { type: 'start', captureId: 11 } }); + capture.port.onmessage({ + data: { type: 'audio', captureId: 11, buffer: new ArrayBuffer(4) }, + }); + const spoken = new ArrayBuffer(4); + new DataView(spoken).setInt16(0, 16384, true); + capture.port.onmessage({ + data: { type: 'audio', captureId: 11, buffer: spoken }, + }); + deliver({ data: { type: 'end', captureId: 11 } }); + assert.deepEqual(order, ['start', 'audio', 'audio', 'end']); + assert.deepEqual(identities, [11, 11, 11, 11]); + assert.deepEqual(gates, [{ enabled: true, captureId: 11 }]); + // Gate D does not relabel delayed B port messages. + audio.setMicrophoneEnabled(false, { captureId: 11 }); + audio.setMicrophoneEnabled(true, { captureId: 12 }); + deliver({ data: { type: 'start', captureId: 11 } }); + deliver({ data: { type: 'audio', captureId: 11, buffer: spoken } }); + deliver({ data: { type: 'end', captureId: 11 } }); + assert.deepEqual(identities.slice(-3), [11, 11, 11]); + const count = order.length; + for (const captureId of [undefined, null, 0, -1, 1.5, '12', Infinity]) { + deliver({ data: { type: 'start', captureId } }); + deliver({ data: { type: 'audio', captureId, buffer: spoken } }); + deliver({ data: { type: 'end', captureId } }); + } + assert.equal(order.length, count); + assert.equal(audio.diagnostics().inputChunks, 3); + assert.equal(audio.diagnostics().nonSilentInputChunks, 2); + assert.equal(audio.diagnostics().inputPeak, 0.5); + assert.equal(emitted.length, 3); + assert.equal(emitted[1].mimeType, 'audio/pcm;rate=16000'); + assert.deepEqual(Object.keys(emitted[1]).sort(), ['data', 'mimeType']); + assert.equal(await audio.close(), true); + assert.equal(audio.diagnostics().contextState, 'closed'); + deliver({ data: { type: 'start', captureId: 11 } }); + deliver({ data: { type: 'audio', captureId: 11, buffer: spoken } }); + deliver({ data: { type: 'end', captureId: 11 } }); + assert.deepEqual(order, [ + 'start', + 'audio', + 'audio', + 'end', + 'start', + 'audio', + 'end', + ]); +}); + +test('worklet rejects missing identity and stale release without changing the active capture', () => { + const { capture, messages } = createCaptureHarness(); + for (const captureId of [undefined, null, 0, -1, 1.5, '1', Infinity]) + capture.port.onmessage({ data: { enabled: true, captureId } }); + capture.process([[new Float32Array([0.5])]]); + assert.equal(messages.length, 0); + capture.port.onmessage({ data: { enabled: true, captureId: 20 } }); + capture.process([[new Float32Array([0.5])]]); + capture.port.onmessage({ data: { enabled: true, captureId: 21 } }); + capture.port.onmessage({ data: { enabled: false, captureId: 19 } }); + assert.equal(messages.length, 1); + capture.port.onmessage({ data: { enabled: false, captureId: 20 } }); + assert.deepEqual( + messages.map((message) => [message.type, message.captureId]), + [ + ['start', 20], + ['audio', 20], + ['end', 20], + ], + ); + capture.port.onmessage({ data: { enabled: true, captureId: 21 } }); + capture.port.onmessage({ data: { enabled: false, captureId: 20 } }); + capture.process([[new Float32Array([0.25])]]); + capture.port.onmessage({ data: { enabled: false, captureId: 21 } }); + assert.deepEqual( + messages.slice(3).map((message) => [message.type, message.captureId]), + [ + ['start', 21], + ['audio', 21], + ['end', 21], + ], + ); +}); + +function capturedPcm(messages, captureId) { + const samples = []; + for (const message of messages) { + if (message.type !== 'audio' || message.captureId !== captureId) continue; + const view = new DataView(message.buffer); + for (let index = 0; index < view.byteLength; index += 2) + samples.push(view.getInt16(index, true)); + } + return samples; +} + +function captureSignal(rate, signal, blockSizes = [128]) { + const harness = createCaptureHarness(rate); + harness.capture.port.onmessage({ data: { enabled: true, captureId: 1 } }); + let block = 0; + for (let offset = 0; offset < signal.length;) { + const size = blockSizes[block++ % blockSizes.length]; + harness.capture.process([[signal.subarray(offset, offset + size)]]); + offset += size; + } + harness.capture.port.onmessage({ data: { enabled: false, captureId: 1 } }); + return { ...harness, pcm: capturedPcm(harness.messages, 1) }; +} + +for (const rate of [44100, 48000, 96000]) { + test(`worklet resamples ${rate} Hz to bounded 16 kHz PCM independently of block boundaries`, () => { + const signal = Float32Array.from( + { length: rate }, + (_, index) => 0.5 * Math.sin((2 * Math.PI * 1000 * index) / rate), + ); + const fragmented = captureSignal(rate, signal, [1, 127, 257, 3, 128]); + const contiguous = captureSignal(rate, signal, [rate]); + assert.equal(fragmented.pcm.length, 16000); + assert.deepEqual(fragmented.pcm, contiguous.pcm); + assert.deepEqual( + fragmented.messages.map((message) => message.type).slice(0, 2), + ['start', 'audio'], + ); + assert.equal(fragmented.messages.at(-1).type, 'end'); + assert.ok(fragmented.messages.every((message) => message.captureId === 1)); + assert.ok( + fragmented.messages + .filter((message) => message.type === 'audio') + .every((message) => message.buffer.byteLength <= 2048), + ); + const rms = Math.sqrt( + fragmented.pcm + .slice(256) + .reduce((total, sample) => total + (sample / 32768) ** 2, 0) / + (fragmented.pcm.length - 256), + ); + assert.ok( + Math.abs(rms - Math.SQRT1_2 * 0.5) < 0.01, + `speech-band RMS ${rms}`, + ); + // Check the actual waveform against its expected frequency and causal + // filter delay, rather than accepting any signal with the same energy. + const error = Math.sqrt( + fragmented.pcm.slice(256).reduce((total, sample, index) => { + const time = (index + 257) / 16000 - 32.5 / rate; + const expected = 0.5 * Math.sin(2 * Math.PI * 1000 * time); + return total + (sample / 32768 - expected) ** 2; + }, 0) / + (fragmented.pcm.length - 256), + ); + assert.ok(error < 0.005, `resampled waveform error ${error}`); + }); +} + +for (const rate of [44100, 48000]) { + test(`worklet suppresses 12 kHz aliasing from a ${rate} Hz microphone`, () => { + const signal = Float32Array.from( + { length: rate }, + (_, index) => 0.5 * Math.sin((2 * Math.PI * 12000 * index) / rate), + ); + const { pcm } = captureSignal(rate, signal); + const rms = Math.sqrt( + pcm + .slice(256) + .reduce((total, sample) => total + (sample / 32768) ** 2, 0) / + (pcm.length - 256), + ); + assert.ok(rms < 0.005, `out-of-band RMS ${rms}`); + }); +} + +test('resampler release flushes only owned PCM and clears phase/history before a successor', () => { + const { capture, messages } = createCaptureHarness(44100); + const gate = (enabled, captureId) => + capture.port.onmessage({ data: { enabled, captureId } }); + gate(true, 10); + capture.process([[new Float32Array(442).fill(1)]]); + gate(false, 9); // A stale release cannot clear the active filter or phase. + capture.process([[new Float32Array(441).fill(1)]]); + gate(false, 10); + assert.equal( + capturedPcm(messages, 10).length, + Math.floor((883 * 16000) / 44100), + ); + gate(true, 11); + gate(false, 10); + capture.process([[new Float32Array(441)]]); + gate(false, 11); + const successor = capturedPcm(messages, 11); + assert.equal(successor.length, 160); + assert.ok(successor.every((sample) => sample === 0)); + assert.deepEqual( + messages.slice(-3).map((message) => [message.type, message.captureId]), + [ + ['start', 11], + ['audio', 11], + ['end', 11], + ], + ); + // A sub-output-sample hold has no residual PCM or phase to transfer. + gate(true, 12); + capture.process([[new Float32Array([1])]]); + gate(false, 12); + gate(true, 13); + capture.process([[new Float32Array(441)]]); + gate(false, 13); + assert.deepEqual(capturedPcm(messages, 13), successor); + assert.deepEqual( + messages + .filter((message) => message.captureId === 12) + .map((message) => message.type), + ['start', 'end'], + ); +}); + +function createContextLifecycleHarness({ + resumeFailure, + moduleFailure = false, + closeFailure, + constructorFailure, + actualPlaybackRate = 24000, +} = {}) { + const contexts = []; + const worklets = []; + const events = []; + let stoppedTracks = 0; + const node = (context) => ({ + context, + gain: { value: 1 }, + connections: [], + connect(target) { + this.connections.push(target); + }, + disconnect() { + this.disconnected = true; + }, + }); + class Context { + constructor(options) { + const index = contexts.length; + if (constructorFailure === index) + throw new Error('context creation failed'); + this.index = index; + this.options = options; + this.sampleRate = index === 0 ? 44100 : actualPlaybackRate; + this.currentTime = index === 0 ? 99 : 7; + this.state = 'suspended'; + this.destination = { context: this }; + this.nodes = []; + this.closeCalls = 0; + this.audioWorklet = { + addModule: async () => { + events.push(`module:${index}`); + if (moduleFailure) throw new Error('module failed'); + }, + }; + contexts.push(this); + } + resume() { + events.push(`resume:${this.index}`); + if (resumeFailure === this.index) throw new Error('resume failed'); + this.state = 'running'; + return Promise.resolve(); + } + close() { + this.closeCalls++; + events.push(`close:${this.index}`); + if (closeFailure === this.index) throw new Error('close failed'); + this.state = 'closed'; + return Promise.resolve(); + } + createGain() { + const result = node(this); + this.nodes.push(result); + return result; + } + createMediaStreamDestination() { + const result = { + ...node(this), + stream: { + getTracks: () => [ + { + stop: () => { + stoppedTracks++; + }, + }, + ], + }, + }; + this.nodes.push(result); + return result; + } + createMediaStreamSource() { + const result = node(this); + this.nodes.push(result); + return result; + } + createBuffer(_channels, length, rate) { + return { + context: this, + rate, + duration: length / rate, + copyToChannel() {}, + }; + } + createBufferSource() { + const result = { + ...node(this), + start(at) { + this.at = at; + }, + stop() { + this.stopped = true; + }, + }; + this.nodes.push(result); + return result; + } + } + class Worklet { + constructor(context) { + Object.assign(this, node(context)); + this.gates = []; + this.port = { + postMessage: (message) => this.gates.push(message), + close: () => { + this.portClosed = true; + }, + }; + worklets.push(this); + } + } + return { + contexts, + worklets, + events, + create: (callbacks = {}) => + createGeminiAudio({ + AudioContextClass: Context, + AudioWorkletNodeClass: Worklet, + ...callbacks, + }), + get stoppedTracks() { + return stoppedTracks; + }, + }; +} + +test('capture uses the device context while playback owns its clock, stream and actual rate diagnostics', async () => { + const harness = createContextLifecycleHarness({ actualPlaybackRate: 48000 }); + const chunks = []; + const errors = []; + const audio = harness.create({ + onAudio: (chunk) => chunks.push(chunk), + onError: (error) => errors.push(error), + }); + assert.deepEqual(harness.events, ['resume:0', 'resume:1']); + assert.equal(harness.contexts[0].options, undefined); + assert.deepEqual(harness.contexts[1].options, { sampleRate: 24000 }); + await audio.initialize(); + assert.deepEqual(harness.events, ['resume:0', 'resume:1', 'module:0']); + audio.connectMicrophone({}, true, { captureId: 5 }); + const capture = harness.worklets[0]; + assert.equal(capture.context, harness.contexts[0]); + assert.deepEqual(capture.gates, [{ enabled: true, captureId: 5 }]); + const staleError = capture.onprocessorerror; + capture.port.onmessage({ + data: { type: 'audio', captureId: 5, buffer: new ArrayBuffer(4) }, + }); + assert.equal(chunks[0].mimeType, 'audio/pcm;rate=16000'); + audio.play('AAA='); + const source = harness.contexts[1].nodes.find((item) => item.buffer); + assert.equal(source.context, harness.contexts[1]); + assert.equal(source.buffer.context, harness.contexts[1]); + assert.equal(source.buffer.rate, 24000); + assert.equal(source.at, 7.02); + assert.ok(audio.diagnostics().queuedSeconds < 0.021); + assert.equal(audio.diagnostics().captureSampleRate, 44100); + assert.equal(audio.diagnostics().playbackSampleRate, 48000); + assert.equal(audio.diagnostics().inputSampleRate, 16000); + assert.equal(audio.diagnostics().contextState, 'running'); + harness.contexts[0].state = 'suspended'; + assert.equal(audio.diagnostics().contextState, 'suspended'); + const closing = audio.close(); + assert.equal(audio.close(), closing); + assert.equal(await closing, true); + assert.ok( + harness.contexts.every( + (context) => context.state === 'closed' && context.closeCalls === 1, + ), + ); + assert.equal(harness.stoppedTracks, 1); + assert.equal(capture.portClosed, true); + assert.equal(capture.disconnected, true); + assert.equal(capture.onprocessorerror, null); + assert.equal(source.stopped, true); + staleError(); + audio.setMicrophoneEnabled(true, { captureId: 6 }); + assert.equal(capture.gates.length, 1); + assert.deepEqual(errors, []); +}); + +for (const resumeFailure of [0, 1]) { + test(`resume failure in context ${resumeFailure} releases both contexts`, async () => { + const harness = createContextLifecycleHarness({ resumeFailure }); + const audio = harness.create(); + assert.deepEqual(harness.events, ['resume:0', 'resume:1']); + await assert.rejects(audio.initialize(), /resume failed/); + assert.ok( + harness.contexts.every( + (context) => context.state === 'closed' && context.closeCalls === 1, + ), + ); + assert.equal(harness.stoppedTracks, 1); + assert.equal(await audio.close(), true); + }); +} + +test('module failure and aborted initialization release both capture and playback', async () => { + for (const moduleFailure of [true, false]) { + const harness = createContextLifecycleHarness({ moduleFailure }); + const audio = harness.create(); + const signal = moduleFailure ? undefined : AbortSignal.abort(); + await assert.rejects( + audio.initialize(signal), + moduleFailure ? /module failed/ : { name: 'AbortError' }, + ); + assert.ok( + harness.contexts.every( + (context) => context.state === 'closed' && context.closeCalls === 1, + ), + ); + assert.equal(harness.stoppedTracks, 1); + } +}); + +for (const closeFailure of [0, 1]) { + test(`context ${closeFailure} close failure does not prevent closing the other context`, async () => { + const harness = createContextLifecycleHarness({ closeFailure }); + const audio = harness.create(); + await audio.initialize(); + const closing = audio.close(); + assert.equal(audio.close(), closing); + assert.equal(await closing, false); + assert.ok(harness.contexts.every((context) => context.closeCalls === 1)); + assert.equal(harness.contexts[1 - closeFailure].state, 'closed'); + assert.equal(harness.stoppedTracks, 1); + }); +} + +test('partial context construction failure closes the previously acquired capture context', async () => { + const harness = createContextLifecycleHarness({ constructorFailure: 1 }); + assert.throws(() => harness.create(), /context creation failed/); + assert.equal(harness.contexts.length, 1); + assert.equal(harness.contexts[0].closeCalls, 1); + assert.equal(harness.contexts[0].state, 'closed'); +}); + +function admissionHarness({ pushToTalk = true, callbacks = {} } = {}) { + const events = []; + const worklets = []; + const node = () => ({ connect() {}, disconnect() {}, gain: { value: 1 } }); + class Context { + constructor() { + this.sampleRate = 16000; + this.currentTime = 0; + this.state = 'running'; + this.destination = {}; + this.audioWorklet = { addModule: async () => {} }; + } + async resume() {} + async close() { + this.state = 'closed'; + } + createGain() { + return node(); + } + createMediaStreamDestination() { + return { ...node(), stream: { getTracks: () => [] } }; + } + createMediaStreamSource() { + return node(); + } + createBuffer(_channels, samples, rate) { + return { duration: samples / rate, copyToChannel() {} }; + } + createBufferSource() { + return { ...node(), start() {}, stop() {} }; + } + } + class Worklet { + constructor() { + Object.assign(this, node()); + worklets.push(this); + const { capture, messages } = createCaptureHarness(); + this.processor = capture; + this.flush = () => { + for (const data of messages.splice(0)) { + // MessagePort structured cloning moves VM buffers into this realm. + if (data.buffer) + data.buffer = new Uint8Array(data.buffer).slice().buffer; + this.port.onmessage?.({ data }); + } + }; + this.port = { + postMessage: (data) => { + capture.port.onmessage({ data }); + this.flush(); + }, + close() {}, + }; + } + feed(samples) { + this.processor.process([[samples]]); + this.flush(); + } + } + const createAudio = (options) => + createGeminiAudio({ + ...options, + AudioContextClass: Context, + AudioWorkletNodeClass: Worklet, + }); + const audio = createAudio({ + pushToTalk, + onAudioStart: (metadata) => { + events.push({ type: 'start', ...metadata }); + callbacks.onAudioStart?.(metadata); + }, + onAudio: (chunk, metadata) => { + events.push({ type: 'audio', ...metadata, chunk }); + callbacks.onAudio?.(chunk, metadata); + }, + onAudioEnd: (metadata) => { + events.push({ type: 'end', ...metadata }); + callbacks.onAudioEnd?.(metadata); + }, + onAudioDiscard: (metadata) => { + events.push({ type: 'discard', ...metadata }); + callbacks.onAudioDiscard?.(metadata); + }, + onError: (error) => { + throw error; + }, + }); + return { audio, events, worklets, createAudio }; +} +function tone(length, amplitude = 0.04) { + return Float32Array.from( + { length }, + (_, i) => amplitude * Math.sin((i * 2 * Math.PI * 220) / 16000), + ); +} +function emittedPcm(events) { + return events + .filter((e) => e.type === 'audio') + .flatMap((e) => { + const bytes = Uint8Array.from(atob(e.chunk.data), (c) => c.charCodeAt(0)); + const view = new DataView(bytes.buffer); + return Array.from({ length: bytes.length / 2 }, (_, i) => + view.getInt16(i * 2, true), + ); + }); +} + +test('PTT admission suppresses zero and near-silent captures without protocol callbacks', async (t) => { + const h = admissionHarness(); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + h.worklets[0].feed(new Float32Array(1024)); + h.worklets[0].feed(tone(320, 0.00018)); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + assert.deepEqual(h.events, [{ type: 'discard', captureId: 1 }]); + assert.equal(h.audio.diagnostics().suppressedPttCaptures, 1); + assert.equal(h.audio.diagnostics().pttBufferedBytes, 0); +}); + +test('PTT admission rejects isolated high-peak clicks across packet boundaries', async (t) => { + const h = admissionHarness(); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + const click = new Float32Array(1024); + click[159] = 1; + click[160] = -1; + h.worklets[0].feed(click); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + assert.deepEqual( + h.events.map((e) => e.type), + ['discard'], + ); +}); + +test('PTT admission preserves quiet short speech and final partial release in exact order', async (t) => { + const h = admissionHarness(); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + const prefix = new Float32Array(73); + const speech = tone(640, 0.005); + const tail = new Float32Array(57); + h.worklets[0].feed(prefix); + h.worklets[0].feed(speech); + h.worklets[0].feed(tail); + assert.equal(h.events.length, 0, 'worklet still owns its incomplete packet'); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + assert.deepEqual( + h.events.map((e) => e.type), + ['start', 'audio', 'end'], + ); + const expected = [...prefix, ...speech, ...tail].map( + (v) => Math.round(v * (v < 0 ? 32768 : 32767)) || 0, + ); + assert.deepEqual(emittedPcm(h.events), expected); +}); + +test('PTT admission bounds a long silent preroll and never regates voiced tails', async (t) => { + const h = admissionHarness(); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + for (let i = 0; i < 200; i++) { + h.worklets[0].feed(new Float32Array(1024)); + assert.ok(h.audio.diagnostics().pttBufferedBytes <= 12800); + } + assert.equal(h.events.length, 0); + h.worklets[0].feed(tone(1024)); + h.worklets[0].feed(new Float32Array(1024)); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + assert.equal(h.events[0].type, 'start'); + assert.equal(h.events.at(-1).type, 'end'); + assert.equal(h.events.filter((e) => e.type === 'start').length, 1); + const pcm = emittedPcm(h.events); + assert.ok(pcm.length <= 9 * 1024); + assert.deepEqual(pcm.slice(-1024), new Array(1024).fill(0)); + assert.equal(h.audio.diagnostics().admittedPttCaptures, 1); +}); + +test('PTT admission isolates repeated holds, stale identities and close callbacks', async (t) => { + const h = admissionHarness(); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + h.worklets[0].feed(tone(160)); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + h.audio.setMicrophoneEnabled(true, { captureId: 2 }); + const deliver = h.worklets[0].port.onmessage; + deliver({ + data: { type: 'audio', captureId: 1, buffer: new ArrayBuffer(2048) }, + }); + deliver({ data: { type: 'end', captureId: 1 } }); + h.worklets[0].feed(tone(160)); + h.audio.setMicrophoneEnabled(false, { captureId: 2 }); + assert.deepEqual( + h.events.map((e) => e.type), + ['discard', 'discard'], + ); + h.audio.setMicrophoneEnabled(true, { captureId: 3 }); + h.worklets[0].feed(new Float32Array(1024)); + await h.audio.close(); + deliver({ data: { type: 'audio', captureId: 3, buffer: tone(1024).buffer } }); + deliver({ data: { type: 'end', captureId: 3 } }); + assert.equal(h.events.length, 2); + assert.equal(h.audio.diagnostics().pttBufferedBytes, 0); +}); + +test('PTT admission leaves open-mic start, silence, PCM and end unchanged', async (t) => { + const h = admissionHarness({ pushToTalk: false }); + t.after(() => h.audio.close()); + await h.audio.initialize(); + h.audio.connectMicrophone({}, true, { captureId: 1 }); + h.worklets[0].feed(new Float32Array(1024)); + h.audio.setMicrophoneEnabled(false, { captureId: 1 }); + assert.deepEqual( + h.events.map((e) => e.type), + ['start', 'audio', 'end'], + ); + assert.deepEqual(emittedPcm(h.events), new Array(1024).fill(0)); +}); diff --git a/src/voice/geminiAutomaticVad.test.mjs b/src/voice/geminiAutomaticVad.test.mjs new file mode 100644 index 000000000..1a6f8af61 --- /dev/null +++ b/src/voice/geminiAutomaticVad.test.mjs @@ -0,0 +1,791 @@ +// Native automatic activity, reply ownership and expiry lifecycle regressions. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + harness, + tick, + deferred, +} from '../testSupport/geminiNativeHarness.mjs'; +const vad = (ws, type) => ws.message({ voiceActivity: { type } }); +const onset = (ws) => { + vad(ws, 'ACTIVITY_START'); + vad(ws, 'ACTIVITY_END'); +}; +const split = (ws) => { + onset(ws); + onset(ws); +}; +const audio = { + serverContent: { + modelTurn: { + parts: [ + { inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } }, + ], + }, + }, +}; +const done = (ws) => ws.message({ serverContent: { turnComplete: true } }); +const tools = (ws, id) => + ws.message({ + toolCall: { + functionCalls: [ + { + id, + name: 'adjust_camera_zoom', + args: { direction: 'out', amount: 'little' }, + }, + ], + }, + }); +const responses = (ws) => + ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []); +function close(h) { + h.c.stop(); + assert.equal(h.timers.size, 0, 'owned timers released'); +} +test('N1 VAD-only split admits the first real answer and completes after playback', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + split(ws); + ws.message(audio); + ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } }); + done(ws); + assert.equal(h.captures[0].played.length, 1); + h.captures[0].drain(); + assert.equal( + h.events.findLast((e) => e.type === 'completion')?.status, + 'completed', + ); + assert.equal(h.c.retiredReplyPending, false); + } finally { + close(h); + } +}); +test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => { + const h = harness(); + try { + await h.c.start(); + split(h.sockets[0]); + h.advance(5001); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.retiredReplyPending, false); + assert.equal(h.c.retiredBoundaryWait, null); + } finally { + close(h); + } +}); +test('N3 server interruption before output still fences until ordered turnComplete', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + split(ws); + ws.message({ serverContent: { interrupted: true } }); + assert.equal(h.c.retiredReplyPending, true); + ws.message(audio); + tools(ws, 'retired-before-boundary'); + await h.c.toolQueue; + assert.equal(h.captures[0].played.length, 0); + assert.equal(h.calls.length, 0); + assert.equal(responses(ws).length, 0); + ws.message({ + serverContent: { ...audio.serverContent, turnComplete: true }, + }); + assert.equal( + h.captures[0].played.length, + 0, + 'coalesced retired content is discarded', + ); + ws.message(audio); + assert.equal(h.captures[0].played.length, 1); + done(ws); + h.captures[0].drain(); + assert.equal( + h.events.findLast((e) => e.type === 'completion')?.status, + 'completed', + ); + } finally { + close(h); + } +}); +test('N4 generationComplete is server evidence and next onset retires it', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message({ serverContent: { generationComplete: true } }); + onset(ws); + assert.equal(h.c.retiredReplyPending, true); + assert.ok(h.c.retiredBoundaryWait); + ws.message(audio); + tools(ws, 'old-generation'); + await h.c.toolQueue; + assert.equal(h.captures[0].played.length, 0); + assert.equal(h.calls.length, 0); + done(ws); + assert.equal(h.c.retiredReplyPending, false); + } finally { + close(h); + } +}); +test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + split(ws); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + assert.ok(h.c.goAway); + assert.equal(ws.closed, undefined); + h.advance(5001); + assert.equal(h.c.isActive(), true); + ws.message(audio); + done(ws); + assert.equal( + h.c.isActive(), + true, + 'pending actual PCM drains before expiry', + ); + h.captures[0].drain(); + assert.equal(h.c.status, 'error'); + assert.equal(ws.closed, true); + } finally { + close(h); + } +}); +test('N6 first server tool after VAD-only split executes exactly once', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + split(ws); + tools(ws, 'new-generation'); + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal( + responses(ws).filter((x) => x.id === 'new-generation').length, + 1, + ); + assert.equal(responses(ws).at(-1).response.ok, true); + tools(ws, 'new-generation'); + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal( + responses(ws).filter((x) => x.id === 'new-generation').length, + 1, + ); + } finally { + close(h); + } +}); +test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(audio); + onset(ws); + assert.equal(h.c.retiredReplyPending, true); + ws.message(audio); + tools(ws, 'late-old'); + await h.c.toolQueue; + assert.equal(h.captures[0].played.length, 1); + assert.equal(h.calls.length, 0); + ws.message({ serverContent: { interrupted: true } }); + ws.message(audio); + assert.equal(h.captures[0].played.length, 1); + done(ws); + ws.message(audio); + assert.equal(h.captures[0].played.length, 2); + } finally { + close(h); + } +}); +test('C2 first server tool in flight is aborted and never answered after new onset', async () => { + const gate = deferred(), + entered = []; + const h = harness({ + runAction: async (_n, _a, o) => { + entered.push(o); + await gate.promise; + return { ok: true }; + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + tools(ws, 'old-running'); + await tick(); + assert.equal(entered.length, 1); + onset(ws); + assert.equal(entered[0].signal.aborted, true); + gate.resolve(); + await h.c.toolQueue; + assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0); + assert.equal(h.c.retiredReplyPending, true); + } finally { + gate.resolve(); + close(h); + } +}); +test('C5 typed request before first server output still retires on next onset', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + assert.equal(h.c.sendTextCommand('Zoom in'), true); + onset(ws); + assert.equal(h.c.retiredReplyPending, true); + ws.message(audio); + tools(ws, 'late-typed'); + await h.c.toolQueue; + assert.equal(h.captures[0].played.length, 0); + assert.equal(h.calls.length, 0); + } finally { + close(h); + } +}); +test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + const oldOwner = h.c.automaticActivityOwner; + ws.message(audio); + onset(ws); + const successor = h.c.automaticActivityOwner; + assert.notEqual(successor, oldOwner); + assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.automaticActivityOwner, successor); + assert.equal(h.c.automaticTurnOpen, true); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + ws.message(audio); + done(ws); + h.captures[0].drain(); + assert.equal(h.c.status, 'error'); + } finally { + close(h); + } +}); +test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + const owner = h.c.automaticActivityOwner; + ws.message(audio); + ws.message({ serverContent: { interrupted: true } }); + assert.equal(h.c.retiredAutomaticActivityOwner, owner); + done(ws); + assert.equal(h.c.automaticTurnOpen, false); + assert.equal(h.c.automaticActivityOwner, null); + assert.equal(h.c.retiredAutomaticActivityOwner, null); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), false, 'no invented successor wait'); + } finally { + close(h); + } +}); +test('N9 coalesced old completion and new VAD preserves the new owner', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + vad(ws, 'ACTIVITY_END'); + ws.message({ + voiceActivity: { type: 'ACTIVITY_START' }, + serverContent: { interrupted: true, turnComplete: true }, + }); + assert.equal(h.c.automaticTurnOpen, true); + assert.ok(h.c.automaticActivityOwner); + assert.equal(h.c.retiredAutomaticActivityOwner, null); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + } finally { + close(h); + } +}); +test('N10 stop, restart and destroy release automatic owner identities and timers', async () => { + const h = harness(); + try { + await h.c.start(); + onset(h.sockets[0]); + const first = h.c.automaticActivityOwner; + h.c.stop(); + assert.equal(h.c.automaticTurnOpen, false); + assert.equal(h.c.automaticActivityOwner, null); + assert.equal(h.c.retiredAutomaticActivityOwner, null); + assert.equal(h.c.automaticActivitySequence, 0); + await h.c.start(); + onset(h.sockets[1]); + assert.notEqual(h.c.automaticActivityOwner, first); + assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch); + h.c.stop({ removeUi: true }); + assert.equal(h.c.disposed, true); + assert.equal(h.c.automaticTurnOpen, false); + assert.equal(h.c.automaticActivityOwner, null); + assert.equal(h.c.retiredAutomaticActivityOwner, null); + assert.equal(h.timers.size, 0); + } finally { + close(h); + } +}); +test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(audio); + assert.equal(h.c.sendTextCommand('Zoom in'), true); + assert.equal(h.c.sendTextCommand('Zoom out'), true); + assert.equal(h.c.deferredInput.kind, 'user-text'); + const before = ws.sent.filter((x) => x.realtimeInput?.audio).length; + const metadata = { captureId: h.c.captureOwner.captureId }; + h.captures[0].callbacks.onAudio( + { data: 'AAAA', mimeType: 'audio/pcm;rate=16000' }, + metadata, + ); + assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before); + assert.equal( + h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio), + false, + ); + assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1); + assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, { + nonPttOwner: 1, + closedActivity: 0, + }); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + done(ws); + assert.equal( + ws.sent.filter((x) => x.realtimeInput?.audio).length, + before, + 'no wrong-owner PCM replay after boundaries', + ); + } finally { + close(h); + } +}); +test('N12 goAway before old completion still waits for the newer automatic owner', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + onset(ws); + const successor = h.c.automaticActivityOwner; + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.automaticActivityOwner, successor); + assert.equal( + h.c.isActive(), + true, + 'old boundary during goAway cannot end successor wait', + ); + ws.message(audio); + done(ws); + h.captures[0].drain(); + assert.equal(h.c.status, 'error'); + } finally { + close(h); + } +}); +for (const interruptedBeforeExpiry of [false, true]) + test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + if (interruptedBeforeExpiry) + ws.message({ serverContent: { interrupted: true } }); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.automaticTurnOpen, false); + assert.equal(h.c.automaticActivityOwner, null); + assert.equal( + h.c.isActive(), + false, + 'no successor finishes at its actual boundary', + ); + } finally { + close(h); + } + }); +test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => { + const gate = deferred(), + entered = []; + const h = harness({ + runAction: async (_n, _a, o) => { + entered.push(o); + await gate.promise; + return { ok: true }; + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + tools(ws, 'expiring-old-tool'); + await tick(); + assert.equal(entered.length, 1); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(entered[0].signal.aborted, true); + gate.resolve(); + await h.c.toolQueue; + assert.equal( + responses(ws).filter((x) => x.id === 'expiring-old-tool').length, + 0, + ); + assert.equal(h.c.isActive(), false); + } finally { + gate.resolve(); + close(h); + } +}); +test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + ws.message({ goAway: { timeLeft: '30s' } }); + const turn = h.c.turn, + sent = ws.sent.length; + assert.equal(h.c.bargeIn(), false); + assert.equal(h.c.sendTextCommand('Zoom out'), false); + h.c.interrupt('push-to-talk'); + assert.equal(h.c.turn, turn); + assert.equal(ws.sent.length, sent); + assert.equal( + h.tracks.every((t) => !t.enabled), + true, + ); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.isActive(), false); + assert.equal( + h.tracks.every((t) => t.readyState === 'ended'), + true, + ); + } finally { + close(h); + } +}); + +test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => { + const h = harness(); + try { + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + a = h.captures[0]; + ws.message(audio); + h.c.input.pushToTalkKeyHeld = true; + assert.equal(h.c.input.bargeIn(), true); + h.c.input.setMicrophoneEnabled(true); + a.callbacks.onAudio( + { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, + a.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + assert.equal(h.c.replyPending, true); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal( + h.c.replyPending, + true, + 'old boundary cannot clear submitted successor', + ); + assert.equal(h.c.sendTextCommand('Zoom out second request'), true); + assert.ok(h.c.currentTypedInput()); + const played = a.played.length; + ws.message(audio); + tools(ws, 'late-ptt-predecessor'); + await h.c.toolQueue; + assert.equal(a.played.length, played); + assert.equal( + h.calls.length, + 0, + 'no old tool under newer local typed input', + ); + done(ws); + tools(ws, 'current-typed'); + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal(h.calls[0].options.userTurn, undefined); + assert.equal(h.calls[0].options.isCurrent(), true); + assert.equal(h.c.currentTypedInput().text, 'Zoom out second request'); + } finally { + close(h); + } +}); +for (const expiryBeforeBoundary of [false, true]) + test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(audio); + onset(ws); + assert.equal(h.c.sendTextCommand('Zoom out successor'), true); + const typed = h.c.currentTypedInput(); + assert.ok(typed); + assert.equal(h.c.replyPending, true); + if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } }); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.replyPending, true); + if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true, 'submitted text must finish'); + tools(ws, 'valid-successor'); + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal(h.calls[0].options.userTurn, undefined); + assert.equal(h.calls[0].options.isCurrent(), true); + assert.equal(h.c.currentTypedInput(), typed); + ws.message(audio); + done(ws); + assert.equal(h.c.isActive(), true); + h.captures[0].drain(); + assert.equal(h.c.isActive(), false); + } finally { + close(h); + } + }); +for (const repeat of [false, true]) + test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(audio); + assert.equal(h.c.sendTextCommand('Zoom out A'), true); + assert.equal(h.c.replyPending, true); + assert.equal(h.c.sendTextCommand('Zoom out B'), true); + assert.equal(h.c.deferredInput.kind, 'user-text'); + assert.ok(h.c.currentTypedInput()); + onset(ws); + assert.equal(h.c.automaticTurnOpen, true); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.deferredInput, null); + if (repeat) { + ws.message({ goAway: { timeLeft: '20s' } }); + ws.message({ goAway: { timeLeft: '10s' } }); + } + assert.equal(h.c.goAway.supersededSuccessor, true); + assert.equal(h.c.isActive(), true); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + tools(ws, 'superseded-A'); + await h.c.toolQueue; + assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority'); + assert.equal( + h.c.isActive(), + false, + 'ordered boundary closes superseded forwarded predecessor', + ); + } finally { + close(h); + } + }); +test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => { + const h = harness(); + try { + await h.c.start(); + onset(h.sockets[0]); + h.sockets[0].message({ goAway: { timeLeft: '30s' } }); + h.advance(29999); + assert.equal(h.c.isActive(), true); + h.advance(1); + assert.equal(h.c.isActive(), false); + assert.equal(h.timers.size, 0); + assert.equal( + h.tracks.every((t) => t.readyState === 'ended'), + true, + ); + } finally { + close(h); + } +}); +test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + ws.message({ + voiceActivity: { type: 'ACTIVITY_START' }, + serverContent: { turnComplete: true }, + }); + assert.equal(h.c.retiredReplyPending, false); + assert.equal(h.c.automaticTurnOpen, true); + ws.message({ goAway: { timeLeft: '30s' } }); + assert.equal(h.c.isActive(), true); + } finally { + close(h); + } +}); +test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message({ serverContent: { generationComplete: true } }); + assert.equal(h.c.replyPending, true); + done(ws); + assert.equal( + h.events.findLast((e) => e.type === 'completion')?.status, + 'completed', + ); + assert.equal(h.c.replyPending, false); + assert.equal(h.c.automaticTurnOpen, false); + assert.equal(h.c.retiredBoundaryWait, null); + assert.equal(h.timers.size, 0); + } finally { + close(h); + } +}); + +test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => { + const h = harness(); + try { + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + a = h.captures[0]; + h.c.input.setMicrophoneEnabled(true); + a.callbacks.onAudio( + { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, + a.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + assert.equal(h.c.userTurnPending, true); + ws.message({ serverContent: { generationComplete: true } }); + done(ws); + const event = h.events.findLast((e) => e.type === 'completion'); + assert.equal(event.status, 'completed'); + assert.equal(event.spoken, false); + assert.equal(a.played.length, 0); + assert.equal(h.c.userTurnPending, false); + } finally { + close(h); + } +}); +test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => { + const gate = deferred(), + entered = []; + const h = harness({ + runAction: async (_n, _a, o) => { + entered.push(o); + await gate.promise; + return { ok: true }; + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + tools(ws, 'pending-terminal-tool'); + await tick(); + assert.equal(entered.length, 1); + done(ws); + assert.equal( + h.events.some((e) => e.type === 'completion'), + false, + ); + assert.equal(h.c.userTurnPending, false); + gate.resolve(); + await h.c.toolQueue; + h.c.finishPlayback(); + assert.equal( + h.events.some((e) => e.type === 'completion'), + false, + ); + assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up'); + done(ws); + assert.equal( + h.events.findLast((e) => e.type === 'completion')?.status, + 'completed', + ); + } finally { + gate.resolve(); + close(h); + } +}); +test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + ws.message(audio); + ws.message({ serverContent: { generationComplete: true } }); + done(ws); + assert.equal( + h.events.some((e) => e.type === 'completion'), + false, + ); + h.captures[0].drain(); + const event = h.events.findLast((e) => e.type === 'completion'); + assert.equal(event.status, 'completed'); + assert.equal(event.spoken, true); + } finally { + close(h); + } +}); +test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(audio); + assert.equal(h.c.sendTextCommand('Zoom out successor'), true); + assert.equal(h.c.userTurnPending, true); + done(ws); + assert.equal(h.c.userTurnPending, true); + assert.equal( + h.events.some((e) => e.type === 'completion'), + false, + ); + } finally { + close(h); + } +}); +test('T-C7 reentrant silent completion observer keeps new input pending', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + const emit = h.c.emit.bind(h.c); + h.c.emit = (event) => { + emit(event); + if (event.type === 'completion') + h.c.sendTextCommand('Zoom out reentrant'); + }; + onset(ws); + ws.message({ serverContent: { generationComplete: true } }); + done(ws); + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + assert.equal(h.c.replyPending, true); + assert.equal(h.c.userTurnPending, true); + assert.ok(h.c.currentTypedInput()); + assert.equal(h.c.modelComplete, false); + } finally { + close(h); + } +}); diff --git a/src/voice/geminiCaptureWorklet.js b/src/voice/geminiCaptureWorklet.js new file mode 100644 index 000000000..f52cf1546 --- /dev/null +++ b/src/voice/geminiCaptureWorklet.js @@ -0,0 +1,125 @@ +const OUTPUT_RATE = 16000; +const FILTER_TAPS = 64; +const FILTER_PHASES = 128; + +// A causal, windowed-sinc low-pass filter avoids folding device frequencies +// above 8 kHz into speech. Precompute phases; process allocates no filter data. +function resamplingFilters(inputRate) { + const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate); + const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES); + for (let phase = 0; phase < FILTER_PHASES; phase++) { + let total = 0; + for (let tap = 0; tap < FILTER_TAPS; tap++) { + const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES; + const sinc = + distance === 0 + ? 2 * cutoff + : Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance); + const window = + 0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1)); + const weight = sinc * window; + filters[phase * FILTER_TAPS + tap] = weight; + total += weight; + } + for (let tap = 0; tap < FILTER_TAPS; tap++) + filters[phase * FILTER_TAPS + tap] /= total; + } + return filters; +} + +/** PCM microphone capture. Only explicitly enabled frames leave this processor. */ +class GeminiCaptureProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.enabled = false; + this.captureId = null; + this.samples = new Int16Array(1024); + this.length = 0; + this.inputRate = sampleRate; + this.filters = + this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate); + this.history = new Float32Array(FILTER_TAPS); + this.resetResampler(); + this.port.onmessage = ({ data }) => { + if (data?.enabled === true) { + if ( + this.enabled || + !Number.isSafeInteger(data.captureId) || + data.captureId <= 0 + ) + return; + this.captureId = data.captureId; + this.resetResampler(); + this.enabled = true; + this.port.postMessage({ type: 'start', captureId: this.captureId }); + } else if ( + data?.enabled === false && + this.enabled && + data.captureId === this.captureId + ) { + this.flush(); + this.port.postMessage({ type: 'end', captureId: this.captureId }); + this.enabled = false; + this.captureId = null; + this.length = 0; + // Do not drain filter tails into a later physical microphone capture. + this.resetResampler(); + } + }; + } + resetResampler() { + this.phase = 0; + this.historyIndex = 0; + this.history.fill(0); + } + appendSample(sample) { + const value = Math.max(-1, Math.min(1, sample)); + this.samples[this.length++] = Math.round( + value * (value < 0 ? 32768 : 32767), + ); + if (this.length === this.samples.length) this.flush(); + } + flush() { + if (!this.length) return; + const buffer = new ArrayBuffer(this.length * 2); + const view = new DataView(buffer); + for (let i = 0; i < this.length; i++) + view.setInt16(i * 2, this.samples[i], true); + this.port.postMessage( + { type: 'audio', buffer, captureId: this.captureId }, + [buffer], + ); + this.length = 0; + } + process(inputs) { + if (!this.enabled) return true; + const channel = inputs[0]?.[0]; + if (!channel) return true; + for (const sample of channel) { + if (!this.filters) { + this.appendSample(sample); + continue; + } + this.history[this.historyIndex] = sample; + this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS; + this.phase += OUTPUT_RATE; + while (this.phase >= this.inputRate) { + this.phase -= this.inputRate; + const phase = Math.min( + FILTER_PHASES - 1, + Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE), + ); + let value = 0; + for (let tap = 0; tap < FILTER_TAPS; tap++) { + const index = + (this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS; + value += + this.history[index] * this.filters[phase * FILTER_TAPS + tap]; + } + this.appendSample(value); + } + } + return true; + } +} +registerProcessor('gev-gemini-capture', GeminiCaptureProcessor); diff --git a/src/voice/geminiDeferredInput.test.mjs b/src/voice/geminiDeferredInput.test.mjs new file mode 100644 index 000000000..e22c32e3d --- /dev/null +++ b/src/voice/geminiDeferredInput.test.mjs @@ -0,0 +1,285 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + nativeIntegrationFixture, + assertSuccessfulZoom, +} from '../testSupport/geminiIntegrationFixture.mjs'; + +const options = { concurrency: false, timeout: 5000 }; +const OLD_PCM = 'BAAFAA=='; +const TEXT = 'Zoom in for the owned typed successor'; + +function exposeCapture(h) { + const createAudio = h.controller.createAudio; + h.controller.createAudio = (callbacks) => { + const audio = createAudio(callbacks); + audio.captureCallbacks = callbacks; + return audio; + }; +} + +async function startManual(h) { + await h.session.start({ pushToTalk: true }); + assert.equal(h.session.state, 'listening'); + assert.equal(h.controller.input.pushToTalkMode, true); + assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false); + return h.sockets.at(-1); +} + +// Explicit fake worklet callbacks retain the native physical-capture path; +// this helper does not populate controller queues or ownership fields itself. +function speakAndRelease(h, data) { + const c = h.controller; + const audio = h.audios.at(-1); + c.input.pushToTalkKeyHeld = true; + c.input.setMicrophoneEnabled(true); + const metadata = audio.metadata; + assert.ok(Number.isSafeInteger(metadata?.captureId)); + audio.captureCallbacks.onAudioStart(metadata); + audio.captureCallbacks.onAudio( + { data, mimeType: 'audio/pcm;rate=16000' }, + metadata, + ); + c.input.pushToTalkKeyHeld = false; + c.input.setMicrophoneEnabled(false); + audio.captureCallbacks.onAudioEnd(metadata); + return metadata; +} + +async function prepareDeferredPtt(h) { + exposeCapture(h); + const socket = await startManual(h); + speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply. + speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A. + const wireBeforeC = socket.sent.length; + const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary. + const owner = h.controller.deferredInput; + assert.equal(owner?.kind, 'push-to-talk'); + assert.equal(owner.captureId, metadata.captureId); + assert.equal( + socket.sent.length, + wireBeforeC, + 'C must still be wholly deferred', + ); + assert.deepEqual( + owner.messages.map((message) => + message.realtimeInput?.activityStart + ? 'start' + : message.realtimeInput?.audio + ? 'audio' + : message.realtimeInput?.activityEnd + ? 'end' + : 'other', + ), + ['start', 'audio', 'end'], + ); + return { socket, owner, metadata }; +} + +async function assertHealthySuccessor(h, socket, callId) { + const beforeHeight = h.state.height; + await h.tool(callId, socket); + assertSuccessfulZoom(h, socket, callId, beforeHeight); + assert.equal(h.controller.pendingTools, 0); + h.complete(socket); // Native post-tool response boundary, with no output PCM. + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.session.state, 'listening'); +} + +for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) { + test( + `F04-5 deferred PTT flush loses its owner on ${mode}`, + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const { socket: oldSocket, owner } = await prepareDeferredPtt(h); + const oldAudio = h.audios.at(-1); + let observed = 0; + let sentAtObserverReturn; + let restart; + let stateAtObserverReturn; + const unsubscribe = h.session.subscribe((event) => { + if ( + event.type !== 'transcript' || + event.role !== 'user' || + event.text !== '' || + observed + ) + return; + observed++; + if (mode === 'typed replacement') { + assert.equal(h.session.sendText(TEXT), true); + } else { + h.session.stop(); + if (mode === 'Stop+restart') + restart = h.session.start({ pushToTalk: true }); + } + sentAtObserverReturn = oldSocket.sent.length; + stateAtObserverReturn = { + epoch: h.controller.epoch, + turn: h.controller.turn, + replyPending: h.controller.replyPending, + forwardedActivityOpen: h.controller.forwardedActivityOpen, + }; + }); + t.after(unsubscribe); + + h.complete(oldSocket); // A's boundary starts flushing C. C's start emits. + assert.equal( + observed, + 1, + 'must enter the actual flush/onset observer boundary', + ); + assert.ok( + stateAtObserverReturn, + 'observer must complete its replacement action', + ); + assert.deepEqual( + oldSocket.sent.slice(sentAtObserverReturn), + [], + 'retired C must send neither queued PCM nor an extra end after the observer returns', + ); + assert.deepEqual( + { + epoch: h.controller.epoch, + turn: h.controller.turn, + replyPending: h.controller.replyPending, + forwardedActivityOpen: h.controller.forwardedActivityOpen, + }, + stateAtObserverReturn, + 'old flush must not write successor state', + ); + assert.equal(h.controller.deferredInput, null); + assert.equal( + h.controller.typedIntent?.text ?? null, + mode === 'typed replacement' ? TEXT : null, + ); + + let successorSocket; + if (mode === 'typed replacement') { + assert.equal(h.controller.epoch, owner.epoch); + assert.notEqual(h.controller.turn, owner.turn); + assert.equal( + oldSocket.sent.some( + (message) => message.realtimeInput?.audio?.data === OLD_PCM, + ), + false, + ); + const typedIndex = oldSocket.sent.findIndex((message) => + message.clientContent?.turns?.some((turn) => + turn.parts?.some((part) => part.text === TEXT), + ), + ); + assert.ok( + typedIndex >= 0, + 'owned replacement must actually reach transport', + ); + // The replacement legitimately closes the already-forwarded C start once. + assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd); + h.complete(oldSocket); // B's remaining retired boundary, deliberately separate. + assert.equal(h.controller.retiredReplyPending, false); + assert.equal(h.controller.replyPending, true); + successorSocket = oldSocket; + } else { + assert.equal(oldAudio.closed, true); + assert.equal(oldSocket.closed, true); + if (!restart) restart = h.session.start({ pushToTalk: true }); + await restart; + successorSocket = h.sockets.at(-1); + assert.notEqual(successorSocket, oldSocket); + assert.equal( + successorSocket.sent.some((message) => message.realtimeInput?.audio), + false, + ); + assert.equal( + successorSocket.sent.some( + (message) => message.realtimeInput?.activityEnd, + ), + false, + ); + assert.equal(h.session.sendText(TEXT), true); + } + await assertHealthySuccessor( + h, + successorSocket, + `healthy-after-deferred-${mode}`, + ); + }, + ); +} + +test( + 'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const oldSocket = await h.start(); + assert.equal(h.session.sendText('A'), true); + assert.equal(h.session.sendText('B'), true); + assert.equal(h.session.sendText('C deferred text'), true); + assert.equal(h.controller.deferredInput?.kind, 'user-text'); + let restart; + let observed = 0; + const unsubscribe = h.session.subscribe((event) => { + if ( + event.type !== 'transcript' || + event.role !== 'user' || + event.text !== 'C deferred text' + ) + return; + observed++; + h.session.stop(); + restart = h.session.start(); + }); + t.after(unsubscribe); + h.complete(oldSocket); + assert.equal(observed, 1); + assert.equal( + h.controller.replyPending, + false, + 'old onSent wrapper must not resurrect a stopped reply', + ); + await restart; + assert.equal(h.controller.replyPending, false); + const socket = h.sockets.at(-1); + assert.equal( + socket.sent.length, + 1, + 'replacement connection should contain setup only', + ); + assert.equal(h.session.sendText(TEXT), true); + await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text'); + }, +); + +test( + 'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const { socket } = await prepareDeferredPtt(h); + const before = socket.sent.length; + h.complete(socket); + assert.deepEqual( + socket.sent + .slice(before) + .map((message) => + message.realtimeInput?.activityStart + ? 'start' + : message.realtimeInput?.audio?.data === OLD_PCM + ? 'audio' + : message.realtimeInput?.activityEnd + ? 'end' + : 'other', + ), + ['start', 'audio', 'end'], + ); + assert.equal(h.controller.deferredInput, null); + assert.equal(h.controller.replyPending, true); + assert.equal(h.controller.retiredReplyPending, true); + h.complete(socket); // B's boundary; C remains the owned submitted input. + assert.equal(h.controller.retiredReplyPending, false); + assert.equal(h.controller.replyPending, true); + await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred'); + }, +); diff --git a/src/voice/geminiIntegration.test.mjs b/src/voice/geminiIntegration.test.mjs new file mode 100644 index 000000000..44c87b341 --- /dev/null +++ b/src/voice/geminiIntegration.test.mjs @@ -0,0 +1,287 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + nativeIntegrationFixture, + assertSuccessfulZoom, + turnOfEventLoop, +} from '../testSupport/geminiIntegrationFixture.mjs'; + +const options = { concurrency: false, timeout: 5000 }; +const resultFor = (h, id) => + h.events.filter( + (event) => event.type === 'action-result' && event.callId === id, + ); +const callFor = (h, id) => + h.events.filter( + (event) => event.type === 'action-call' && event.callId === id, + ); + +test( + 'native tool reaches actual common session and camera runner without a caption', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const ws = await h.start(); + h.activity(ws); // Native activity is input framing, not transcript admission. + await h.tool('ordinary', ws); + assertSuccessfulZoom(h, ws, 'ordinary', 1000); + assert.equal(h.runnerCalls.length, 1); + assert.equal(h.runnerCalls[0].options.callId, 'ordinary'); + assert.equal(h.runnerCalls[0].options.signal.aborted, false); + assert.equal(h.runnerCalls[0].options.isCurrent(), true); + h.complete(ws); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + }, +); + +for (const cancellation of ['stop', 'native interruption']) { + test( + `${cancellation} discards a stale completed action result; successor changes camera once`, + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const oldSocket = await h.start(); + h.activity(oldSocket); + const gate = h.holdResult('old'); + const oldDispatch = h.tool('old', oldSocket); + const committedResult = await gate.ready; + assert.equal(committedResult.ok, true); + assert.equal( + h.state.height, + 750, + 'the legitimate first action committed before cancellation', + ); + assert.equal(callFor(h, 'old').length, 1); + assert.deepEqual(resultFor(h, 'old'), []); + assert.deepEqual(h.outputs(oldSocket), []); + + if (cancellation === 'stop') h.session.stop(); + else + oldSocket.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + assert.equal(h.runnerCalls[0].options.signal.aborted, true); + assert.equal(h.runnerCalls[0].options.isCurrent(), false); + await oldDispatch; // Cancellation must settle before the held result returns. + gate.release(); + await turnOfEventLoop(); + assert.equal( + h.state.height, + 750, + 'settling a cancelled result adds no camera mutation', + ); + assert.equal(h.state.zooms.length, 1); + assert.deepEqual(resultFor(h, 'old'), []); + assert.deepEqual(h.outputs(oldSocket), []); + if (cancellation === 'native interruption') + assert.ok(h.events.some((event) => event.type === 'interruption')); + + const successorSocket = + cancellation === 'stop' ? await h.start() : oldSocket; + h.activity(successorSocket); + await h.tool('successor', successorSocket); + assertSuccessfulZoom(h, successorSocket, 'successor', 750); + assert.equal(h.state.zooms.length, 2); + assert.equal(h.runnerCalls.length, 2); + assert.equal( + h.outputs(successorSocket).some((output) => output.id === 'old'), + false, + ); + h.complete(successorSocket); + }, + ); +} + +for (const phase of ['common connecting', 'adapter connecting']) { + test( + `Stop from ${phase} observer prevents acquisition and permits fresh Start`, + options, + async (t) => { + const h = nativeIntegrationFixture(t); + let stopped = false; + const remove = h.session.subscribe((event) => { + const matchingDetail = + phase === 'common connecting' + ? event.detail === undefined + : typeof event.detail === 'string' && event.detail.length > 0; + if ( + !stopped && + event.type === 'state' && + event.state === 'connecting' && + matchingDetail + ) { + stopped = true; + h.session.stop(); + } + }); + await h.session.start(); + remove(); + assert.equal( + stopped, + true, + 'the chosen production observer boundary was exercised', + ); + assert.deepEqual(h.acquisitions, { + microphone: 0, + audio: 0, + token: 0, + socket: 0, + }); + assert.equal(h.session.state, 'idle'); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + assert.equal(h.clock.timers.size, 0); + const ws = await h.start(); + h.activity(ws); + await h.tool('fresh-after-start-cancel', ws); + assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000); + h.complete(ws); + }, + ); +} + +test( + 'Stop from common action-call observer prevents runner and leaves no unhandled rejection', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const ws = await h.start(); + const remove = h.session.subscribe((event) => { + if (event.type === 'action-call' && event.callId === 'cancel-at-call') + h.session.stop(); + }); + h.activity(ws); + await h.tool('cancel-at-call', ws); + // Do not install a swallowing process rejection listener. node:test records + // an unhandled rejection as a failure; these checkpoints let it surface here. + await turnOfEventLoop(); + await turnOfEventLoop(); + remove(); + assert.equal(callFor(h, 'cancel-at-call').length, 1); + assert.deepEqual(resultFor(h, 'cancel-at-call'), []); + assert.equal(h.runnerCalls.length, 0); + assert.equal(h.state.height, 1000); + assert.deepEqual(h.outputs(ws), []); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + assert.equal(h.clock.timers.size, 0); + const successor = await h.start(); + h.activity(successor); + await h.tool('after-call-cancel', successor); + assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000); + h.complete(successor); + }, +); + +test( + 'Stop from executing observer cannot recreate narration or dispatch the runner', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const ws = await h.start(); + let stopped = false; + const remove = h.session.subscribe((event) => { + if (!stopped && event.type === 'state' && event.state === 'executing') { + stopped = true; + h.session.stop(); + } + }); + h.activity(ws); + await h.tool('cancel-at-executing', ws); + remove(); + assert.equal(stopped, true); + assert.equal(h.runnerCalls.length, 0); + assert.deepEqual(callFor(h, 'cancel-at-executing'), []); + assert.deepEqual(resultFor(h, 'cancel-at-executing'), []); + assert.equal(h.state.height, 1000); + assert.deepEqual(h.outputs(ws), []); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + assert.equal(h.clock.timers.size, 0); + assert.equal(ws.closed, true); + assert.ok(h.audios.every((audio) => audio.closed)); + assert.ok(h.tracks.every((track) => track.readyState === 'ended')); + const before = { + events: h.events.length, + wire: ws.sent.length, + earcon: h.state.earconStarts, + }; + h.clock.advance(60_000); + assert.deepEqual( + { + events: h.events.length, + wire: ws.sent.length, + earcon: h.state.earconStarts, + }, + before, + ); + const successor = await h.start(); + h.activity(successor); + await h.tool('after-executing-cancel', successor); + assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000); + h.complete(successor); + }, +); + +test( + 'native idle observer restart remains connecting and public Stop cancels its pending permission', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + await h.start(); + let grant; + const permission = new Promise((resolve) => { + grant = resolve; + }); + h.controller.getUserMedia = () => permission; + const states = []; + let restarted = false; + let startup; + const remove = h.session.subscribe((event) => { + if (event.type === 'state') states.push(event.state); + if (!restarted && event.type === 'state' && event.state === 'idle') { + restarted = true; + startup = h.session.start(); + } + }); + h.session.stop(); + remove(); + assert.equal(restarted, true); + assert.equal(states.filter((state) => state === 'idle').length, 1); + assert.equal(h.session.state, 'connecting'); + assert.equal(h.controller.status, 'connecting'); + assert.equal( + h.session.isActive(), + true, + 'the common button/facade must offer Stop', + ); + h.session.stop(); + assert.equal(h.session.state, 'idle'); + assert.equal(h.controller.status, 'idle'); + const lateTrack = { + readyState: 'live', + stopCalls: 0, + stop() { + this.stopCalls++; + this.readyState = 'ended'; + }, + }; + h.tracks.push(lateTrack); + grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] }); + await startup; + await turnOfEventLoop(); + assert.equal(lateTrack.stopCalls, 1); + assert.equal( + h.requests.length, + 1, + 'cancelled restart must not mint another token', + ); + assert.equal( + h.sockets.length, + 1, + 'cancelled restart must not open another socket', + ); + assert.equal(h.session.state, 'idle'); + }, +); diff --git a/src/voice/geminiNativeCaptionEvents.test.mjs b/src/voice/geminiNativeCaptionEvents.test.mjs new file mode 100644 index 000000000..0157ee8fc --- /dev/null +++ b/src/voice/geminiNativeCaptionEvents.test.mjs @@ -0,0 +1,85 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { harness } from '../testSupport/geminiNativeHarness.mjs'; + +const caption = (socket, text) => + socket.message({ serverContent: { inputTranscription: { text } } }); +const userEvents = (events) => + events.filter( + (event) => event.type === 'transcript' && event.role === 'user', + ); + +test('native caption events preserve received advisory segments without granting tool authority', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + const start = h.events.length; + for (const text of ['little. ', 'Zoom in a ']) caption(ws, text); + const received = userEvents(h.events.slice(start)); + assert.deepEqual( + received.map((event) => event.text), + ['little. ', 'Zoom in a '], + ); + assert.equal( + received.every( + (event) => + event.delta === true && + event.final === false && + event.attribution === 'unverified' && + event.responseId === null, + ), + true, + ); + assert.equal(h.c.currentTypedInput(), null); + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'ordinary-native', + name: 'get_current_view_state', + args: {}, + }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal(h.calls[0].options.userTurn, undefined); + assert.equal(h.calls[0].options.isCurrent(), true); + ws.message({ serverContent: { turnComplete: true } }); + assert.equal( + userEvents(h.events.slice(start)).some((event) => event.final), + false, + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); +}); + +test('native old socket captions cannot enter a restarted controller event stream', async () => { + const h = harness(); + try { + await h.c.start(); + const old = h.sockets[0]; + caption(old, 'old partial'); + h.c.stop(); + await h.c.start(); + const start = h.events.length; + const current = h.sockets[1]; + caption(current, 'Zoom '); + caption(old, 'stale old socket'); + caption(current, 'in a little.'); + current.message({ serverContent: { turnComplete: true } }); + assert.deepEqual( + userEvents(h.events.slice(start)).map((event) => event.text), + ['Zoom ', 'in a little.'], + ); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.captures[0].closed, true); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); +}); diff --git a/src/voice/geminiNativeOwnership.test.mjs b/src/voice/geminiNativeOwnership.test.mjs new file mode 100644 index 000000000..043ebbe31 --- /dev/null +++ b/src/voice/geminiNativeOwnership.test.mjs @@ -0,0 +1,667 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + harness, + tick, + deferred, +} from '../testSupport/geminiNativeHarness.mjs'; +import { withToolCatalog } from './gevRealtime.js'; + +const audioContent = (data) => ({ + modelTurn: { + parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }], + }, +}); +const request = (ws, id, name = 'get_current_view_state', args = {}) => + ws.message({ toolCall: { functionCalls: [{ id, name, args }] } }); +const responses = (ws) => + ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []); +const onset = (ws) => { + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); +}; +const caption = (ws, text) => + ws.message({ serverContent: { inputTranscription: { text } } }); + +for (const [id, name, args] of [ + ['H01', 'get_entity_context', {}], + ['H02', 'control_radio', { action: 'status' }], +]) { + test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => { + const ready = deferred(), + query = deferred(), + executions = []; + const h = harness({ + runAction: (name, args, options) => { + executions.push({ name, args, options }); + return args.action === 'play' + ? Promise.resolve({ + ok: true, + action: 'control_radio', + radioAction: 'play', + radioPlaybackRequested: true, + }) + : query.promise; + }, + }); + h.c.radio.radioLayer.playForVoice = () => ready.promise; + try { + await h.c.start(); + h.c.sendTextCommand('Play radio'); + const ws = h.sockets[0]; + request(ws, 'owner', 'control_radio', { action: 'play' }); + await h.c.toolQueue; + ws.message({ + serverContent: { ...audioContent('confirmation'), turnComplete: true }, + }); + h.captures[0].drain(); + assert.equal(h.c.radio.radioHandoffInFlight, true); + request(ws, 'query', name, args); + await tick(); + assert.equal( + executions.length, + 2, + 'current native query may start after provider completion', + ); + const epoch = h.c.radio.radioHandoffEpoch; + const pending = h.c.radio.pendingRadioPlaybackResult; + const attempt = h.c.radio.radioHandoffAttemptId; + const ownerCurrent = executions[0].options.isCurrent(); + ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } }); + await h.c.toolQueue; + assert.equal(executions[1].options.signal.aborted, true); + assert.equal(executions[1].options.isCurrent(), false); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + assert.equal(executions[0].options.signal.aborted, false); + assert.equal(executions[0].options.isCurrent(), ownerCurrent); + assert.equal(h.c.radio.pendingRadioPlaybackResult, pending); + assert.equal(h.c.radio.radioHandoffAttemptId, attempt); + assert.equal(h.c.radio.radioHandoffInFlight, true); + assert.equal(h.state.stopped, 0); + assert.equal(h.c.pendingTools, 0); + assert.deepEqual( + responses(ws).map((r) => r.id), + ['owner'], + ); + assert.equal(responses(ws)[0].response.ok, true); + query.resolve({ ok: true }); + ready.resolve(true); + await tick(); + assert.equal(h.c.status, 'idle'); + assert.equal(h.state.ducked, false); + assert.deepEqual( + responses(ws).map((r) => r.id), + ['owner'], + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const [id, prior] of [ + ['H03', 'playing'], + ['H04', 'before-first-output'], +]) { + test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => { + const h = harness(); + try { + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0]; + if (prior === 'playing') + ws.message({ serverContent: audioContent('old') }); + else { + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + } + h.c.input.pushToTalkKeyHeld = true; + assert.equal(h.c.input.bargeIn(), true); + h.c.input.setMicrophoneEnabled(true); + h.c.input.setMicrophoneEnabled(false); + const turn = h.c.turn, + played = audio.played.length, + firstEvent = h.events.length; + ws.message({ + serverContent: { + ...audioContent('late-old'), + outputTranscription: { text: 'retired output' }, + inputTranscription: { text: 'new request' }, + }, + }); + request(ws, 'retired-call', 'fly_to_location', { query: 'old target' }); + await h.c.toolQueue; + assert.equal(audio.played.length, played); + assert.equal(audio.pending, false); + assert.equal(h.calls.length, 0); + assert.equal( + h.events.slice(firstEvent).some((e) => e.role === 'assistant'), + false, + ); + assert.equal( + h.events + .slice(firstEvent) + .some((e) => e.role === 'user' && e.text === 'new request'), + true, + ); + ws.message({ serverContent: { interrupted: true } }); + assert.equal(h.c.turn, turn); + ws.message({ serverContent: audioContent('after-interrupted') }); + assert.equal( + audio.played.length, + played, + 'interrupted alone cannot release the fence', + ); + ws.message({ + serverContent: { + ...audioContent('boundary-old'), + outputTranscription: { text: 'still retired' }, + turnComplete: true, + }, + }); + assert.equal( + audio.played.length, + played, + 'coalesced old boundary output is discarded', + ); + assert.equal( + h.events.slice(firstEvent).some((e) => e.type === 'completion'), + false, + ); + caption(ws, 'unrelated advisory words'); + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'retired-call', + name: 'fly_to_location', + args: { query: 'replayed old target' }, + }, + { + id: 'new-call', + name: 'fly_to_location', + args: { query: 'new target' }, + }, + ], + }, + }); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((c) => c.args.query), + ['new target'], + ); + assert.equal(h.calls[0].options.userTurn, undefined); + ws.message({ + serverContent: { + ...audioContent('new'), + outputTranscription: { text: 'new reply' }, + turnComplete: true, + }, + }); + assert.equal(audio.played.at(-1).data, 'new'); + audio.drain(); + assert.equal( + h.events.findLast((e) => e.type === 'completion').status, + 'completed', + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => { + const h = harness(); + try { + await h.c.start(); + const ws = h.sockets[0], + audio = h.captures[0]; + h.c.sendTextCommand('Show the current view'); + const response = h.c.currentTypedInput(); + assert.equal(response.text, 'Show the current view'); + assert.equal( + h.events.findLast((e) => e.type === 'transcript' && e.role === 'user') + .text, + response.text, + ); + request(ws, 'first'); + await h.c.toolQueue; + assert.equal(h.calls[0].options.userTurn, undefined); + assert.equal(h.calls[0].options.isCurrent(), true); + ws.message({ + serverContent: { + ...audioContent('confirmation'), + outputTranscription: { text: 'Here is the current view' }, + turnComplete: true, + }, + }); + assert.equal( + h.events.some((e) => e.type === 'completion'), + false, + ); + assert.equal( + h.events.findLast((e) => e.role === 'assistant').responseId, + response.responseId, + ); + audio.drain(); + const completion = h.events.findLast((e) => e.type === 'completion'); + assert.equal(completion.responseId, response.responseId); + assert.equal(completion.spoken, true); + assert.equal(completion.audioDrained, true); + assert.equal(h.c.currentTypedInput(), response); + assert.equal(response.text, 'Show the current view'); + request(ws, 'same-turn'); + await h.c.toolQueue; + assert.equal(h.calls[1].options.userTurn, undefined); + assert.equal(h.c.currentTypedInput(), response); + h.c.sendTextCommand('New request'); + assert.notEqual(h.c.currentTypedInput(), response); + assert.equal(h.c.currentTypedInput().text, 'New request'); + assert.equal(h.calls[0].options.isCurrent(), false); + assert.equal(h.calls[1].options.isCurrent(), false); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); +}); + +test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => { + const gate = deferred(), + executions = []; + const h = harness({ + runAction: (name, args, options) => { + executions.push({ name, args, options }); + return options.callId === 'old' + ? gate.promise + : Promise.resolve({ ok: true }); + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + request(ws, 'old'); + await tick(); + const oldWork = h.c.toolQueue; + assert.equal(h.c.pendingTools, 1); + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + assert.equal(h.c.pendingTools, 0); + await oldWork; + assert.equal(h.c.pendingTools, 0); + assert.equal(executions[0].options.signal.aborted, true); + assert.equal(executions[0].options.isCurrent(), false); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + request(ws, 'fresh'); + await h.c.toolQueue; + assert.equal(executions.at(-1).options.callId, 'fresh'); + assert.equal(h.c.pendingTools, 0); + gate.resolve({ ok: true }); + await tick(); + assert.equal(h.c.pendingTools, 0); + assert.deepEqual( + responses(ws).map((r) => r.id), + ['fresh'], + ); + ws.message({ + serverContent: { ...audioContent('fresh reply'), turnComplete: true }, + }); + h.captures[0].drain(); + assert.equal( + h.events.findLast((e) => e.type === 'completion').status, + 'completed', + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); +}); + +test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => { + const gate = deferred(), + executions = []; + const h = harness({ + runAction: (name, args, options) => { + executions.push({ name, args, options }); + return gate.promise; + }, + }); + try { + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0]; + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { data: 'AA==', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + caption(ws, 'Show the current view'); + assert.equal(h.c.currentTypedInput(), null); + assert.equal( + h.events.findLast((e) => e.type === 'transcript' && e.role === 'user') + .text, + 'Show the current view', + ); + request(ws, 'held'); + await tick(); + const oldWork = h.c.toolQueue; + assert.equal(executions.length, 1); + assert.equal(executions[0].options.userTurn, undefined); + assert.equal(executions[0].options.isCurrent(), true); + h.c.sendTextCommand('new intent'); + await oldWork; + assert.equal(executions[0].options.signal.aborted, true); + assert.equal(executions[0].options.isCurrent(), false); + gate.resolve({ ok: true }); + await tick(); + assert.equal( + responses(ws).some((r) => r.id === 'held'), + false, + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); +}); + +for (const [id, restart] of [ + ['H11', false], + ['H12', true], +]) { + test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => { + const gate = deferred(), + mutations = [], + executions = []; + const h = harness({ + runAction: async (name, args, options) => { + executions.push(options); + await gate.promise; + if (options.isCurrent()) mutations.push(name); + return { ok: true }; + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + request(ws, 'old'); + await tick(); + const pending = h.c.toolQueue; + assert.equal(h.c.pendingTools, 1); + h.c.stop(); + const boundary = h.events.length; + const successor = restart ? h.c.start() : null; + await pending; + assert.equal( + h.events + .slice(boundary) + .some((e) => e.type === 'state' && e.state === 'listening'), + false, + ); + assert.equal(executions[0].signal.aborted, true); + assert.equal(executions[0].isCurrent(), false); + assert.equal(h.c.pendingTools, 0); + if (successor) await successor; + const laterBoundary = h.events.length; + gate.resolve(); + caption(ws, 'late old speech'); + await tick(); + assert.deepEqual(mutations, []); + assert.equal( + h.events.slice(laterBoundary).some((e) => e.type === 'state'), + false, + ); + assert.equal(responses(ws).length, 0); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const [id, boundary] of [ + ['H14', 'interrupted'], + ['H15', 'natural'], +]) { + test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => { + const mutations = [], + observed = []; + const guarded = withToolCatalog( + async (name, args) => { + mutations.push({ name, args }); + return { ok: true }; + }, + async () => null, + ); + const h = harness({ + runAction: (name, args, options) => { + observed.push(options); + return guarded(name, args, options); + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: audioContent('old reply') }); + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ + serverContent: { + ...(boundary === 'interrupted' ? { interrupted: true } : {}), + turnComplete: true, + }, + }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + caption(ws, 'zoom in'); + request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' }); + await tick(); + assert.equal( + mutations.length, + 1, + 'native action executes without a transcript wait', + ); + assert.equal(observed[0].userTurn, undefined); + assert.equal(h.c.pendingTools, 0); + assert.equal( + responses(ws).some((r) => r.response.voiceIntentUnverified), + false, + ); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const [id, exit] of [ + ['H18', 'stop'], + ['H19', 'replacement'], +]) { + test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => { + const mutations = [], + observed = []; + const guarded = withToolCatalog( + async (name) => { + mutations.push(name); + return { ok: true }; + }, + async () => null, + ); + const h = harness({ + runAction: (name, args, options) => { + observed.push(name); + return guarded(name, args, options); + }, + }); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' }); + const oldWork = h.c.toolQueue; + assert.equal(h.c.pendingTools, 1); + if (exit === 'stop') h.c.stop(); + else h.c.sendTextCommand('Show the whole globe'); + caption(ws, 'zoom in'); + await oldWork; + assert.deepEqual(mutations, []); + assert.deepEqual(observed, []); + assert.equal(h.c.pendingTools, 0); + if (exit === 'replacement') { + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + request(ws, 'fresh-camera', 'zoom_to_globe'); + await h.c.toolQueue; + assert.deepEqual(mutations, ['zoom_to_globe']); + assert.deepEqual(observed, ['zoom_to_globe']); + assert.equal(responses(ws).at(-1).response.ok, true); + } + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const settlement of ['queued', 'running']) { + test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => { + const result = deferred(); + let executions = 0; + const h = harness({ + runAction: () => { + executions++; + return result.promise; + }, + }); + try { + await h.c.start(); + h.c.sendTextCommand('Read current view'); + const ws = h.sockets[0]; + request(ws, 'cancelled-query'); + if (settlement === 'running') await tick(); + ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } }); + ws.message({ serverContent: { turnComplete: true } }); + await h.c.toolQueue; + assert.equal(h.c.pendingTools, 0); + assert.equal(executions, settlement === 'queued' ? 0 : 1); + assert.deepEqual(responses(ws), []); + assert.equal( + h.events.filter((event) => event.type === 'completion').length, + 1, + ); + assert.equal(h.c.pendingCompletionStatus, null); + result.resolve({ ok: true }); + await tick(); + h.captures[0].drain(); + assert.equal( + h.events.filter((event) => event.type === 'completion').length, + 1, + ); + assert.deepEqual(responses(ws), []); + } finally { + result.resolve({ ok: true }); + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const order of ['before', 'after']) { + test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => { + const h = harness(); + try { + await h.c.start(); + h.c.sendTextCommand('Read current view'); + const ws = h.sockets[0]; + ws.bufferedAmount = 1e9; + request(ws, 'queued-output'); + await h.c.toolQueue; + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.controlQueue.length, 1); + assert.equal(responses(ws).length, 0); + if (order === 'after') + ws.message({ serverContent: { turnComplete: true } }); + ws.message({ toolCallCancellation: { ids: ['queued-output'] } }); + if (order === 'before') + ws.message({ serverContent: { turnComplete: true } }); + assert.equal(h.c.controlQueue.length, 0); + assert.equal(h.c.controlTimer, null); + assert.equal(h.c.pendingCompletionStatus, null); + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + ws.bufferedAmount = 0; + h.advance(1000); + h.captures[0].drain(); + assert.deepEqual(responses(ws), []); + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const successor of ['input', 'stop']) { + test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => { + let h; + let successorTurn; + h = harness({ + emit: (event) => { + h.events.push(event); + if (event.type !== 'completion') return; + h.sockets[0].bufferedAmount = 0; + if (successor === 'stop') h.c.stop(); + else { + h.c.sendTextCommand('Show the whole globe'); + successorTurn = h.c.turn; + } + }, + }); + try { + await h.c.start(); + h.c.sendTextCommand('Read current view'); + const ws = h.sockets[0]; + ws.bufferedAmount = 1e9; + request(ws, 'queued-output'); + await h.c.toolQueue; + ws.message({ serverContent: { turnComplete: true } }); + const callCount = h.calls.length; + ws.message({ + toolCallCancellation: { ids: ['queued-output'] }, + serverContent: { + outputTranscription: { text: 'Retired packet caption' }, + turnComplete: true, + }, + toolCall: { + functionCalls: [ + { id: 'retired-tool', name: 'get_current_view_state', args: {} }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + assert.equal(h.calls.length, callCount); + assert.deepEqual(responses(ws), []); + assert.ok( + !h.events.some((e) => + JSON.stringify(e).includes('Retired packet caption'), + ), + ); + if (successor === 'input') { + assert.equal(h.c.turn, successorTurn); + assert.equal(h.c.modelComplete, false); + assert.equal(h.c.replyPending, true); + } else assert.equal(h.c.isActive(), false); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} diff --git a/src/voice/geminiNativeRadio.test.mjs b/src/voice/geminiNativeRadio.test.mjs new file mode 100644 index 000000000..38ba8ff79 --- /dev/null +++ b/src/voice/geminiNativeRadio.test.mjs @@ -0,0 +1,734 @@ +// Native ownership replacements for historical Radio regression cases. +// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + harness, + tick, + deferred, +} from '../testSupport/geminiNativeHarness.mjs'; +import { createGevActionRunner } from './gevActions.js'; +import { createVoiceSession } from './session.js'; +import { DataLayerManager } from '../data/manager.js'; + +const viewerFixture = () => ({ + clock: { onTick: { addEventListener: () => () => {} } }, + camera: { moveEnd: { addEventListener() {} } }, + scene: { canvas: { addEventListener() {}, removeEventListener() {} } }, +}); +const request = (ws, id, name, args = {}) => + ws.message({ toolCall: { functionCalls: [{ id, name, args }] } }); +const responses = (ws) => + ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []); +const responseFor = (ws, id) => + responses(ws).findLast((response) => response.id === id)?.response; +const speechTurn = (ws) => { + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); +}; +function drainRadioReply(h) { + h.sockets.at(-1).message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + turnComplete: true, + }, + }); + h.captures.at(-1).drain(); +} + +function radioSiblingFixture() { + const viewer = viewerFixture(); + const manager = new DataLayerManager(viewer); + const lookupEntered = deferred(); + const lookupReleased = deferred(); + const state = { + stationCount: 2, + selected: { id: 'existing-station', name: 'Existing Station' }, + audioState: 'stopped', + }; + const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() }; + const radio = { + async init() {}, + async enable() {}, + async disable() {}, + async update() {}, + getStats: () => ({ count: 2 }), + getUIState: () => ({ ...state }), + setVoiceDucked() {}, + pause({ origin } = {}) { + if (origin === 'voice') trace.pauses++; + state.audioState = 'paused'; + return true; + }, + stopPlayback() { + trace.stops++; + state.audioState = 'stopped'; + return true; + }, + selectRequestedStation() { + trace.selections++; + state.selected = { id: 'late-station', name: 'Late Station' }; + return state.selected; + }, + subscribePlaybackControls: () => () => {}, + }; + manager.register( + Object.assign(radio, { + id: 'radio', + name: 'Radio', + updateInterval: -1, + }), + ); + const runner = createGevActionRunner({ + viewer, + dataManager: manager, + placeSearch: { + async geocode(_query, { signal }) { + trace.lookupSignal = signal; + lookupEntered.resolve(); + // Deliberately ignore abort: production currentness must still prevent + // this late service completion from committing a station selection. + await lookupReleased.promise; + return { place: { lat: 10, lng: 20, label: 'Deferred Place' } }; + }, + }, + }); + const h = harness({ + dataManager: manager, + radioLayer: radio, + runAction(name, args, options) { + trace.options.set(options.callId, options); + return runner(name, args, options); + }, + }); + return { + h, + manager, + runner, + radio, + trace, + state, + lookupEntered, + lookupReleased, + }; +} + +for (const action of ['select', 'play']) { + for (const stronger of ['stop', 'pause', 'disable', 'off']) { + test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => { + const f = radioSiblingFixture(); + const { h, trace, state } = f; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + const turn = h.c.turn; + request(ws, 'slow-selection', 'control_radio', { + action, + locationQuery: 'Deferred Place', + }); + await f.lookupEntered.promise; + request(ws, 'query-sibling', 'control_radio', { action: 'status' }); + request( + ws, + 'stronger-radio', + stronger === 'off' ? 'set_layer_visibility' : 'control_radio', + stronger === 'off' + ? { layerId: 'radio', enabled: false } + : { action: stronger }, + ); + await tick(); + assert.equal( + responseFor(ws, 'stronger-radio')?.ok, + true, + 'stronger control must finish before lookup release', + ); + assert.equal(trace.lookupSignal.aborted, true); + await h.c.toolQueue; + assert.equal( + h.c.turn, + turn, + 'sibling control cannot invent a new turn', + ); + assert.equal(trace.selections, 0); + assert.equal(state.selected.id, 'existing-station'); + assert.equal(responseFor(ws, 'slow-selection').ok, false); + assert.match(responseFor(ws, 'slow-selection').error, /cancelled/); + assert.equal( + responseFor(ws, 'slow-selection').enabled, + !['disable', 'off'].includes(stronger), + ); + assert.equal(responseFor(ws, 'query-sibling').ok, true); + assert.equal(trace.options.get('query-sibling').signal.aborted, false); + assert.equal( + responseFor(ws, 'stronger-radio').enabled, + !['disable', 'off'].includes(stronger), + ); + assert.equal(trace.stops, stronger === 'stop' ? 1 : 0); + assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0); + f.lookupReleased.resolve(); + await tick(); + assert.equal(trace.selections, 0, 'late lookup cannot commit'); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.tools.size, 0); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + for (const id of ['slow-selection', 'query-sibling', 'stronger-radio']) + assert.equal(responses(ws).filter((r) => r.id === id).length, 1); + } finally { + f.lookupReleased.resolve(); + h.c.stop(); + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); + } +} + +for (const guard of ['malformed', 'replay', 'invalid-arguments']) { + test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => { + const f = radioSiblingFixture(); + const { h, trace } = f; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + if (guard === 'replay') { + request(ws, 'stronger-radio', 'control_radio', { action: 'stop' }); + await h.c.toolQueue; + } + speechTurn(ws); + if (guard === 'replay') + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + request(ws, 'slow-selection', 'control_radio', { + action: 'select', + locationQuery: 'Deferred Place', + }); + await f.lookupEntered.promise; + const stops = trace.stops; + const handoffEpoch = h.c.radio.radioHandoffEpoch; + request( + ws, + guard === 'malformed' ? '\n' : 'stronger-radio', + 'control_radio', + { + action: 'stop', + ...(guard === 'invalid-arguments' + ? { latitude: 91, longitude: 0 } + : {}), + }, + ); + await tick(); + assert.equal(trace.lookupSignal.aborted, false); + assert.equal(trace.stops, stops); + assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + if (guard === 'invalid-arguments') { + assert.equal(responseFor(ws, 'stronger-radio').ok, false); + assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/); + } + f.lookupReleased.resolve(); + await h.c.toolQueue; + assert.equal( + trace.selections, + 1, + 'older eligible work retains authority', + ); + assert.equal(responseFor(ws, 'slow-selection').ok, true); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.tools.size, 0); + } finally { + f.lookupReleased.resolve(); + h.c.stop(); + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); +} + +test('native failed Stop releases its reservation and lets the older selection finish', async () => { + const f = radioSiblingFixture(); + const { h, trace } = f; + const stopReleased = deferred(); + f.radio.stopPlayback = () => stopReleased.promise; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + request(ws, 'slow-selection', 'control_radio', { + action: 'select', + locationQuery: 'Deferred Place', + }); + await f.lookupEntered.promise; + request(ws, 'failed-stop', 'control_radio', { action: 'stop' }); + await tick(); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 1); + assert.equal( + trace.lookupSignal.aborted, + false, + 'provisional Stop cannot revoke work', + ); + assert.equal(h.c.pendingTools, 2); + stopReleased.resolve(false); + await tick(); + assert.equal(responseFor(ws, 'failed-stop').ok, false); + assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + assert.equal(trace.lookupSignal.aborted, false); + f.lookupReleased.resolve(); + await h.c.toolQueue; + assert.equal(responseFor(ws, 'slow-selection').ok, true); + assert.equal(trace.selections, 1); + assert.equal(f.state.selected.id, 'late-station'); + assert.equal(h.c.pendingTools, 0); + } finally { + stopReleased.resolve(false); + f.lookupReleased.resolve(); + h.c.stop(); + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); +}); + +test('native pending Stop preserves selection completed before its own successful settlement', async () => { + const f = radioSiblingFixture(); + const { h } = f; + const stopReleased = deferred(); + f.radio.stopPlayback = () => stopReleased.promise; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + request(ws, 'slow-selection', 'control_radio', { + action: 'select', + locationQuery: 'Deferred Place', + }); + await f.lookupEntered.promise; + request(ws, 'pending-stop', 'control_radio', { action: 'stop' }); + await tick(); + f.lookupReleased.resolve(); + await tick(); + assert.equal(responseFor(ws, 'slow-selection').ok, true); + assert.equal(f.state.selected.id, 'late-station'); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 1); + assert.ok(h.c.radio.pendingRadioPlaybackResult); + stopReleased.resolve(true); + await h.c.toolQueue; + assert.equal(responseFor(ws, 'pending-stop').ok, true); + assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station'); + assert.equal(f.state.selected.id, 'late-station'); + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(h.c.pendingTools, 0); + } finally { + stopReleased.resolve(false); + f.lookupReleased.resolve(); + h.c.stop(); + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); +}); + +for (const restart of [false, true]) { + test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => { + const f = radioSiblingFixture(); + const { h } = f; + const stopReleased = deferred(); + f.radio.stopPlayback = () => stopReleased.promise; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + request(ws, 'slow-selection', 'control_radio', { + action: 'select', + locationQuery: 'Deferred Place', + }); + await f.lookupEntered.promise; + request(ws, 'pending-stop', 'control_radio', { action: 'stop' }); + await tick(); + const oldDrain = h.c.toolQueue; + const stopOptions = f.trace.options.get('pending-stop'); + assert.equal(h.c.pendingTools, 2); + h.c.stop(); + assert.equal(stopOptions.signal.aborted, true); + assert.equal(f.trace.lookupSignal.aborted, true); + if (restart) { + await h.c.start(); + request(h.sockets[1], 'new-query', 'control_radio', { + action: 'status', + }); + await h.c.toolQueue; + assert.equal(responseFor(h.sockets[1], 'new-query').ok, true); + } + stopReleased.resolve(true); + f.lookupReleased.resolve(); + await oldDrain; + await tick(); + assert.equal(responseFor(ws, 'pending-stop'), undefined); + assert.equal(responseFor(ws, 'slow-selection'), undefined); + assert.equal(f.trace.selections, 0); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.tools.size, 0); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + assert.equal(h.c.isActive(), restart); + } finally { + stopReleased.resolve(false); + f.lookupReleased.resolve(); + h.c.stop(); + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const phase of ['prepared', 'preflight']) { + test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => { + const f = radioSiblingFixture(); + const { h } = f; + const ready = deferred(); + const stopped = []; + let preflight; + f.radio.playForVoice = () => ready.promise; + f.radio.stopPlayback = (options) => { + stopped.push(options); + return true; + }; + try { + await f.manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + request(ws, 'prepared-play', 'control_radio', { action: 'play' }); + await h.c.toolQueue; + assert.equal( + responseFor(ws, 'prepared-play').radioPlaybackRequested, + true, + ); + if (phase === 'preflight') + preflight = h.c.radio.startPendingRadioHandoff(); + const attemptId = h.c.radio.radioHandoffAttemptId; + request(ws, 'stronger-stop', 'control_radio', { action: 'stop' }); + await h.c.toolQueue; + assert.equal(responseFor(ws, 'stronger-stop').ok, true); + assert.equal(f.state.selected.id, 'existing-station'); + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(h.c.radio.radioHandoffInFlight, false); + const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup'); + assert.deepEqual( + cleanups, + phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [], + ); + ready.resolve(true); + await preflight; + await tick(); + assert.equal( + h.c.isActive(), + true, + 'late old readiness cannot stop voice', + ); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + assert.equal(h.c.pendingTools, 0); + } finally { + ready.resolve(false); + h.c.stop(); + await preflight; + f.runner.dispose(); + await f.manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); +} + +for (const phase of ['prepared', 'preflight', 'settling']) { + test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => { + const viewer = viewerFixture(); + const manager = new DataLayerManager(viewer); + const prepared = deferred(); + const settleResult = deferred(); + const playbackReady = deferred(); + let playCalls = 0; + let stopCalls = 0; + let selections = 0; + let activeOptions; + const state = { + stationCount: 2, + selected: null, + audioState: 'stopped', + voiceDucked: false, + }; + const radio = { + async init() {}, + async enable() {}, + async disable() {}, + async update() {}, + getStats: () => ({ count: 2 }), + getUIState: () => ({ ...state }), + setVoiceDucked(value) { + state.voiceDucked = value; + }, + pause() { + state.audioState = 'paused'; + }, + subscribePlaybackControls() { + return () => {}; + }, + selectRequestedStation(_criteria, options) { + assert.equal(options.autoplay, false); + selections++; + state.selected = { id: 'station-1', name: 'Station One' }; + return state.selected; + }, + playForVoice() { + playCalls++; + return playbackReady.promise; + }, + stopPlayback() { + stopCalls++; + state.audioState = 'stopped'; + }, + }; + manager.register({ + id: 'radio', + name: 'Radio', + updateInterval: -1, + ...radio, + }); + const runner = createGevActionRunner({ viewer, dataManager: manager }); + let h; + let preflight; + const session = createVoiceSession({ + runner: async (name, args, options) => { + activeOptions = options; + const result = await runner(name, args, options); + prepared.resolve(result); + if (phase === 'settling') await settleResult.promise; + return result; + }, + createAdapter: (shared) => { + h = harness({ ...shared, dataManager: manager, radioLayer: radio }); + return { + start: (settings) => h.c.start(settings), + stop: (settings) => h.c.stop(settings), + sendText: (text) => h.c.sendTextCommand(text), + sendMapEvent: (event) => h.c.notifyMapEvent(event), + }; + }, + }); + try { + await session.start(); + const ws = h.sockets[0]; + assert.equal(session.sendText('Select Station One on Radio'), true); + request(ws, 'prepared-radio', 'control_radio', { + action: 'select', + stationQuery: 'Station One', + }); + const actual = await prepared.promise; + assert.equal(actual.ok, true); + assert.equal(actual.radioPlaybackRequested, true); + assert.equal(actual.stationId, 'station-1'); + assert.equal(manager.isEnabled('radio'), true); + if (phase !== 'settling') { + await h.c.toolQueue; + assert.ok(h.c.radio.pendingRadioPlaybackResult); + } + if (phase === 'preflight') { + preflight = h.c.radio.startPendingRadioHandoff(); + assert.equal(h.c.radio.radioHandoffInFlight, true); + assert.equal(playCalls, 1); + } + // Native call ownership revokes only this deferred playback. No caption + // parsing or caption-driven provider policy participates in cancellation. + ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } }); + if (phase === 'settling') + assert.equal(activeOptions.signal.aborted, true); + settleResult.resolve(); + await h.c.toolQueue; + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(h.c.radio.radioHandoffInFlight, false); + assert.equal(manager.isEnabled('radio'), true); + assert.equal(state.selected.id, 'station-1'); + assert.equal(selections, 1); + const modelResult = responseFor(ws, 'prepared-radio'); + if (phase === 'settling') + assert.equal( + modelResult, + undefined, + 'cancelled active call never sends a late result', + ); + else { + assert.equal(modelResult.ok, true); + assert.equal(modelResult.stationId, 'station-1'); + } + const stopsAfterCancellation = stopCalls; + assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0); + drainRadioReply(h); + playbackReady.resolve(true); + await preflight; + await tick(); + assert.equal( + playCalls, + phase === 'preflight' ? 1 : 0, + 'drain cannot revive cancelled playback', + ); + assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice'); + assert.equal(h.c.radio.radioHandoffInFlight, false); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.tools.size, 0); + assert.equal(session.sendText('Play radio'), true); + request(ws, 'fresh-radio', 'control_radio', { action: 'play' }); + await h.c.toolQueue; + assert.ok(h.c.radio.pendingRadioPlaybackResult); + drainRadioReply(h); + await tick(); + assert.equal(playCalls, phase === 'preflight' ? 2 : 1); + assert.equal( + h.c.isActive(), + false, + 'fresh typed Radio handoff alone ends voice', + ); + } finally { + settleResult.resolve(); + playbackReady.resolve(false); + session.destroy(); + await preflight; + runner.dispose(); + await manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); +} + +// A stronger Radio control skips the ordered queue. An older Radio call still +// queued behind a slow sibling must not start after that control succeeded, +// while a failed control or a later call in the same turn keeps its order. +for (const [label, ids, stopOk, expectPending] of [ + [ + 'older queued select is revoked by a later successful stop', + ['slow', 'sel', 'stop'], + true, + false, + ], + [ + 'a failed stop leaves the older queued select current', + ['slow', 'sel', 'stop'], + false, + true, + ], + [ + 'a select after the stop in the same turn still runs', + ['slow', 'stop', 'sel'], + true, + true, + ], +]) { + test(`stronger Radio ordering: ${label}`, async () => { + const viewer = viewerFixture(); + const manager = new DataLayerManager(viewer); + const state = { + stationCount: 2, + selected: { id: 'existing', name: 'Existing' }, + audioState: 'stopped', + }; + const radio = { + id: 'radio', + name: 'Radio', + updateInterval: -1, + async init() {}, + async enable() {}, + async disable() {}, + async update() {}, + getStats: () => ({ count: 2 }), + getUIState: () => ({ ...state }), + setVoiceDucked() {}, + pause() { + state.audioState = 'paused'; + return true; + }, + stopPlayback() { + state.audioState = 'stopped'; + return stopOk; + }, + selectRequestedStation() { + state.selected = { id: 'late', name: 'Late' }; + return state.selected; + }, + async playForVoice() { + return true; + }, + subscribePlaybackControls: () => () => {}, + }; + manager.register(radio); + const runner = createGevActionRunner({ + viewer, + dataManager: manager, + placeSearch: { + async geocode() { + return { place: { lat: 10, lng: 20, label: 'Place' } }; + }, + }, + }); + const slow = deferred(); + const h = harness({ + dataManager: manager, + radioLayer: radio, + async runAction(name, args, options) { + if (name === 'slow_sibling') { + await slow.promise; + return { ok: true, action: 'slow_sibling' }; + } + if (name === 'control_radio' && args.action === 'stop' && !stopOk) + return { + ok: false, + action: 'control_radio', + error: 'Radio stop failed', + }; + return runner(name, args, options); + }, + }); + const calls = { + slow: { id: 'slow', name: 'slow_sibling', args: {} }, + sel: { + id: 'sel', + name: 'control_radio', + args: { action: 'select', locationQuery: 'Place' }, + }, + stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } }, + }; + try { + await manager.setEnabled('radio', true); + await h.c.start(); + const ws = h.sockets[0]; + speechTurn(ws); + ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } }); + await tick(); + assert.equal( + responseFor(ws, 'stop')?.ok, + stopOk, + 'stop settles without waiting', + ); + slow.resolve(); + await h.c.toolQueue; + await tick(); + assert.equal( + Boolean(h.c.radio.pendingRadioPlaybackResult), + expectPending, + ); + assert.equal(responseFor(ws, 'sel')?.ok, expectPending); + } finally { + slow.resolve(); + h.c.stop(); + runner.dispose(); + await manager.setEnabled('radio', false); + } + assert.equal(h.timers.size, 0); + }); +} diff --git a/src/voice/geminiNativeRecovery.test.mjs b/src/voice/geminiNativeRecovery.test.mjs new file mode 100644 index 000000000..10d23a609 --- /dev/null +++ b/src/voice/geminiNativeRecovery.test.mjs @@ -0,0 +1,280 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + harness, + tick, + deferred, +} from '../testSupport/geminiNativeHarness.mjs'; +import { createGevActionRunner } from './gevActions.js'; + +const tool = (ws, id) => + ws.message({ + toolCall: { + functionCalls: [ + { + id, + name: 'adjust_camera_zoom', + args: { direction: 'in', amount: 'little' }, + }, + ], + }, + }); +const caption = (ws, text) => + ws.message({ serverContent: { inputTranscription: { text } } }); +const onset = (ws) => { + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); +}; +const complete = (ws) => ws.message({ serverContent: { turnComplete: true } }); +const oldAudio = { + serverContent: { + modelTurn: { + parts: [ + { + inlineData: { + data: 'AAAA', + mimeType: 'audio/pcm;rate=24000', + }, + }, + ], + }, + }, +}; +const outputs = (ws) => + ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []); + +// Socket/media are deterministic. Mutations use the production action runner. +function fixture({ holdFirst = false } = {}) { + const state = { height: 1000, renders: 0, providerWrites: [], panels: [] }; + const position = { x: 6379137, y: 0, z: 0 }; + const viewer = { + clock: { onTick: { addEventListener: () => () => {} } }, + camera: { + positionWC: position, + positionCartographic: { height: 1000 }, + moveEnd: { addEventListener() {} }, + cancelFlight() {}, + zoomIn(n) { + position.x -= n; + state.height -= n; + this.positionCartographic.height = state.height; + }, + zoomOut(n) { + position.x += n; + state.height += n; + this.positionCartographic.height = state.height; + }, + }, + scene: { + canvas: { + clientWidth: 800, + clientHeight: 600, + addEventListener() {}, + removeEventListener() {}, + }, + requestRender() { + state.renders++; + }, + }, + }; + const runner = createGevActionRunner({ + viewer, + dataManager: { layers: new Map() }, + styleManager: { + setVoiceProvider: (p) => state.providerWrites.push(p), + setPanelCollapsed: (...args) => state.panels.push(args), + }, + }); + const entered = []; + const gate = deferred(); + const h = harness({ + runAction: async (name, args, options) => { + entered.push({ name, args, options }); + if (holdFirst && entered.length === 1) await gate.promise; + if (!options.isCurrent()) + return { ok: false, action: name, changed: false }; + return runner(name, args, options); + }, + }); + return { + ...h, + state, + entered, + gate, + close() { + gate.resolve(); + h.c.stop(); + runner.dispose(); + assert.equal(h.timers.size, 0); + assert.deepEqual(state.providerWrites, []); + assert.deepEqual(state.panels, []); + }, + }; +} + +for (const [id, order] of [ + ['H20', 'before-boundary'], + ['H21', 'after-boundary'], +]) { + test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => { + const h = fixture(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message(oldAudio); + onset(ws); + caption(ws, 'zoom in'); + if (order === 'before-boundary') { + tool(ws, 'ambiguous-before'); + await tick(); + assert.equal(h.state.height, 1000); + assert.equal(h.entered.length, 0); + assert.equal(outputs(ws).length, 0); + } + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + if (order === 'after-boundary') { + tool(ws, 'native-after'); + await h.c.toolQueue; + assert.equal(h.state.height, 750); + assert.equal(h.entered.length, 1); + assert.equal(outputs(ws).at(-1).response.ok, true); + complete(ws); + } + onset(ws); + caption(ws, 'zoom in'); + tool(ws, 'fresh-clean-onset'); + await h.c.toolQueue; + assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5); + assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2); + assert.equal(h.entered.at(-1).options.userTurn, undefined); + assert.equal(h.c.pendingTools, 0); + assert.equal( + outputs(ws).some((o) => o.id === 'ambiguous-before'), + false, + ); + assert.equal( + outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok, + true, + ); + } finally { + h.close(); + } + }); +} + +test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => { + const h = fixture(); + try { + await h.c.start(); + const ws = h.sockets[0]; + onset(ws); + caption(ws, 'switch to'); + tool(ws, 'ordinary-before-caption-resolution'); + await tick(); + await h.c.toolQueue; + assert.equal(h.state.height, 750); + assert.equal(h.c.pendingTools, 0); + h.advance(1499); + caption(ws, 'Austin'); + h.advance(5000); + tool(ws, 'released-sibling'); + await h.c.toolQueue; + assert.equal(h.state.height, 562.5); + assert.equal(h.entered.length, 2); + assert.equal( + h.entered.every((c) => c.options.userTurn === undefined), + true, + ); + assert.equal(outputs(ws).length, 2); + assert.equal( + outputs(ws).every((o) => o.response.ok), + true, + ); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.status, 'listening'); + } finally { + h.close(); + } +}); + +for (const [id, exit] of [ + ['H32', 'stop'], + ['H33', 'restart'], + ['H34', 'new-input'], +]) { + test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => { + const h = fixture({ holdFirst: true }); + try { + await h.c.start(); + let ws = h.sockets[0]; + onset(ws); + caption(ws, 'switch to'); + tool(ws, 'old-owned-action'); + await tick(); + assert.equal(h.entered.length, 1); + assert.equal(h.c.pendingTools, 1); + assert.equal(h.state.height, 1000); + const oldWork = h.c.toolQueue; + const oldMessage = ws.onmessage; + const oldOwner = h.entered[0].options; + if (exit === 'new-input') h.c.sendTextCommand('zoom out'); + else h.c.stop(); + assert.equal(oldOwner.signal.aborted, true); + assert.equal(oldOwner.isCurrent(), false); + if (exit === 'restart') { + await h.c.start(); + ws = h.sockets.at(-1); + } + if (exit !== 'new-input') + oldMessage({ + data: JSON.stringify({ + toolCall: { + functionCalls: [ + { + id: 'late-old-call', + name: 'adjust_camera_zoom', + args: { direction: 'in' }, + }, + ], + }, + }), + }); + caption(ws, 'Gemini'); + h.advance(1500); + h.gate.resolve(); + await oldWork; + await tick(); + assert.equal(h.state.height, 1000); + assert.equal(h.c.pendingTools, 0); + assert.equal( + outputs(ws).some( + (o) => o.id === 'old-owned-action' || o.id === 'late-old-call', + ), + false, + ); + if (exit === 'stop') { + assert.equal(h.c.status, 'idle'); + assert.equal(h.c.isActive(), false); + } else { + if (exit === 'new-input') { + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + } else h.c.sendTextCommand('zoom out'); + tool(ws, 'fresh-after-retirement'); + await h.c.toolQueue; + assert.equal(h.state.height, 750); + assert.equal(h.entered.length, 2); + assert.equal( + outputs(ws).find((o) => o.id === 'fresh-after-retirement').response + .ok, + true, + ); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.isActive(), true); + } + } finally { + h.close(); + } + }); +} diff --git a/src/voice/geminiPacketOwnership.test.mjs b/src/voice/geminiPacketOwnership.test.mjs new file mode 100644 index 000000000..4188cf7be --- /dev/null +++ b/src/voice/geminiPacketOwnership.test.mjs @@ -0,0 +1,529 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + nativeIntegrationFixture, + assertSuccessfulZoom, + turnOfEventLoop, +} from '../testSupport/geminiIntegrationFixture.mjs'; + +const options = { concurrency: false, timeout: 5000 }; +const OLD_TEXT = 'Old packet caption.'; +const OLD_PCM = 'AAABAA=='; +const POST_TOOL_PCM = 'AgADAA=='; +const OLD_CALL = 'old-coalesced-call'; + +function coalescedPacket(callId = OLD_CALL) { + return { + serverContent: { + outputTranscription: { text: OLD_TEXT }, + modelTurn: { + parts: [ + { inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } }, + ], + }, + turnComplete: true, + }, + toolCall: { + functionCalls: [ + { + id: callId, + name: 'adjust_camera_zoom', + args: { direction: 'in', amount: 'little' }, + }, + ], + }, + }; +} + +// Local test observations wrap only injected device fakes and the actual +// scheduler methods, delegating unchanged. No frozen fixture file is modified. +function observePacketEffects(h) { + const played = []; + const narration = { audible: 0, toolStarted: 0, toolFinished: 0 }; + const createAudio = h.controller.createAudio; + h.controller.createAudio = (callbacks) => { + const audio = createAudio(callbacks); + const play = audio.play.bind(audio); + audio.play = (data, mimeType) => { + played.push({ audio, data, mimeType }); + return play(data, mimeType); + }; + return audio; + }; + for (const method of Object.keys(narration)) { + const invoke = h.controller.narration[method].bind(h.controller.narration); + h.controller.narration[method] = (...args) => { + narration[method]++; + return invoke(...args); + }; + } + return { played, narration }; +} + +function packetState(h) { + const c = h.controller; + return { + epoch: c.epoch, + inputGeneration: c.inputGeneration, + turn: c.turn, + completedTurns: c.completedTurns, + pendingTools: c.pendingTools, + modelComplete: c.modelComplete, + pendingCompletionStatus: c.pendingCompletionStatus, + postToolOutput: c.postToolOutput, + modelSpoke: c.modelSpoke, + assistant: c.transcripts.assistant, + replyPending: c.replyPending, + retiredReplyPending: c.retiredReplyPending, + typedText: c.typedIntent?.text ?? null, + typedResponseId: c.typedIntent?.responseId ?? null, + narrationTools: c.narration.state().tools, + narrationTimers: c.narration.state().pendingTimers, + ownedTimers: h.clock.timers.size, + }; +} + +const completions = (h) => + h.events.filter((event) => event.type === 'completion'); +const toolEvents = (h, callId) => + h.events.filter( + (event) => + ['action-call', 'action-result', 'progress'].includes(event.type) && + event.callId === callId, + ); + +function assertOldRemainderAbsent(h, observed) { + assert.equal(h.state.height, 1000); + assert.deepEqual(h.state.zooms, []); + assert.deepEqual(h.runnerCalls, []); + assert.deepEqual(toolEvents(h, OLD_CALL), []); + assert.deepEqual(observed.played, []); + assert.deepEqual(observed.narration, { + audible: 0, + toolStarted: 0, + toolFinished: 0, + }); + assert.deepEqual(completions(h), []); + assert.equal(h.controller.completedTurns, 0); + assert.equal(h.controller.pendingTools, 0); + assert.equal( + h.events.filter( + (event) => + event.type === 'transcript' && + event.role === 'assistant' && + event.final && + event.text === OLD_TEXT, + ).length, + 0, + ); + for (const socket of h.sockets) + assert.equal( + h.outputs(socket).some((output) => output.id === OLD_CALL), + false, + ); +} + +for (const mode of ['Stop', 'Stop then Start', 'typed successor']) { + test( + `assistant transcript observer ${mode} fences the rest of a coalesced native packet`, + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const observed = observePacketEffects(h); + const oldSocket = await h.start(); + const oldAudio = h.audios[0]; + h.activity(oldSocket); + const before = packetState(h); + let reached = false; + let restart; + let typedAccepted; + let afterObserver; + const remove = h.session.subscribe((event) => { + if ( + reached || + event.type !== 'transcript' || + event.role !== 'assistant' || + event.final || + event.text !== OLD_TEXT + ) + return; + reached = true; + if (mode === 'typed successor') { + typedAccepted = h.session.sendText('Zoom in for the typed successor'); + } else { + h.session.stop(); + if (mode === 'Stop then Start') restart = h.session.start(); + } + afterObserver = packetState(h); + }); + oldSocket.message(coalescedPacket()); + remove(); + assert.equal( + reached, + true, + 'the actual common transcript observer ran synchronously', + ); + // A retired reply may consume its own received terminal, but cannot + // complete or overwrite the successor. Stop/restart permits no old writes. + const afterPacketExpected = + mode === 'typed successor' + ? { + ...afterObserver, + retiredReplyPending: false, + ownedTimers: afterObserver.ownedTimers - 1, + } + : afterObserver; + assert.deepEqual(packetState(h), afterPacketExpected); + await h.controller.toolQueue; + await turnOfEventLoop(); + if (restart) await restart; + assertOldRemainderAbsent(h, observed); + + let successorSocket; + if (mode === 'typed successor') { + assert.equal(typedAccepted, true); + assert.equal( + h.controller.epoch, + before.epoch, + 'typed replacement shares the socket epoch', + ); + assert.ok(h.controller.inputGeneration > before.inputGeneration); + assert.ok(h.controller.turn > before.turn); + assert.deepEqual(packetState(h), afterPacketExpected); + assert.equal( + h.controller.typedIntent.text, + 'Zoom in for the typed successor', + ); + assert.equal(h.controller.replyPending, true); + assert.equal(h.controller.retiredReplyPending, false); + assert.equal(oldAudio.closed, false); + const typedMessages = oldSocket.sent.filter((message) => + message.clientContent?.turns?.some((turn) => + turn.parts?.some( + (part) => part.text === 'Zoom in for the typed successor', + ), + ), + ); + assert.equal(typedMessages.length, 1); + // The received packet already carried the old terminal. No extra + // provider boundary may be invented before testing the actual successor. + assert.equal(h.controller.retiredBoundaryWait, null); + assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true); + assert.equal(h.controller.replyPending, true); + assert.equal(h.controller.completedTurns, 0); + assert.deepEqual(completions(h), []); + successorSocket = oldSocket; + } else { + assert.equal(oldSocket.closed, true); + assert.equal(oldAudio.closed, true); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + assert.equal(h.clock.timers.size, 0); + if (mode === 'Stop') { + assert.equal(h.session.state, 'idle'); + successorSocket = await h.start(); + } else { + assert.equal(h.session.state, 'listening'); + assert.equal(h.sockets.length, 2); + assert.equal(h.audios.length, 2); + assert.notEqual(h.audios[1], oldAudio); + assert.equal(h.audios[1].closed, false); + assert.equal(h.audios[1].pending, false); + successorSocket = h.sockets[1]; + } + h.activity(successorSocket); + } + await h.tool('owned-after-packet-cancel', successorSocket); + assertSuccessfulZoom( + h, + successorSocket, + 'owned-after-packet-cancel', + 1000, + ); + assert.equal(h.state.zooms.length, 1); + assert.equal(h.runnerCalls.length, 1); + assert.equal( + h.outputs(successorSocket).some((output) => output.id === OLD_CALL), + false, + ); + h.complete(successorSocket); + assert.equal(completions(h).length, 1); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + }, + ); +} + +test( + 'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply', + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const observed = observePacketEffects(h); + const ws = await h.start(); + h.activity(ws); + ws.message(coalescedPacket('ordinary-coalesced')); + assert.equal( + h.controller.completedTurns, + 1, + 'first generation terminal is counted once', + ); + assert.equal(h.controller.pendingTools, 1); + assert.deepEqual( + observed.played.map(({ data }) => data), + [OLD_PCM], + ); + assert.equal(observed.played[0].audio, h.audios[0]); + assert.deepEqual(completions(h), [], 'tool and playback are still pending'); + await h.controller.toolQueue; + assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000); + assert.equal(h.controller.pendingTools, 0); + assert.equal(observed.narration.toolStarted, 1); + assert.equal(observed.narration.toolFinished, 1); + assert.equal(h.controller.narration.state().tools, 0); + h.audios[0].drain(); + assert.equal( + h.controller.modelComplete, + false, + 'function response waits for the post-tool generation', + ); + assert.equal(h.controller.completedTurns, 1); + assert.deepEqual( + completions(h), + [], + 'draining first-generation PCM cannot finish the tool reply', + ); + + // The accepted contract has two generations: the function response prompts + // a fresh model reply. This boundary is intentional, not a stale-packet fix. + ws.message({ + serverContent: { + outputTranscription: { text: 'The requested zoom is complete.' }, + modelTurn: { + parts: [ + { + inlineData: { + data: POST_TOOL_PCM, + mimeType: 'audio/pcm;rate=24000', + }, + }, + ], + }, + turnComplete: true, + }, + }); + assert.equal(h.controller.completedTurns, 2); + assert.deepEqual( + observed.played.map(({ data }) => data), + [OLD_PCM, POST_TOOL_PCM], + ); + assert.deepEqual( + completions(h), + [], + 'post-tool playback must drain before completion', + ); + h.audios[0].drain(); + assert.equal(completions(h).length, 1); + assert.equal(completions(h)[0].status, 'completed'); + assert.equal(completions(h)[0].spoken, true); + assert.equal(completions(h)[0].audioDrained, true); + assert.equal(h.controller.pendingCompletionStatus, null); + assert.equal(h.controller.narration.state().tools, 0); + assert.equal(h.controller.narration.state().pendingTimers, 0); + assert.equal(h.clock.timers.size, 0); + assert.equal(observed.narration.audible, 2); + assert.equal(h.runnerCalls.length, 1); + assert.equal(h.state.zooms.length, 1); + assert.equal(h.outputs(ws).length, 1); + h.audios[0].drain(); + assert.equal( + completions(h).length, + 1, + 'a duplicate device drain cannot complete twice', + ); + }, +); + +// Real native message-type fields are separate packets here; no extra old +// completion is inserted before the accepted successor. +const BOUNDARY_OLD = 'Prior reply caption.'; +const BOUNDARY_NEXT = 'Zoom in for the typed successor'; +const boundaryZoomCall = (id) => ({ + toolCall: { + functionCalls: [ + { + id, + name: 'adjust_camera_zoom', + args: { direction: 'in', amount: 'little' }, + }, + ], + }, +}); +function boundarySnapshot(h) { + const c = h.controller; + return { + retiredReplyPending: c.retiredReplyPending, + retiredBoundaryWait: Boolean(c.retiredBoundaryWait), + replyPending: c.replyPending, + typedText: c.typedIntent?.text ?? null, + completions: h.events.filter((event) => event.type === 'completion').length, + }; +} + +for (const mode of [ + 'partial assistant transcript', + 'input transcript', + 'interruption event', + 'listening event', + 'final assistant transcript control', +]) { + test( + `received native terminal preserves typed successor after ${mode}`, + { concurrency: false, timeout: 5000 }, + async (t) => { + const h = nativeIntegrationFixture(t); + const ws = await h.start(); + h.activity(ws); + let oldDispatch, gate; + const interruptionMode = + mode === 'interruption event' || mode === 'listening event'; + if (interruptionMode) { + h.session.sendText('Prior zoom request'); + gate = h.holdResult('already-committed-old-tool'); + oldDispatch = h.tool('already-committed-old-tool', ws); + await gate.ready; + } else if (mode === 'input transcript') { + ws.message({ + serverContent: { + outputTranscription: { text: 'Previously emitted output' }, + }, + }); + } + let entered = 0, + accepted; + const remove = h.session.subscribe((event) => { + if (entered) return; + const matches = + mode === 'interruption event' + ? event.type === 'interruption' && event.reason === 'user-speech' + : mode === 'listening event' + ? event.type === 'state' && event.state === 'listening' + : mode === 'input transcript' + ? event.type === 'transcript' && + event.role === 'user' && + event.text === 'Old input caption' + : event.type === 'transcript' && + event.role === 'assistant' && + event.text === BOUNDARY_OLD && + event.final === + (mode === 'final assistant transcript control'); + if (!matches) return; + entered++; + accepted = h.session.sendText(BOUNDARY_NEXT); + }); + if (interruptionMode) + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + else + ws.message({ + serverContent: { + ...(mode === 'input transcript' + ? { inputTranscription: { text: 'Old input caption' } } + : {}), + outputTranscription: { text: BOUNDARY_OLD }, + turnComplete: true, + }, + }); + remove(); + if (oldDispatch) { + await oldDispatch; + gate.release(); + } + await turnOfEventLoop(); + const afterPacket = boundarySnapshot(h); + const beforeSuccessorHeight = h.state.height; + // No extra synthetic old boundary. This is the next reply to the accepted text. + ws.message({ + serverContent: { + outputTranscription: { text: 'Handling your new zoom.' }, + }, + }); + ws.message(boundaryZoomCall('typed-successor-call')); + await h.controller.toolQueue; + await turnOfEventLoop(); + ws.message({ serverContent: { turnComplete: true } }); + const afterSuccessor = boundarySnapshot(h); + assert.equal(entered, 1); + assert.equal(accepted, true); + assert.equal( + afterPacket.retiredReplyPending, + false, + 'received old terminal must settle its retirement', + ); + assert.equal(afterPacket.retiredBoundaryWait, false); + assert.equal(afterPacket.typedText, BOUNDARY_NEXT); + assert.equal( + h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call') + .length, + 1, + 'accepted typed successor must reach the actual runner', + ); + assert.equal(h.state.height, beforeSuccessorHeight * 0.75); + assert.equal(afterSuccessor.completions, 1); + assert.equal(afterSuccessor.replyPending, false); + assert.equal( + h.controller.cancelledCalls.has('typed-successor-call'), + false, + ); + }, + ); +} + +for (const terminalAlreadyApplied of [false, true]) { + test( + `typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`, + options, + async (t) => { + const h = nativeIntegrationFixture(t); + const ws = await h.start(); + h.activity(ws); + let entered = false; + const remove = h.session.subscribe((event) => { + if ( + entered || + event.type !== 'transcript' || + event.role !== 'assistant' || + event.text !== 'Boundary control' || + event.final !== terminalAlreadyApplied + ) + return; + entered = true; + assert.equal(h.session.sendText('First typed replacement'), true); + if (terminalAlreadyApplied) + assert.equal(h.session.sendText('Second typed replacement'), true); + }); + ws.message({ + serverContent: { + outputTranscription: { text: 'Boundary control' }, + ...(terminalAlreadyApplied ? { turnComplete: true } : {}), + }, + }); + remove(); + assert.equal(entered, true); + // With no old terminal received, or when that terminal was consumed + // before the observer, a real predecessor boundary is still outstanding. + assert.equal(h.controller.retiredReplyPending, true); + assert.ok(h.controller.retiredBoundaryWait); + assert.equal(completions(h).length, 0); + assert.equal(h.runnerCalls.length, 0); + ws.message({ serverContent: { turnComplete: true } }); + assert.equal(h.controller.retiredReplyPending, false); + assert.equal(h.controller.retiredBoundaryWait, null); + await h.tool('after-real-pending-boundary', ws); + assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000); + h.complete(ws); + assert.equal(completions(h).length, 1); + }, + ); +} diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js new file mode 100644 index 000000000..04a573f76 --- /dev/null +++ b/src/voice/geminiSession.js @@ -0,0 +1,2511 @@ +import { t, translateKnown } from '../i18n/index.js'; +import { msg } from '../i18n/marker.js'; +import { createGeminiAudio } from './geminiAudio.js'; +import { RealtimeInput } from './realtimeInput.js'; +import { RealtimeRadio } from './realtimeRadio.js'; +import { captureViewportImage } from './realtimeViewport.js'; +import { createNarrationScheduler } from './narration.js'; +import { createEarcon } from './earcon.js'; +import { narrationLabel } from './speech.js'; +import { waitForSignal } from '../sources/requestDeadline.js'; +import { readResponseJsonCapped } from '../sources/httpBody.js'; +import { readLayerLifecycleSummary } from './layerSummary.js'; +import { + POINTER_OPEN_MIC_RECENT_MS, + pointerAnnouncement, + pointerTurnContext, + pointerTurnEvent, +} from './pointerContext.js'; +import { + hasStructuredViewIdentity, + shouldSendViewportImage, + shouldStopVoiceAfterRadioTool, + responseInstructionForToolResult, +} from './realtimeProtocol.js'; + +const SOCKET_PATH = + /^\/ws\/google\.ai\.generativelanguage\.v1(?:alpha|beta)\.GenerativeService\.BidiGenerateContentConstrained$/; +const MAX_BUFFERED_BYTES = 256 * 1024; +const MAX_CONTROL_BYTES = 1024 * 1024; +const MAX_CONTROL_MESSAGES = 64; +const CONTROL_PRESSURE_TIMEOUT_MS = 5_000; +const CONTROL_POLL_MS = 25; +const MAX_GO_AWAY_MS = 10 * 60 * 1000; +const MAX_MESSAGE_LENGTH = 3 * 1024 * 1024; +const MAX_TOOL_CALLS = 16; +const MAX_SEEN_CALLS = 4096; +const MAX_CALL_ID_LENGTH = 1024; +const MAX_MAP_EVENTS = 16; +const MAX_MAP_CONTEXT_LENGTH = 16000; + +/** Only an ephemeral token can be attached to the fixed Google Live endpoint. */ +export function geminiSocketUrl({ token, websocketUrl }) { + if ( + typeof token !== 'string' || + !token.startsWith('auth_tokens/') || + token.length > 8192 + ) + throw new Error( + msg('Gemini token response did not include an ephemeral credential'), + ); + let url; + try { + url = new URL(websocketUrl); + } catch { + throw new Error(msg('Invalid Gemini connection endpoint')); + } + if ( + url.protocol !== 'wss:' || + url.hostname !== 'generativelanguage.googleapis.com' || + url.port || + url.username || + url.password || + url.search || + url.hash || + !SOCKET_PATH.test(url.pathname) + ) + throw new Error(msg('Invalid Gemini connection endpoint')); + url.searchParams.set('access_token', token); + return url.href; +} + +// Known local-server failures are translated; unexpected diagnostics stay intact. +function tokenErrorMessage(error) { + switch (error) { + case msg('Gemini voice could not connect. Try again.'): + case msg('Gemini voice requires a same-origin app request'): + case msg('Add a Gemini API key in Provider Settings'): + case msg('The Gemini Live model configuration is invalid'): + case msg('Gemini voice is starting too often. Try again shortly.'): + case msg('Gemini quota is unavailable. Try again later.'): + case msg('Gemini access was refused. Check the API key and model access.'): + case msg('Gemini Live is unavailable for the configured model.'): + case msg('Token request is too large'): + case msg('Use an empty JSON token request'): + case msg('Method not allowed'): + return t(error); + default: + return error; + } +} + +function validCallId(id) { + return ( + typeof id === 'string' && + id.length > 0 && + id.length <= MAX_CALL_ID_LENGTH && + id.trim().length > 0 && + !/[\u0000-\u001f\u007f]/u.test(id) + ); +} + +function cancelled() { + return new DOMException('Voice operation cancelled', 'AbortError'); +} + +// Match the existing OpenAI/Radio classification; transport ownership stays local. +function classifyGeminiTool(name, args = {}) { + const radioControl = name === 'control_radio'; + const radioVisibility = + name === 'set_layer_visibility' && args.layerId === 'radio'; + const radioAction = radioControl + ? String(args.action || '').toLowerCase() + : null; + const authorityDomain = + radioVisibility || ['enable', 'disable'].includes(radioAction) + ? 'visibility' + : radioAction === 'status' + ? 'query' + : radioControl + ? 'playback' + : null; + const stronger = + (radioControl && ['disable', 'pause', 'stop'].includes(radioAction)) || + (radioVisibility && args.enabled === false); + return { + radioControl, + radioVisibility, + authorityDomain, + stronger, + abortScope: + radioAction === 'disable' || radioVisibility ? 'all' : 'playback', + }; +} + +/** Gemini owns its socket/media; the common session owns action authorization. */ +export class GeminiSessionController { + constructor({ + emit, + runAction, + signal, + ui, + radioLayer = null, + dataManager = null, + pointer = null, + tokenEndpoint = '/api/gemini/token', + startupTimeoutMs = 30_000, + inputBoundaryTimeoutMs = 5_000, + fetchImpl = (...args) => fetch(...args), + WebSocketClass = globalThis.WebSocket, + getUserMedia = (settings) => navigator.mediaDevices.getUserMedia(settings), + createAudio = createGeminiAudio, + createInput = (settings) => new RealtimeInput(settings), + createNarration = createNarrationScheduler, + createProgressEarcon = createEarcon, + captureViewport = captureViewportImage, + now = () => Date.now(), + setTimer = setTimeout, + clearTimer = clearTimeout, + }) { + Object.assign(this, { + emit, + runAction, + signal, + ui, + tokenEndpoint, + startupTimeoutMs, + inputBoundaryTimeoutMs, + fetchImpl, + WebSocketClass, + getUserMedia, + createAudio, + captureViewport, + pointer, + radioLayer, + dataManager, + now, + // Browser timer functions reject the controller as their receiver. + setTimer: (...args) => setTimer(...args), + clearTimer: (...args) => clearTimer(...args), + }); + this.status = 'idle'; + this.model = null; + this.epoch = 0; + this.turn = 0; + this.inputGeneration = 0; + this.turnAbort = new AbortController(); + this.disposed = false; + this.socket = null; + this.channel = null; + this.stream = null; + this.audio = null; + this.abort = null; + this.tools = new Map(); + this.radioPlaybackCalls = new WeakMap(); + this.seenCalls = new Set(); + this.cancelledCalls = new Set(); + this.toolQueue = Promise.resolve(); + this.pendingTools = 0; + this.modelComplete = false; + this.pendingCompletionStatus = null; + this.postToolOutput = false; + this.stopAfterReply = false; + this.modelSpoke = false; + this.transcripts = { assistant: '' }; + this.pendingMapEvents = []; + this.userTurnPending = false; + // Live output has no response ID. A locally retired reply keeps ownership + // of incoming output until its ordered server turnComplete boundary. + this.replyPending = false; + // A reply Gemini started without a request; it gets no tool authority. + this.unrequestedReply = false; + this.automaticTurnOpen = false; + this.automaticActivityOwner = null; + this.retiredAutomaticActivityOwner = null; + this.automaticActivitySequence = 0; + this.retiredReplyPending = false; + this.deferredInput = null; + this.retiredBoundaryWait = null; + this.forwardedActivityOpen = false; + this.forwardedCaptureId = null; + this.captureGeneration = 0; + this.captureEnabled = false; + this.pttPlaybackSuppressed = false; + this.microphoneRequested = false; + this.captureOwner = null; + this.userActivityOpen = false; + this.pointerSentKey = null; + this.pointerAnnounced = null; + this.pendingPointerContext = null; + this.typedIntent = null; + this.toolSequence = 0; + this.radioStrongerCommit = null; + this.intentSequence = 0; + this.intentItemId = null; + this.intentResponseId = null; + this.intentCaptureId = null; + this.controlQueue = []; + this.controlBytes = 0; + this.controlTimer = null; + this.goAway = null; + this.droppedAudioChunks = 0; + this.deferredAudioDrops = { nonPttOwner: 0, closedActivity: 0 }; + this.droppedImages = 0; + this.lastClose = null; + this.lastError = null; + this.completedTurns = 0; + this.lastAudio = null; + this.lastViewportAt = -Infinity; + this.input = createInput({ + readUi: () => ui, + readStream: () => this.stream, + readStatus: () => this.status, + operations: { + isActive: () => this.isActive(), + start: (settings) => this.start(settings), + setStatus: (...args) => this.setStatus(...args), + pauseRadioForVoice: () => this.radio.pauseRadioForVoice(), + bargeIn: () => this.claimPttPlayback(), + holdPointer: () => { + this.heldPointerSnapshot = this.pointer?.hold('keydown'); + }, + releasePointerHold: () => this.pointer?.releaseHold(), + beginPointerTurn: (at) => + this.input.pushToTalkMode && at === 'keydown' + ? this.heldPointerSnapshot + : this.beginPointerTurn(at), + mayClaimSpeaker: () => this.radio.mayVoiceClaimSpeaker(), + }, + }); + const setMicrophoneEnabled = this.input.setMicrophoneEnabled.bind( + this.input, + ); + this.applyMicrophoneEnabled = (enabled) => { + enabled = Boolean(enabled) && !this.goAway; + setMicrophoneEnabled(enabled); + if (enabled && !this.captureEnabled) { + // The physical gate owns this generation, independently of tool turns. + this.captureOwner = { + captureId: ++this.captureGeneration, + context: null, + snapshot: this.heldPointerSnapshot, + retired: false, + admitted: false, + settled: false, + }; + } + this.captureEnabled = enabled; + const metadata = this.captureOwner + ? { captureId: this.captureOwner.captureId } + : null; + this.audio?.setMicrophoneEnabled(enabled, metadata); + if (enabled && this.channel?.readyState === 'open') { + if (this.input.pushToTalkMode) this.radio.cancelRadioHandoff(); + else this.noteUserActivity(); + } + }; + this.input.setMicrophoneEnabled = (enabled) => { + this.microphoneRequested = Boolean(enabled) && !this.goAway; + this.applyMicrophoneEnabled(this.microphoneRequested); + }; + this.radio = new RealtimeRadio({ + readRadioLayer: () => radioLayer, + readDataManager: () => dataManager, + readChannel: () => this.channel, + readUserTurnPending: () => this.userTurnPending, + readSessionId: () => `gemini-${this.epoch}`, + operations: { + abortTools: () => this.abortTools(), + isActive: () => this.isActive(), + stop: (settings) => this.stop(settings), + setStatus: (...args) => this.setStatus(...args), + // The shared Radio owner already surfaces a failed start in the UI. + queueResponseCreate: () => {}, + debugLog: () => {}, + }, + }); + this.earcon = createProgressEarcon({ + isMicrophoneMuted: () => + Boolean( + this.isActive() && + this.channel?.readyState === 'open' && + this.input.pushToTalkMode && + !this.input.pushToTalkKeyHeld && + this.stream && + this.stream.getAudioTracks().every((track) => !track.enabled) && + !this.audio?.pending && + this.radio.mayVoiceClaimSpeaker(), + ), + }); + this.narration = createNarration({ + // Live clientContent interrupts generation, even with turnComplete:false. + // Reuse shared progress timing and muted-PTT earcons without injecting a + // second conversation turn. The model's own preamble supplies speech. + speak: () => false, + earcon: this.earcon, + canSpeak: () => false, + }); + this.radio.observe(); + this.lifetimeAbort = () => this.stop({ removeUi: true }); + signal?.addEventListener('abort', this.lifetimeAbort, { once: true }); + } + + get spaceKeyHeld() { + return this.input.spaceKeyHeld; + } + isActive() { + return !['idle', 'error'].includes(this.status); + } + owns(epoch) { + return ( + !this.disposed && epoch === this.epoch && !this.abort?.signal.aborted + ); + } + + setStatus(status, detail, recovery) { + this.status = status; + if (status === 'listening' && this.input.pushToTalkMode) + detail = this.input.pushToTalkKeyHeld + ? msg('Release Space to send') + : msg('Hold Space to talk'); + const event = { + type: 'state', + state: status, + detail: translateKnown(detail), + }; + if (recovery) event.recovery = translateKnown(recovery); + this.emit(event); + this.input.updateVoiceButtonLabel(); + if (['idle', 'connecting', 'error'].includes(status)) + this.input.setVoiceSpeaker('idle'); + } + + send( + message, + { + beforeSetup = false, + captureId = null, + onSent = null, + durable = false, + } = {}, + ) { + if ( + this.socket?.readyState !== 1 || + (!beforeSetup && this.channel?.readyState !== 'open') + ) + return false; + const body = JSON.stringify(message); + const bytes = new TextEncoder().encode(body).byteLength; + if (bytes > MAX_MESSAGE_LENGTH) + throw new Error(msg('Voice command context is too large')); + const optional = Boolean( + message.realtimeInput?.audio || message.realtimeInput?.video, + ); + if (optional) { + this.pruneControls(); + if ( + this.goAway || + this.controlQueue.length || + this.socket.bufferedAmount + bytes > MAX_BUFFERED_BYTES + ) { + if (message.realtimeInput.audio) this.droppedAudioChunks++; + else this.droppedImages++; + return false; + } + this.socket.send(body); + onSent?.(); + return true; + } + this.pruneControls(); + const entry = { + epoch: this.epoch, + turn: this.turn, + captureId, + durable, + message, + body, + bytes, + onSent, + deadline: this.now() + CONTROL_PRESSURE_TIMEOUT_MS, + }; + if (!this.controlQueue.length && this.canSendControl(entry)) { + this.writeControl(entry); + return true; + } + if ( + bytes > MAX_CONTROL_BYTES || + this.controlBytes + bytes > MAX_CONTROL_BYTES || + this.controlQueue.length >= MAX_CONTROL_MESSAGES + ) + throw new Error( + msg( + 'Gemini reliable transport exceeded its queue limit. Start voice again.', + ), + ); + this.controlQueue.push(entry); + this.controlBytes += bytes; + this.scheduleControls(); + return true; + } + + controlCurrent(entry) { + if (!this.owns(entry.epoch)) return false; + if (entry.durable) return true; + const input = entry.message.realtimeInput; + if ( + entry.message.toolResponse?.functionResponses?.some((call) => + this.cancelledCalls.has(call.id), + ) + ) + return false; + if (input?.activityEnd) + return ( + (this.forwardedActivityOpen && + this.forwardedCaptureId === entry.captureId) || + (this.acceptsCapture({ captureId: entry.captureId }) && + this.controlQueue.some( + (pending) => + pending.message.realtimeInput?.activityStart && + pending.captureId === entry.captureId && + this.controlCurrent(pending), + )) + ); + if (entry.captureId !== null) + return this.acceptsCapture({ captureId: entry.captureId }); + return entry.turn === this.turn; + } + + pruneControls() { + this.controlQueue = this.controlQueue.filter((entry) => + this.controlCurrent(entry), + ); + this.controlBytes = this.controlQueue.reduce( + (total, entry) => total + entry.bytes, + 0, + ); + if (!this.controlQueue.length) { + if (this.controlTimer !== null) this.clearTimer(this.controlTimer); + this.controlTimer = null; + } + } + + canSendControl(entry) { + return ( + entry.bytes <= MAX_CONTROL_BYTES && + (this.socket.bufferedAmount === 0 || + this.socket.bufferedAmount + entry.bytes <= MAX_BUFFERED_BYTES) + ); + } + + writeControl(entry) { + this.socket.send(entry.body); + if (!this.controlCurrent(entry)) return; + const input = entry.message.realtimeInput; + // Only markers actually written to the socket own its physical activity. + if (input?.activityStart) { + this.forwardedActivityOpen = true; + this.forwardedCaptureId = entry.captureId; + this.beginSpeechIntent(entry.captureId); + if (!this.controlCurrent(entry)) return; + } + if (input?.activityEnd) { + this.forwardedActivityOpen = false; + this.forwardedCaptureId = null; + if (this.intentCaptureId === entry.captureId) this.bindSpeechIntent(); + } + entry.onSent?.(); + } + + scheduleControls() { + if (this.controlTimer !== null || !this.controlQueue.length) return; + const epoch = this.epoch; + this.controlTimer = this.setTimer( + () => { + if (!this.owns(epoch)) return; + this.controlTimer = null; + try { + this.flushControls(); + } catch (error) { + this.fail(error); + } + }, + Math.min( + CONTROL_POLL_MS, + Math.max(0, this.controlQueue[0].deadline - this.now()), + ), + ); + this.controlTimer?.unref?.(); + } + + flushControls() { + this.pruneControls(); + while (this.controlQueue.length) { + const entry = this.controlQueue[0]; + if (this.now() >= entry.deadline) + throw new Error( + msg('Gemini reliable transport did not drain. Start voice again.'), + ); + if (!this.canSendControl(entry)) break; + this.controlQueue.shift(); + this.controlBytes -= entry.bytes; + if (this.controlCurrent(entry)) this.writeControl(entry); + } + this.scheduleControls(); + if (!this.controlQueue.length) this.finishPlayback(); + } + + clearControls() { + if (this.controlTimer !== null) this.clearTimer(this.controlTimer); + this.controlTimer = null; + this.controlQueue = []; + this.controlBytes = 0; + } + + async requestToken(signal, pushToTalk = false) { + signal.throwIfAborted(); + let acquiredResponse; + const acquisition = Promise.resolve( + this.fetchImpl(this.tokenEndpoint, { + method: 'POST', + cache: 'no-store', + redirect: 'error', + signal, + headers: { 'Content-Type': 'application/json' }, + body: pushToTalk ? '{"inputMode":"push-to-talk"}' : '{}', + }), + ); + void acquisition.then( + (response) => { + acquiredResponse = response; + if (signal.aborted) void response.body?.cancel().catch(() => {}); + }, + () => {}, + ); + try { + const response = await waitForSignal(acquisition, signal); + signal.throwIfAborted(); + const data = await readResponseJsonCapped(response, 256 * 1024, signal); + signal.throwIfAborted(); + if (!response.ok) + throw new Error( + typeof data?.error === 'string' + ? tokenErrorMessage(data.error) + : t('Gemini token request failed (HTTP {status})', { + status: response.status, + }), + ); + if ( + typeof data?.model !== 'string' || + !/^gemini-[a-z0-9.-]+$/.test(data.model) + ) + throw new Error( + msg('Gemini token response did not include a valid model'), + ); + if ( + !data.config || + typeof data.config !== 'object' || + Array.isArray(data.config) + ) + throw new Error( + msg('Gemini token response did not include session settings'), + ); + const manual = + data.config.realtimeInputConfig?.automaticActivityDetection?.disabled; + if ( + pushToTalk ? manual !== true : manual !== undefined && manual !== false + ) + throw new Error( + msg('Gemini session settings do not match the input mode'), + ); + const url = geminiSocketUrl(data); + return { ...data, url }; + } finally { + if (signal.aborted) void acquiredResponse?.body?.cancel().catch(() => {}); + } + } + + async start({ pushToTalk = false } = {}) { + if (this.disposed || this.isActive() || this.signal?.aborted) return; + const held = pushToTalk && this.input.pushToTalkKeyHeld; + const spaceHeld = this.input.spaceKeyHeld; + this.stop({ preserveStatus: true }); + this.input.pushToTalkMode = pushToTalk; + this.input.pushToTalkKeyHeld = held; + this.input.spaceKeyHeld = spaceHeld; + this.abort = new AbortController(); + const epoch = this.epoch; + const permissionSignal = AbortSignal.any([ + this.abort.signal, + ...(this.signal ? [this.signal] : []), + ]); + let signal = permissionSignal; + this.lastError = null; + this.lastAudio = null; + this.completedTurns = 0; + let acquiredStream = null; + let releasedStream = null; + const releaseStream = (stream) => { + if (!stream || stream === releasedStream) return; + releasedStream = stream; + stream.getTracks().forEach((track) => track.stop()); + }; + try { + this.setStatus('connecting', msg('Requesting microphone')); + if (!this.owns(epoch)) return; + signal.throwIfAborted(); + this.radio.pauseRadioForVoice(); + if (!this.owns(epoch)) return; + // The worklet serializes start, PCM and flushed end notifications. A + // keydown must not send a new start ahead of the previous queued end. + let activityCaptureId = null; + const audio = this.createAudio({ + pushToTalk, + onAudioStart: (metadata) => { + if ( + !this.owns(epoch) || + this.goAway || + !this.acceptsCapture(metadata) + ) + return; + if (!pushToTalk || activityCaptureId === metadata.captureId) return; + this.captureOwner.admitted = true; + if ( + !this.interrupt('push-to-talk', { + keepPlayback: !this.pttPlaybackSuppressed, + }) + ) + return; + if (!this.owns(epoch) || !this.acceptsCapture(metadata)) return; + const activityGeneration = this.inputGeneration; + const activityCurrent = () => + this.owns(epoch) && + this.inputGeneration === activityGeneration && + this.acceptsCapture(metadata); + this.beginPointerTurn( + 'speech_start', + this.captureOwner.snapshot, + activityCurrent, + ); + if (!activityCurrent()) return; + const context = this.pendingPointerContext; + this.captureOwner.context = context; + if (!this.sendPointerContext(context, metadata)) { + if (this.owns(epoch)) + this.fail( + new Error( + msg('Gemini could not submit this input. Start voice again.'), + ), + ); + return; + } + if (!activityCurrent()) return; + const accepted = this.sendUserInput( + { realtimeInput: { activityStart: {} } }, + metadata, + ); + if (!activityCurrent()) return; + // Quiet context and an empty hold have no response to retire later. + if (accepted) activityCaptureId = metadata.captureId; + else if (this.owns(epoch)) + this.fail( + new Error( + msg('Gemini could not submit this input. Start voice again.'), + ), + ); + }, + onAudio: (chunk, metadata) => { + if (!this.owns(epoch) || !this.acceptsCapture(metadata)) return; + if (!pushToTalk || activityCaptureId === metadata.captureId) { + const sent = this.sendUserInput( + { + realtimeInput: { audio: chunk }, + }, + metadata, + ); + if (sent && pushToTalk) this.replyPending = true; + } + }, + onAudioEnd: (metadata) => { + if (!this.owns(epoch) || !this.validCaptureMetadata(metadata)) return; + if (pushToTalk) { + if (this.captureOwner?.captureId === metadata.captureId) + this.captureOwner.settled = true; + if (activityCaptureId !== metadata.captureId) { + this.finishPlayback(); + return; + } + this.sendUserInput( + { realtimeInput: { activityEnd: {} } }, + metadata, + ); + if (this.acceptsCapture(metadata)) this.narration.userTurnEnded(); + activityCaptureId = null; + } else if (this.acceptsCapture(metadata)) { + this.send({ realtimeInput: { audioStreamEnd: true } }); + } + }, + onAudioDiscard: (metadata) => { + if ( + !this.owns(epoch) || + !pushToTalk || + !this.validCaptureMetadata(metadata) || + this.captureOwner?.captureId !== metadata.captureId + ) + return; + this.captureOwner.settled = true; + if (!this.acceptsCapture(metadata)) { + // New typed input owns completion; its retired capture cannot + // revoke that request or leave its presentation waiting. + this.finishPlayback(); + return; + } + if ( + activityCaptureId === metadata.captureId || + this.forwardedCaptureId === metadata.captureId || + this.deferredInput?.captureId === metadata.captureId + ) + return; + // No provider turn was submitted or retired. Keep an older request's + // tool/completion ownership; the physical claim still revokes Radio. + this.radio.cancelRadioHandoff(); + if (!this.audio?.pending) this.input.setVoiceSpeaker('idle'); + this.finishPlayback(); + }, + onDrain: () => { + if (this.owns(epoch)) { + this.narration.quiet(); + this.finishPlayback(); + } + }, + onError: (error) => { + if (this.owns(epoch)) this.fail(error); + }, + }); + if (!this.owns(epoch) || signal.aborted) { + void Promise.resolve(audio.close()).catch(() => {}); + return; + } + this.audio = audio; + // Invoke resume in the initiating gesture, but do not mint the short-lived + // admission credential while an uncancellable permission prompt is open. + const initialized = Promise.resolve(audio.initialize(permissionSignal)); + void initialized.catch(() => {}); + if (!this.owns(epoch)) return; + signal.throwIfAborted(); + const mediaRequest = Promise.resolve( + this.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + }, + }), + ); + // Permission prompts cannot be cancelled. A later grant still belongs + // to this attempt and must release its own tracks after stop/timeout. + void mediaRequest.then( + (stream) => { + if (!this.owns(epoch) || signal.aborted) releaseStream(stream); + }, + () => {}, + ); + acquiredStream = await waitForSignal(mediaRequest, signal); + if (!this.owns(epoch)) { + releaseStream(acquiredStream); + return; + } + signal.throwIfAborted(); + this.stream = acquiredStream; + acquiredStream.getAudioTracks().forEach((track) => { + track.enabled = false; + }); + signal = AbortSignal.any([ + permissionSignal, + AbortSignal.timeout(this.startupTimeoutMs), + ]); + this.setStatus('connecting', msg('Connecting to Gemini')); + if (!this.owns(epoch)) return; + signal.throwIfAborted(); + const startup = Promise.all([ + initialized, + this.requestToken(signal, pushToTalk), + ]); + void startup.catch(() => {}); + const [, credential] = await waitForSignal(startup, signal); + if (!this.owns(epoch)) return; + this.model = credential.model; + await this.connect(credential, epoch, signal); + if (!this.owns(epoch)) return; + signal.throwIfAborted(); + const enabled = + !this.input.pushToTalkMode || this.input.pushToTalkKeyHeld; + this.input.setMicrophoneEnabled(enabled); + audio.connectMicrophone( + acquiredStream, + enabled, + this.captureOwner ? { captureId: this.captureOwner.captureId } : null, + ); + this.input.startVoiceVisualizer(acquiredStream); + this.input.startAssistantVoiceVisualizer(audio.outputStream); + this.setStatus('listening', msg('Ask or command')); + } catch (error) { + if (acquiredStream && acquiredStream !== this.stream) + releaseStream(acquiredStream); + if (this.owns(epoch)) this.fail(error); + } + } + + connect(credential, epoch, signal) { + return new Promise((resolve, reject) => { + signal.throwIfAborted(); + if (!this.owns(epoch)) return reject(cancelled()); + const socket = new this.WebSocketClass(credential.url); + socket.binaryType = 'arraybuffer'; + this.socket = socket; + const channel = { readyState: 'connecting' }; + this.channel = channel; + let connected = false; + const current = () => this.owns(epoch) && this.socket === socket; + const abort = () => { + signal.removeEventListener('abort', abort); + reject(signal.reason || cancelled()); + }; + signal.addEventListener('abort', abort, { once: true }); + socket.onopen = () => { + if (!current()) return; + try { + this.send( + { + setup: { + ...credential.config, + model: `models/${credential.model}`, + }, + }, + { beforeSetup: true }, + ); + } catch (error) { + reject(error); + } + }; + socket.onmessage = ({ data }) => { + if (!current()) return; + try { + const text = + typeof data === 'string' ? data : new TextDecoder().decode(data); + if (text.length > MAX_MESSAGE_LENGTH) + throw new Error(msg('Gemini returned an oversized message')); + const message = JSON.parse(text); + if (message.setupComplete) { + if (connected) return; + connected = true; + channel.readyState = 'open'; + signal.removeEventListener('abort', abort); + resolve(); + } + this.handleMessage(message, epoch); + } catch (error) { + if (!connected) reject(error); + else if (current()) this.fail(error); + } + }; + socket.onerror = () => { + const error = new Error( + msg( + 'Gemini Live connection failed. Check network access, then start voice again.', + ), + ); + if (!current()) return; + if (!connected) reject(error); + else this.fail(error); + }; + socket.onclose = (event) => { + if (!current()) return; + const code = + Number.isInteger(event?.code) && + event.code >= 1000 && + event.code <= 4999 + ? event.code + : null; + this.lastClose = { + code, + category: + code === 1000 + ? 'normal' + : code === 1008 + ? 'policy' + : code === 1011 + ? 'server' + : 'connection', + }; + const error = new Error( + connected + ? msg( + 'Gemini Live connection ended. Start voice again for a new conversation.', + ) + : msg( + 'Gemini Live could not start. Start voice again to request a fresh session.', + ), + ); + error.recovery = connected + ? msg('Start voice again for a fresh token and a new conversation.') + : msg('Start voice again to request a fresh session credential.'); + if (!connected) reject(error); + else this.fail(error); + }; + }); + } + + fail(error, recovery = error?.recovery) { + const detail = + error?.name === 'NotAllowedError' + ? msg( + 'Microphone permission was denied. Allow microphone access and try again.', + ) + : error?.name === 'TimeoutError' + ? msg('Gemini connection timed out. Try again.') + : error?.message || msg('Gemini voice could not start'); + this.lastError = detail; + this.stop({ preserveStatus: true }); + this.setStatus('error', detail, recovery); + } + + abortTools() { + const resources = this.turnAbort; + const tools = [...this.tools.values()]; + this.turnAbort = new AbortController(); + this.turn++; + this.typedIntent = null; + this.radioStrongerCommit = null; + this.pruneControls(); + this.tools.clear(); + this.radio?.clearTools(); + this.pendingTools = 0; + this.toolQueue = Promise.resolve(); + this.narration?.cancel(); + resources.abort(); + for (const entry of tools) entry.abort(); + } + + noteUserActivity() { + this.narration?.cancel(); + this.modelSpoke = false; + this.userTurnPending = true; + this.radio.cancelRadioHandoff(); + } + + /** A physical hold silences playback; only admitted speech retires a turn. */ + claimPttPlayback() { + if ( + this.channel?.readyState !== 'open' || + this.goAway || + !this.radio.mayVoiceClaimSpeaker() + ) + return false; + this.pttPlaybackSuppressed = true; + this.audio?.clearPlayback(); + this.input.setVoiceSpeaker('user'); + return true; + } + + /** A claimed Space hold stops local playback before its audio reaches Live. */ + bargeIn() { + if ( + this.channel?.readyState !== 'open' || + this.goAway || + !this.radio.mayVoiceClaimSpeaker() + ) + return false; + return this.interrupt('push-to-talk'); + } + + interrupt( + reason = 'user-speech', + { + keepPlayback = false, + preserveAdvisoryCaption = false, + serverInterrupted = false, + } = {}, + ) { + if (this.goAway && !serverInterrupted) return false; + const epoch = this.epoch; + const generation = ++this.inputGeneration; + const channel = this.channel; + const owned = () => + this.owns(epoch) && + this.inputGeneration === generation && + this.channel === channel; + this.pttPlaybackSuppressed = false; + // Arm only for an outstanding reply: an idle hold/first text command has + // no old boundary to wait for. This also covers generation before the + // first output packet, because input submission marks replyPending. + const deferred = this.deferredInput; + const deferInput = + deferred || (this.retiredReplyPending && this.replyPending); + if (this.replyPending) { + if (!this.retiredReplyPending) + this.retiredAutomaticActivityOwner = this.automaticActivityOwner; + this.retiredReplyPending = true; + } + this.replyPending = false; + this.unrequestedReply = false; + if (['push-to-talk', 'user-text'].includes(reason)) { + this.automaticTurnOpen = false; + this.automaticActivityOwner = null; + } + this.clearDeferredInput(); + this.abortTools(); + if (!owned()) return false; + this.intentItemId = null; + this.intentResponseId = null; + this.intentCaptureId = null; + if (deferInput) { + // Only one successor may reach Live before the previous boundary. A + // newer local turn replaces unsent input and waits to interrupt that + // successor, rather than creating indistinguishable overlapping replies. + const owner = { + epoch: this.epoch, + turn: this.turn, + messages: [], + bytes: 0, + activityOpen: false, + kind: reason, + context: null, + deadline: + deferred?.deadline || + this.retiredBoundaryWait?.deadline || + this.now() + this.inputBoundaryTimeoutMs, + }; + this.clearRetiredBoundaryWait(); + this.deferredInput = owner; + owner.timer = this.setTimer( + () => { + if (this.deferredInput === owner && this.owns(owner.epoch)) + this.fail( + new Error( + msg( + 'Gemini did not finish the interrupted reply. Start voice again.', + ), + ), + ); + }, + Math.max(0, owner.deadline - this.now()), + ); + owner.timer.unref?.(); + } else this.boundRetiredReply(); + this.noteUserActivity(); + if (!owned()) return false; + if (!keepPlayback) this.audio?.clearPlayback(); + if (!owned()) return false; + this.modelComplete = false; + this.pendingCompletionStatus = null; + this.postToolOutput = false; + this.stopAfterReply = false; + this.transcripts.assistant = ''; + this.emit({ + type: 'interruption', + reason, + ...(preserveAdvisoryCaption ? { preserveAdvisoryCaption: true } : {}), + }); + if (!owned()) return false; + this.input.setVoiceSpeaker(this.goAway ? 'idle' : 'user'); + if (!owned()) return false; + this.setStatus( + 'listening', + this.goAway + ? msg('Gemini session expiring; finishing the current reply') + : msg('Ask or command'), + ); + return owned(); + } + + discardRetiredReply(message, epoch) { + if (!this.owns(epoch) || !this.retiredReplyPending) return false; + const content = message.serverContent; + // Retired output proves the old reply is still running; only silence + // after it counts toward the first retirement's recovery bound. + if ( + this.retiredBoundaryWait?.extendable && + !content?.turnComplete && + (content?.modelTurn || + content?.outputTranscription || + content?.generationComplete || + message.toolCall) + ) { + this.clearRetiredBoundaryWait(); + this.boundRetiredReply(); + } + const retiredIds = (message.toolCall?.functionCalls || []) + .map((call) => call?.id) + .filter(validCallId); + this.rememberCallIds(retiredIds); + for (const id of retiredIds) this.cancelledCalls.add(id); + // Gemini documents interrupted -> turnComplete. Discard all retired + // output, including content coalesced with either marker. A natural + // completion racing the local claim also closes the old reply safely. + if (content?.turnComplete) { + this.clearRetiredBoundaryWait(); + this.retiredReplyPending = false; + if ( + this.typedIntent?.retiredAutomaticPredecessor === + this.retiredAutomaticActivityOwner + ) + this.typedIntent.retiredAutomaticPredecessor = null; + // A true replyPending now belongs to a successor submitted while + // this older reply was retired. Its boundary cannot revoke that input. + if (this.automaticActivityOwner === this.retiredAutomaticActivityOwner) { + this.automaticTurnOpen = false; + this.automaticActivityOwner = null; + } + this.retiredAutomaticActivityOwner = null; + if (this.goAway) { + this.clearDeferredInput(); + if ( + this.goAway.supersededSuccessor || + (!this.replyPending && + !this.automaticTurnOpen && + !this.pendingTools && + !this.audio?.pending && + !this.controlQueue.length) + ) + this.finishGoAway(); + } else this.flushDeferredInput(); + } + return true; + } + + handleMessage(message, epoch = this.epoch) { + if (!this.owns(epoch)) return; + const initialGeneration = this.inputGeneration; + const initialTurn = this.turn; + const initialCurrent = () => + this.owns(epoch) && + this.inputGeneration === initialGeneration && + this.turn === initialTurn; + if (this.goAway && this.now() >= this.goAway.deadline) { + this.finishGoAway(); + return; + } + if (message.error) { + this.fail( + new Error( + msg( + 'Gemini rejected the voice session. Start voice again to request a fresh session.', + ), + ), + msg( + 'Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.', + ), + ); + return; + } + if (message.goAway) { + this.beginGoAway(message.goAway.timeLeft, epoch); + if (!initialCurrent()) return; + } + let prunedCancelledControls = false; + for (const id of message.toolCallCancellation?.ids || []) { + const action = this.tools.get(id); + const results = [ + this.radio.pendingRadioPlaybackResult, + this.radio.radioHandoffInFlightResult, + ].filter((result) => { + const owner = result && this.radioPlaybackCalls.get(result); + return ( + owner?.id === id && owner.epoch === epoch && owner.turn === this.turn + ); + }); + const queuedResponse = this.controlQueue.some((entry) => + entry.message.toolResponse?.functionResponses?.some( + (call) => call.id === id, + ), + ); + // A queued response still owns transport work until it reaches the wire. + if (!action && !results.length && !queuedResponse) continue; + this.cancelledCalls.add(id); + action?.abort(); + if (!initialCurrent()) return; + for (const result of results) this.radio.cancelPlaybackResult(result); + if (!initialCurrent()) return; + const queuedControls = this.controlQueue.length; + this.pruneControls(); + prunedCancelledControls ||= this.controlQueue.length < queuedControls; + } + if (prunedCancelledControls) { + // Removing the last queued response can release an already complete turn. + this.finishPlayback(); + // Completion observers may stop this session or accept a successor input. + if (!initialCurrent()) return; + } + const content = message.serverContent; + let receivedReplyPending = this.replyPending || this.retiredReplyPending; + const settleFencedBoundary = () => { + // Preserve this packet's old terminal without publishing its caption, + // audio or tools into the new input. A terminal already processed below + // must never be reused to close a successor retired by a final observer. + if (receivedReplyPending && content?.turnComplete) + this.discardRetiredReply(message, epoch); + }; + // A server acknowledgment of a local interruption belongs to the retired + // reply; it must not revoke the successor's pointer, timing or turn token. + if (content?.interrupted) { + const newAutomaticActivity = + !this.input.pushToTalkMode && + !this.goAway && + message.voiceActivity?.type === 'ACTIVITY_START' && + !this.userActivityOpen; + const predecessor = this.typedIntent?.automaticPredecessor; + if ( + !this.retiredReplyPending && + !newAutomaticActivity && + predecessor?.epoch === epoch && + this.currentTypedInput() + ) { + // Automatic VAD may not have produced output before text superseded + // it. This specific local predecessor owns one interruption boundary; + // the acknowledgment is not another user input. Do not infer this + // from output silence alone, or wait for a boundary until it arrives. + this.retiredAutomaticActivityOwner = predecessor; + this.typedIntent.retiredAutomaticPredecessor = predecessor; + this.typedIntent.automaticPredecessor = null; + } else if (!this.retiredReplyPending) { + if ( + !this.interrupt('user-speech', { + preserveAdvisoryCaption: true, + serverInterrupted: true, + }) + ) { + settleFencedBoundary(); + return; + } + } + this.retiredReplyPending = true; + receivedReplyPending = true; + this.boundRetiredReply(); + } + // Manual PTT owns its onset locally. Automatic speech takes one snapshot + // per activity, regardless of duplicate/late transcription notifications. + if (!this.input.pushToTalkMode && !this.goAway) { + if ( + message.voiceActivity?.type === 'ACTIVITY_START' && + !this.userActivityOpen + ) { + this.userActivityOpen = true; + const typedPredecessor = this.typedIntent?.retiredAutomaticPredecessor; + if ( + this.retiredReplyPending && + typedPredecessor && + typedPredecessor === this.retiredAutomaticActivityOwner && + this.currentTypedInput() + ) { + // This precise wait still belongs to the automatic predecessor, not + // the submitted text. New speech revokes that text without inventing + // a second retirement/deferred input or renewing the existing bound. + this.replyPending = false; + if (!this.interrupt()) return; + } else if (!this.retiredReplyPending && !this.interrupt()) return; + // Local activity ownership is only for expiry liveness, not tools. + // Assign after retiring observed output so its boundary cannot clear + // a newer detected activity that has not received a reply yet. + this.automaticActivityOwner = { + epoch, + sequence: ++this.automaticActivitySequence, + }; + this.automaticTurnOpen = true; + const activityGeneration = this.inputGeneration; + const activityCurrent = () => + this.owns(epoch) && this.inputGeneration === activityGeneration; + this.beginSpeechIntent(null); + if (!activityCurrent()) return; + this.beginPointerTurn('speech_start', undefined, activityCurrent); + if (!activityCurrent()) return; + const context = this.pendingPointerContext; + if ( + context && + this.send({ + realtimeInput: { text: JSON.stringify(context.item) }, + }) && + activityCurrent() + ) + this.consumePointerContext(context); + } else if ( + message.voiceActivity?.type === 'ACTIVITY_END' && + this.userActivityOpen + ) { + this.userActivityOpen = false; + this.bindSpeechIntent(); + this.narration.userTurnEnded(); + } + } + // Native activity above may intentionally advance the local input. From + // here, observers must not let the rest of this packet reach a successor. + const messageGeneration = this.inputGeneration; + const messageTurn = this.turn; + const messageCurrent = () => + this.owns(epoch) && + this.inputGeneration === messageGeneration && + this.turn === messageTurn; + // Native Live captions are unordered advisory segments. Never attach them + // to a command token or use them to grant/revoke a tool's lifetime. + if (content?.inputTranscription?.text) { + this.transcript('user', content.inputTranscription); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } + } + if (this.discardRetiredReply(message, epoch)) return; + // Push-to-talk owns every input boundary locally: a call that arrives with + // no input, reply or tool result outstanding was not requested, and nor is + // anything else in the reply it starts. Open-mic turns start on Gemini's + // server-side activity detection, which reports no boundary to gate on. + const unrequested = + this.input.pushToTalkMode && + !this.replyPending && + !this.userTurnPending && + !this.pendingTools; + let toolCalls = message.toolCall?.functionCalls || []; + if (toolCalls.length && (unrequested || this.unrequestedReply)) { + this.refuseUnrequestedTools(toolCalls); + toolCalls = []; + } + // In push-to-talk, a caption alone after its turn ended still belongs to + // that turn. It must not open a reply the next input would retire, + // discarding that input's own tool calls while it waits for a boundary + // that never comes. Open mic cannot tell it from a new reply's first + // caption, so there it still opens one. + const lateCaption = + this.input.pushToTalkMode && + !this.replyPending && + !this.userTurnPending && + !this.automaticTurnOpen && + !this.pendingTools && + !content?.modelTurn && + !content?.generationComplete && + !toolCalls.length; + if ( + content?.modelTurn || + (content?.outputTranscription && !lateCaption) || + content?.generationComplete || + toolCalls.length + ) { + // Output with nothing outstanding starts a reply nobody requested; a + // later call in it is refused like one that arrived first. + if (unrequested) this.unrequestedReply = true; + // Once successor output is observed, an unattributed later interruption + // cannot still claim the unobserved automatic predecessor's boundary. + if (this.typedIntent) { + this.typedIntent.automaticPredecessor = null; + this.typedIntent.retiredAutomaticPredecessor = null; + } + this.replyPending = true; + receivedReplyPending = true; + } + if (content?.outputTranscription?.text) { + this.userTurnPending = false; + this.postToolOutput = true; + this.transcript('assistant', content.outputTranscription); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } + } + for (const part of (content?.interrupted + ? [] + : content?.modelTurn?.parts) || []) { + if (part.thought) continue; + if (part.inlineData?.data) { + this.userTurnPending = false; + this.modelComplete = false; + this.postToolOutput = true; + if (!this.pttPlaybackSuppressed) { + this.input.setVoiceSpeaker('ai'); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } + this.audio?.play(part.inlineData.data, part.inlineData.mimeType); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } + this.modelSpoke = true; + this.narration.audible(); + if (!messageCurrent()) { + settleFencedBoundary(); + return; + } + } + } + } + if (toolCalls.length) this.enqueueTools(toolCalls, epoch); + if (content?.turnComplete) { + if (this.typedIntent) { + this.typedIntent.automaticPredecessor = null; + this.typedIntent.retiredAutomaticPredecessor = null; + } + this.pttPlaybackSuppressed = false; + // A native terminal boundary also completes a silent user turn. Output, + // pending tools and queued controls still gate presentation completion. + this.userTurnPending = false; + this.replyPending = false; + this.unrequestedReply = false; + this.automaticTurnOpen = false; + this.automaticActivityOwner = null; + this.retiredAutomaticActivityOwner = null; + this.modelComplete = true; + this.pendingCompletionStatus = content.interrupted + ? 'interrupted' + : 'completed'; + this.completedTurns++; + // Native input captions have no response identity. A model completion + // cannot finalize them or join speech from either side of an interruption. + if (this.transcripts.assistant) { + this.emit({ + type: 'transcript', + role: 'assistant', + text: this.transcripts.assistant, + final: true, + responseId: this.intentResponseId, + }); + if (!messageCurrent()) return; + this.transcripts.assistant = ''; + } + this.finishPlayback(); + } + } + + transcript(role, value) { + if (role === 'assistant') + this.transcripts.assistant = ( + this.transcripts.assistant + value.text + ).slice(-16000); + this.emit({ + type: 'transcript', + role, + text: value.text, + final: false, + responseId: role === 'user' ? null : this.intentResponseId, + ...(role === 'user' ? { attribution: 'unverified', delta: true } : {}), + }); + } + + /** + * Answer unrequested calls without running them: every declaration is + * BLOCKING, so Live waits for a result. The reply stays tracked, so the next + * input retires it and its late boundary cannot close the user's turn. + */ + refuseUnrequestedTools(calls) { + const refused = new Map(); + for (const call of calls) + if ( + validCallId(call?.id) && + typeof call.name === 'string' && + !this.seenCalls.has(call.id) + ) + refused.set(call.id, call.name); + this.rememberCallIds([...refused.keys()]); + if (!refused.size) return; + this.unrequestedReply = true; + this.replyPending = true; + this.send({ + toolResponse: { + functionResponses: [...refused].map(([id, name]) => ({ + id, + name, + response: { + ok: false, + action: name, + changed: false, + error: 'No user request is active.', + responseGuidance: + 'Do not call another tool or claim an action ran. Wait for the user.', + }, + })), + }, + }); + } + + rememberCallIds(ids) { + const freshIds = new Set(ids.filter((id) => !this.seenCalls.has(id))); + // Forgetting an old ID would let its replay acquire a new turn's authority. + // Reserve the entire batch before changing policy or queuing any action. + if (this.seenCalls.size + freshIds.size > MAX_SEEN_CALLS) + throw new Error( + msg( + 'Gemini action history is full. Start voice again for a fresh session.', + ), + ); + for (const id of freshIds) this.seenCalls.add(id); + } + + enqueueTools(calls, epoch) { + if ( + calls.length > MAX_TOOL_CALLS || + this.pendingTools + calls.length > MAX_TOOL_CALLS + ) + throw new Error(msg('Gemini requested too many simultaneous actions')); + const turn = this.turn; + this.userTurnPending = false; + this.modelComplete = false; + // Admit each valid native call ID once per connection. + const batchIds = new Set(); + const acceptedCalls = calls.filter((call) => { + if ( + !validCallId(call?.id) || + typeof call.name !== 'string' || + this.seenCalls.has(call.id) || + batchIds.has(call.id) + ) + return false; + batchIds.add(call.id); + return true; + }); + this.rememberCallIds(acceptedCalls.map((call) => call.id)); + for (const call of acceptedCalls) { + this.pendingTools++; + const action = new AbortController(); + action.call = { id: call.id, name: call.name }; + action.sequence = ++this.toolSequence; + this.tools.set(call.id, action); + const priorTools = this.toolQueue; + const args = + call.args && typeof call.args === 'object' && !Array.isArray(call.args) + ? call.args + : {}; + // Stronger Radio controls must reach their shared reservation/result + // owner while an older selection is awaiting a service. Keep ordinary + // siblings ordered, and retain every in-flight call in the drain promise. + // executeTool still checks currentness before any + // reservation; only a successful stronger result revokes older work. + const { stronger } = classifyGeminiTool(call.name, args); + const execution = (stronger ? Promise.resolve() : priorTools) + .then(async () => { + if (!this.owns(epoch) || turn !== this.turn) return; + if (this.cancelledCalls.has(call.id)) { + if (this.tools.get(call.id) === action) this.tools.delete(call.id); + this.pendingTools--; + if (!this.pendingTools) { + this.setStatus('listening', msg('Ask or command')); + if (this.owns(epoch) && turn === this.turn) this.finishPlayback(); + } + return; + } + await this.executeTool(call, epoch, turn, action); + }) + .catch((error) => { + if (this.owns(epoch) && turn === this.turn) this.fail(error); + }); + this.toolQueue = stronger + ? Promise.all([priorTools, execution]).then(() => {}) + : execution; + } + } + + async executeTool(call, epoch, turn, action) { + const args = + call.args && typeof call.args === 'object' && !Array.isArray(call.args) + ? call.args + : {}; + const deadline = AbortSignal.timeout(45_000); + const expire = () => action.abort(deadline.reason); + deadline.addEventListener('abort', expire, { once: true }); + const toolSignal = AbortSignal.any([action.signal, this.turnAbort.signal]); + const { + radioControl, + radioVisibility, + authorityDomain: authority, + stronger, + abortScope, + } = classifyGeminiTool(call.name, args); + let reservation = null; + let handoffEpoch = this.radio.radioHandoffEpoch; + const current = () => + this.owns(epoch) && this.turn === turn && !toolSignal.aborted; + const actionCurrent = () => + current() && + !this.radioSuperseded(action, authority, epoch, turn) && + (authority !== 'playback' || + handoffEpoch === this.radio.radioHandoffEpoch); + let result; + try { + if (!actionCurrent()) throw cancelled(); + if (stronger) + reservation = this.radio.reserveRadioToolHandoff({ abortScope }); + handoffEpoch = this.radio.radioHandoffEpoch; + if (!actionCurrent()) throw cancelled(); + if (authority) + this.radio.registerTool(action, { + responseId: String(turn), + authorityDomain: authority, + }); + this.setStatus('executing', msg('Running command')); + // A common state observer may synchronously stop/replace the session. + if (!actionCurrent()) throw cancelled(); + this.radio.pauseRadioForVoice(); + if (!actionCurrent()) throw cancelled(); + if (this.modelSpoke) this.narration.preamble(); + this.narration.toolStarted(call.id, { + name: call.name, + label: narrationLabel(call.name, args), + narrate: !radioControl && !radioVisibility, + }); + if (!actionCurrent()) throw cancelled(); + const execution = Promise.resolve( + this.runAction(call.name, args, { + signal: toolSignal, + callId: call.id, + progress: (update) => { + if (actionCurrent()) this.narration.progress(call.id, update); + }, + isCurrent: actionCurrent, + }), + ); + // action-call observers can abort before waitForSignal receives the + // already-started operation; retain its rejection handler in that case. + void execution.catch(() => {}); + result = await waitForSignal(execution, toolSignal); + if (!actionCurrent()) throw cancelled(); + if (result?.ok && result.radioPlaybackRequested) { + const playbackResult = { ...result }; + this.radioPlaybackCalls.set(playbackResult, { + id: call.id, + epoch, + turn, + }); + this.radio.setPendingPlayback(playbackResult); + } + if ( + result?.action === 'get_entity_context' && + shouldSendViewportImage(result.scene?.basemap?.viewScale) && + !hasStructuredViewIdentity(result) && + this.now() - this.lastViewportAt >= 1000 + ) { + const capture = Promise.resolve(this.captureViewport()); + void capture.catch(() => {}); + const image = await waitForSignal(capture, toolSignal); + if (actionCurrent() && image?.startsWith('data:image/jpeg;base64,')) { + this.lastViewportAt = this.now(); + this.send({ + realtimeInput: { + video: { + mimeType: 'image/jpeg', + data: image.slice(image.indexOf(',') + 1), + }, + }, + }); + } + } + } catch (error) { + result = { + ok: false, + tool: call.name, + ...(radioVisibility ? { layerId: 'radio' } : {}), + error: + error?.name === 'AbortError' + ? 'Action cancelled by a newer request' + : error?.message || 'GEV command failed', + }; + } finally { + deadline.removeEventListener('abort', expire); + if (this.tools.get(call.id) === action) this.tools.delete(call.id); + this.radio.releaseTool(action); + if (this.owns(epoch) && turn === this.turn) + this.narration.toolFinished(call.id); + } + if (!this.owns(epoch) || turn !== this.turn) return; + this.pendingTools--; + // Successful stronger ownership commits before the result is serialized. + // A failed reservation stays held until after its failure reaches transport. + if (reservation && result?.ok) { + this.radio.settleRadioToolHandoffReservation(reservation, { + commit: true, + responseId: String(turn), + }); + reservation = null; + this.noteRadioStrongerCommit(action, abortScope, turn); + } + if (!this.owns(epoch) || turn !== this.turn) return; + const playbackObsolete = + authority === 'playback' && handoffEpoch !== this.radio.radioHandoffEpoch; + if (radioControl || radioVisibility) { + const state = this.radioLayer?.getUIState?.() || {}; + const lifecycle = readLayerLifecycleSummary(this.dataManager, 'radio', { + fallbackEnabled: state.enabled ?? result?.enabled, + }); + result = { + ...result, + ...lifecycle, + audioState: state.audioState || result?.audioState || 'stopped', + ...(result?.radioAction === 'pause' && lifecycle.enabled === false + ? { changed: false } + : {}), + ...(playbackObsolete && result?.ok && result.radioPlaybackRequested + ? { radioPlaybackRequested: false, radioPlaybackSuppressed: true } + : {}), + }; + } + if ( + !this.cancelledCalls.has(call.id) && + shouldStopVoiceAfterRadioTool(result) && + !result.radioPlaybackRequested && + !result.radioPlaybackSuppressed + ) + this.stopAfterReply = true; + // Native call cancellation suppresses its output, without canceling siblings. + if (!this.cancelledCalls.has(call.id)) { + this.postToolOutput = false; + this.modelComplete = false; + this.replyPending = true; + const mapSnapshot = [...this.pendingMapEvents]; + const mapEvents = mapSnapshot.map((text) => JSON.parse(text)); + const response = { + ...(result ?? { ok: false, error: 'Command returned no result' }), + responseGuidance: responseInstructionForToolResult(result), + }; + this.send( + { + toolResponse: { + functionResponses: [ + { + id: call.id, + name: call.name, + response: mapEvents.length + ? { ...response, mapEvents } + : response, + }, + ], + }, + }, + { onSent: () => this.consumeMapEvents(mapSnapshot) }, + ); + } + if (!this.owns(epoch) || turn !== this.turn) return; + if (reservation) + this.radio.settleRadioToolHandoffReservation(reservation, { + commit: false, + responseId: String(turn), + }); + if (this.owns(epoch) && turn === this.turn && !this.pendingTools) { + this.setStatus('listening', msg('Ask or command')); + if (this.owns(epoch) && turn === this.turn) this.finishPlayback(); + } + } + + finishPlayback() { + if (this.goAway && this.now() >= this.goAway.deadline) { + this.finishGoAway(); + return; + } + if ( + !this.isActive() || + !this.modelComplete || + this.userTurnPending || + this.pendingTools || + (this.input.pushToTalkMode && + this.captureOwner && + !this.captureOwner.retired && + !this.captureOwner.admitted && + !this.captureOwner.settled) || + this.audio?.pending || + this.controlQueue.length + ) + return; + const completionStatus = this.pendingCompletionStatus; + this.pendingCompletionStatus = null; + this.narration.cancel(); + if (completionStatus) { + const responseId = this.intentResponseId; + const completionEpoch = this.epoch; + const completionTurn = this.turn; + this.emit({ + type: 'completion', + status: completionStatus, + responseId, + spoken: Boolean(this.modelSpoke), + audioDrained: true, + }); + // Observers may synchronously accept new input or replace the session. + // Old completion must not clear its metadata or overwrite its status. + if (this.epoch !== completionEpoch || this.turn !== completionTurn) + return; + // Caption completion does not create an input or tool authority boundary. + this.clearSettledSpeechIntent(); + } + // A completion observer may replace this provider immediately. Do not let + // the retired session update UI or start delayed Radio work afterward. + if (!this.isActive()) return; + this.input.setVoiceSpeaker('idle'); + this.setStatus('listening', msg('Ask or command')); + if (this.stopAfterReply && this.postToolOutput) { + this.stop({ preserveRadioPlayback: true }); + return; + } + if (this.goAway) { + this.finishGoAway(); + return; + } + if (this.postToolOutput && this.radio.pendingRadioPlaybackResult) + void this.radio.startPendingRadioHandoff(); + } + + sendTextCommand(text) { + const clean = String(text || '').trim(); + if (!clean || this.goAway) return false; + if (clean.length > 16000) + throw new Error(msg('Voice text command is too long')); + if (this.channel?.readyState !== 'open') + throw new Error(msg('Gemini voice is not connected')); + if (this.input.pushToTalkMode && this.captureOwner) + this.captureOwner.retired = true; + const automaticPredecessor = + this.automaticTurnOpen && + !this.replyPending && + !this.retiredReplyPending && + !this.deferredInput + ? this.automaticActivityOwner + : null; + if (!this.interrupt('user-text')) return false; + const epoch = this.epoch; + const turn = this.turn; + const generation = this.inputGeneration; + const owned = () => + this.owns(epoch) && + this.turn === turn && + this.inputGeneration === generation; + // Typed intent supersedes the held speech even if its worklet end is late. + // Close the forwarded manual activity before submitting the new text. + if (this.input.pushToTalkMode && this.forwardedActivityOpen) + this.send( + { realtimeInput: { activityEnd: {} } }, + { captureId: this.forwardedCaptureId }, + ); + if (!owned()) return false; + const typedIntent = { + epoch: this.epoch, + turn: this.turn, + text: clean, + responseId: `gemini-response-${this.epoch}-${++this.intentSequence}`, + automaticPredecessor, + retiredAutomaticPredecessor: null, + }; + // Keep local presentation ownership while reliable transport drains. + this.typedIntent = typedIntent; + this.beginPointerTurn('text', undefined, owned); + if (!owned()) return false; + const context = this.pendingPointerContext; + const mapSnapshot = [...this.pendingMapEvents]; + const parts = context ? [{ text: JSON.stringify(context.item) }] : []; + if (this.pendingMapEvents.length) + parts.push({ + text: JSON.stringify({ + type: 'map_events', + events: mapSnapshot.map((value) => JSON.parse(value)), + }), + }); + parts.push({ text: clean }); + const sent = this.sendUserInput( + { + clientContent: { turns: [{ role: 'user', parts }], turnComplete: true }, + }, + undefined, + { + onSent: () => { + if (!owned() || this.typedIntent !== typedIntent) return; + this.replyPending = true; + this.bindTextIntent(typedIntent); + if (!owned() || this.typedIntent !== typedIntent) return; + this.consumePointerContext(context); + this.consumeMapEvents(mapSnapshot); + }, + }, + ); + if (sent && owned()) { + this.narration.userTurnEnded(); + } + return sent && owned(); + } + + consumeMapEvents(snapshot) { + // Queue acceptance is not delivery. A cancelled envelope retains passive + // context for the next genuine turn, and new events cannot be erased. + if (snapshot.every((text, index) => this.pendingMapEvents[index] === text)) + this.pendingMapEvents.splice(0, snapshot.length); + } + + validCaptureMetadata(metadata) { + return Number.isSafeInteger(metadata?.captureId) && metadata.captureId > 0; + } + + acceptsCapture(metadata) { + return ( + this.validCaptureMetadata(metadata) && + metadata.captureId === this.captureOwner?.captureId && + !this.captureOwner.retired + ); + } + + sendUserInput(message, metadata, { onSent = null } = {}) { + const owner = this.deferredInput; + const input = message.realtimeInput; + const captureContext = message.clientContent?.turnComplete === false; + // A superseded start cannot read the current pointer or acquire its queue. + // Only an end for the actually forwarded old activity may close that wire. + if ( + captureContext || + input?.activityStart || + input?.activityEnd || + input?.audio + ) { + if (!this.validCaptureMetadata(metadata)) return false; + if (!this.acceptsCapture(metadata)) { + return input?.activityEnd && + this.forwardedActivityOpen && + this.forwardedCaptureId === metadata.captureId + ? this.send(message, { captureId: metadata.captureId, onSent }) + : false; + } + } + if ( + input?.activityEnd && + !this.forwardedActivityOpen && + !this.controlQueue.some( + (entry) => + entry.captureId === metadata?.captureId && + entry.message.realtimeInput?.activityStart, + ) && + !owner?.activityOpen + ) + return true; + if (!owner) + return this.send(message, { + captureId: metadata?.captureId ?? null, + onSent, + }); + if (!this.owns(owner.epoch) || owner.turn !== this.turn) return false; + // Worklet callbacks keep the previous end ahead of the new start. After + // replacing unsent input, that old end/PCM has no new gesture to belong to. + if (input?.activityEnd && !owner.activityOpen) + return this.forwardedActivityOpen + ? this.send(message, { captureId: this.forwardedCaptureId, onSent }) + : true; + if ( + input?.audio && + (!owner.activityOpen || owner.kind !== 'push-to-talk') + ) { + // These frames cannot be replayed under the deferred explicit owner. + // Preserve that discard policy while making the loss observable. + this.droppedAudioChunks++; + this.deferredAudioDrops[ + owner.kind !== 'push-to-talk' ? 'nonPttOwner' : 'closedActivity' + ]++; + return true; + } + const bytes = new TextEncoder().encode(JSON.stringify(message)).byteLength; + if ( + owner.bytes + bytes > MAX_BUFFERED_BYTES || + this.now() >= owner.deadline + ) { + this.fail( + new Error( + msg( + 'Gemini interrupted input exceeded its buffer or wait limit. Start voice again.', + ), + ), + ); + return false; + } + if (input?.activityStart) { + owner.activityOpen = true; + owner.captureId = metadata?.captureId; + } + if (input?.activityEnd) owner.activityOpen = false; + owner.messages.push(message); + if (onSent) { + owner.onSent ||= new Map(); + owner.onSent.set(message, onSent); + } + owner.bytes += bytes; + return true; + } + + clearDeferredInput() { + if (this.deferredInput) this.clearTimer(this.deferredInput.timer); + this.deferredInput = null; + } + + clearRetiredBoundaryWait() { + if (this.retiredBoundaryWait) + this.clearTimer(this.retiredBoundaryWait.timer); + this.retiredBoundaryWait = null; + } + + waitForRetiredBoundary(owner) { + this.clearRetiredBoundaryWait(); + const wait = { epoch: owner.epoch, deadline: owner.deadline }; + this.retiredBoundaryWait = wait; + wait.timer = this.setTimer( + () => { + if ( + this.retiredBoundaryWait === wait && + this.owns(wait.epoch) && + this.retiredReplyPending + ) + this.fail( + new Error( + msg( + 'Gemini did not finish the interrupted reply. Start voice again.', + ), + ), + ); + }, + Math.max(0, wait.deadline - this.now()), + ); + wait.timer.unref?.(); + } + + // A first retirement has no deferred owner, yet everything Live sends is + // discarded until the old boundary. Never leave that wait unbounded. + boundRetiredReply() { + if ( + !this.retiredReplyPending || + this.deferredInput || + this.retiredBoundaryWait + ) + return; + this.waitForRetiredBoundary({ + epoch: this.epoch, + deadline: this.now() + this.inputBoundaryTimeoutMs, + }); + this.retiredBoundaryWait.extendable = true; + } + + flushDeferredInput() { + const owner = this.deferredInput; + if (!owner) return; + this.clearDeferredInput(); + const generation = this.inputGeneration; + const current = () => + this.owns(owner.epoch) && + owner.turn === this.turn && + generation === this.inputGeneration; + if (!current()) return; + if (this.now() >= owner.deadline) { + this.fail( + new Error( + msg( + 'Gemini did not finish the interrupted reply. Start voice again.', + ), + ), + ); + return; + } + // The already submitted successor is now the one being retired. Send the + // newest start/text immediately to trigger its own provider boundary. + this.retiredReplyPending = true; + // The forwarded predecessor here is explicit PTT/text, not a later VAD + // activity that may have arrived while its boundary was pending. + this.retiredAutomaticActivityOwner = null; + // Flushing (even an empty capture queue) must not remove the absolute + // recovery bound while the forwarded predecessor still owns output. + this.waitForRetiredBoundary(owner); + this.replyPending = false; + for (const message of owner.messages) { + if (!current()) return; + const onSent = owner.onSent?.get(message); + this.send(message, { + captureId: owner.captureId ?? null, + onSent: () => { + if (!current()) return; + onSent?.(); + if (!current()) return; + if ( + (message.clientContent && + message.clientContent.turnComplete !== false) || + message.realtimeInput?.audio || + message.realtimeInput?.text + ) + this.replyPending = true; + }, + }); + if (!current()) return; + } + if (current() && owner.context && !this.controlQueue.length) + this.consumePointerContext(owner.context); + } + + /** Use the common pointer owner; held speech context precedes audio quietly. */ + beginPointerTurn(at, capturedSnapshot = undefined, isCurrent = () => true) { + if (!this.pointer || !isCurrent()) return null; + // The delayed manual speech notification still refers to the released hold. + if ( + capturedSnapshot === undefined && + at === 'speech_start' && + this.input.pushToTalkMode && + !this.pointer.isHeld?.() + ) + return this.pointer.activeSnapshot?.() || null; + const gesture = + at === 'keydown' || (at === 'speech_start' && this.input.pushToTalkMode); + let snapshot = null; + try { + snapshot = this.pointer.beginTurn( + at, + capturedSnapshot !== undefined + ? { capturedSnapshot } + : gesture + ? {} + : { recentMs: POINTER_OPEN_MIC_RECENT_MS }, + ); + } catch { + snapshot = null; + } + if (!isCurrent()) return null; + const event = pointerTurnEvent(snapshot, gesture); + this.pointerAnnounced = event.announced; + this.emit({ type: 'pointer', pointer: event.pointer }); + if (!isCurrent()) return null; + this.pendingPointerContext = pointerTurnContext(snapshot, { + at, + gesture, + sentKey: this.pointerSentKey, + }); + return snapshot; + } + + /** Reveal an implicit referent only after the shared action runner uses it. */ + announcePointer() { + const snapshot = this.pointer?.activeSnapshot?.() || null; + const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced); + if (!pointer) return false; + this.pointerAnnounced = snapshot; + this.emit({ type: 'pointer', pointer }); + return true; + } + + /** Pointing accompanies speech without asking Live to answer the context. */ + sendPointerContext(context, metadata) { + if (!context) return true; + const message = { + clientContent: { + turns: [ + { role: 'user', parts: [{ text: JSON.stringify(context.item) }] }, + ], + turnComplete: false, + }, + }; + // Context has the same physical owner and queue as its PTT start/PCM/end. + const deferred = Boolean(this.deferredInput); + const accepted = this.sendUserInput(message, metadata, { + onSent: () => this.consumePointerContext(context), + }); + if (accepted && deferred) this.consumePointerContext(context); + return accepted; + } + + consumePointerContext(context) { + if (!context) return; + if (this.deferredInput) { + this.deferredInput.context = context; + if (this.pendingPointerContext === context) + this.pendingPointerContext = null; + return; + } + this.pointerSentKey = context.key; + if (this.pendingPointerContext === context) + this.pendingPointerContext = null; + } + + // Any clientContent message can interrupt Live generation, including one + // with turnComplete:false. Passive map results wait for an explicit turn + // or a tool response; they must never cut off spoken output or start speech. + notifyMapEvent(event) { + if (this.goAway || this.channel?.readyState !== 'open') return false; + let text; + try { + text = JSON.stringify(event); + } catch { + return false; + } + if (!text || text.length > MAX_MAP_CONTEXT_LENGTH) return false; + this.pendingMapEvents.push(text); + while ( + this.pendingMapEvents.length > MAX_MAP_EVENTS || + this.pendingMapEvents.reduce((sum, value) => sum + value.length, 0) > + MAX_MAP_CONTEXT_LENGTH + ) + this.pendingMapEvents.shift(); + return true; + } + + beginSpeechIntent(captureId) { + if (this.retiredReplyPending && this.currentTypedInput()) return; + // Local onset labels serve timing/UI only. They do not assert native + // transcript ownership or mint a completed-speech authorization token. + this.clearSettledSpeechIntent(); + // interrupt() revokes old input when a new local turn actually starts. + // Onsets racing a retired reply must not erase its typed successor. + this.intentItemId = `gemini-input-${this.epoch}-${++this.intentSequence}`; + this.intentResponseId = null; + this.intentCaptureId = captureId; + // A physical speech onset clears display-only recent captions, including + // when retirement already fenced the reply and no second interrupt runs. + this.emit({ type: 'transcript', role: 'user', text: '', final: false }); + } + + bindSpeechIntent() { + if (!this.intentItemId || this.intentResponseId) return; + this.intentResponseId = `gemini-response-${this.epoch}-${this.intentSequence}`; + } + + currentTypedInput(turn = this.turn) { + const input = this.typedIntent; + return input?.epoch === this.epoch && input.turn === turn ? input : null; + } + + // Stronger Radio controls skip the ordered queue. An older Radio call still + // waiting behind a sibling must not start after a successful stronger + // result has revoked it; later calls in the same turn remain the model's. + noteRadioStrongerCommit(action, abortScope, turn) { + let commit = this.radioStrongerCommit; + if (commit?.epoch !== this.epoch || commit.turn !== turn) + commit = this.radioStrongerCommit = { + epoch: this.epoch, + turn, + all: 0, + playback: 0, + }; + const scope = abortScope === 'all' ? 'all' : 'playback'; + commit[scope] = Math.max(commit[scope], action.sequence); + } + + radioSuperseded(action, authority, epoch, turn) { + const commit = this.radioStrongerCommit; + // Read-only status queries stay current, as for the shared sibling abort. + if ( + !authority || + authority === 'query' || + commit?.epoch !== epoch || + commit.turn !== turn + ) + return false; + return ( + action.sequence < commit.all || + (authority === 'playback' && action.sequence < commit.playback) + ); + } + + bindTextIntent(intent) { + if ( + intent !== this.typedIntent || + intent.epoch !== this.epoch || + intent.turn !== this.turn + ) + return; + this.clearSettledSpeechIntent(); + this.intentResponseId = intent.responseId; + this.emit({ + type: 'transcript', + role: 'user', + text: intent.text, + final: true, + }); + } + + clearSettledSpeechIntent() { + this.intentResponseId = null; + this.intentItemId = null; + this.intentCaptureId = null; + } + + beginGoAway(timeLeft, epoch) { + const duration = + typeof timeLeft === 'string' && + /^(?:\d{1,6})(?:\.\d{1,9})?s$/.test(timeLeft) + ? Number(timeLeft.slice(0, -1)) * 1000 + : 0; + const milliseconds = Math.min( + MAX_GO_AWAY_MS, + Number.isFinite(duration) ? duration : 0, + ); + const deadline = this.now() + milliseconds; + if (this.goAway && this.goAway.deadline <= deadline) return; + const supersededSuccessor = + Boolean(this.deferredInput) || Boolean(this.goAway?.supersededSuccessor); + if (this.goAway) this.clearTimer(this.goAway.timer); + const owner = { + epoch, + turn: this.turn, + deadline, + timer: null, + // Dropping the newest unsent input must not make its superseded, + // already forwarded predecessor current at the retired boundary. + supersededSuccessor, + }; + this.goAway = owner; + this.clearDeferredInput(); + owner.timer = this.setTimer(() => { + if (this.goAway === owner && this.owns(epoch)) this.finishGoAway(); + }, milliseconds); + owner.timer?.unref?.(); + this.input.setMicrophoneEnabled(false); + this.radio.cancelRadioHandoff(); + this.setStatus( + this.status, + msg('Gemini session expiring; finishing the current reply'), + ); + if ( + milliseconds === 0 || + (!this.replyPending && + !this.automaticTurnOpen && + !this.pendingTools && + !this.audio?.pending && + !this.controlQueue.length) + ) + this.finishGoAway(); + else this.finishPlayback(); + } + + finishGoAway() { + if (!this.goAway) return; + this.fail( + new Error( + msg( + 'Gemini Live connection is expiring. Start voice again for a new conversation.', + ), + ), + msg('Start voice again for a fresh token and a new conversation.'), + ); + } + + getDiagnostics() { + return { + provider: 'gemini', + status: this.status, + model: this.model, + connected: this.channel?.readyState === 'open', + microphoneActive: Boolean( + this.stream + ?.getAudioTracks() + .some((track) => track.enabled && track.readyState !== 'ended'), + ), + pendingTools: this.pendingTools, + narration: this.narration.state(), + completedTurns: this.completedTurns, + audio: this.audio?.diagnostics() || this.lastAudio, + error: this.lastError, + close: this.lastClose, + droppedAudioChunks: this.droppedAudioChunks, + deferredAudioDrops: { ...this.deferredAudioDrops }, + droppedImages: this.droppedImages, + queuedControls: this.controlQueue.length, + expiring: Boolean(this.goAway), + }; + } + + stop({ + removeUi = false, + preserveStatus = false, + preserveRadioPlayback = false, + } = {}) { + if (this.disposed) return; + this.epoch++; + this.inputGeneration++; + this.clearSettledSpeechIntent(); + this.clearControls(); + if (this.goAway) this.clearTimer(this.goAway.timer); + this.goAway = null; + this.abort?.abort(); + this.abort = null; + this.abortTools(); + this.earcon.dispose(); + this.radio.invalidateHandoff(); + this.radio.stopHandoff({ preserveRadioPlayback }); + this.radio.clearPendingPlayback(); + if (this.channel) this.channel.readyState = 'closed'; + this.channel = null; + const socket = this.socket; + this.socket = null; + if (socket) { + socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null; + socket.close(); + } + if (this.audio) { + const lastAudio = { + ...this.audio.diagnostics(), + queuedSeconds: 0, + resourcesReleased: true, + contextState: 'closing', + }; + this.lastAudio = lastAudio; + void Promise.resolve(this.audio.close()).then( + (closed) => { + lastAudio.contextState = closed === false ? 'close-failed' : 'closed'; + }, + () => { + lastAudio.contextState = 'close-failed'; + }, + ); + } + this.audio = null; + this.stream?.getTracks().forEach((track) => track.stop()); + this.stream = null; + this.input.cancelPushToTalkHold(); + this.input.stopVoiceVisualizer(); + this.pointer?.clear({ keepHold: this.input.pushToTalkKeyHeld }); + if (!this.input.pushToTalkKeyHeld) this.heldPointerSnapshot = undefined; + this.input.resetSession(); + this.radio.setRadioVoiceDucking(false); + this.modelComplete = false; + this.pendingCompletionStatus = null; + this.postToolOutput = false; + this.modelSpoke = false; + this.stopAfterReply = false; + this.seenCalls.clear(); + this.cancelledCalls.clear(); + this.pendingMapEvents = []; + this.userTurnPending = false; + this.replyPending = false; + this.unrequestedReply = false; + this.automaticTurnOpen = false; + this.automaticActivityOwner = null; + this.retiredAutomaticActivityOwner = null; + this.automaticActivitySequence = 0; + this.retiredReplyPending = false; + this.clearDeferredInput(); + this.clearRetiredBoundaryWait(); + this.forwardedActivityOpen = false; + this.forwardedCaptureId = null; + this.captureEnabled = false; + this.pttPlaybackSuppressed = false; + this.microphoneRequested = false; + this.captureOwner = null; + this.userActivityOpen = false; + this.pointerSentKey = null; + this.pointerAnnounced = null; + this.pendingPointerContext = null; + this.transcripts = { assistant: '' }; + if (removeUi) { + this.disposed = true; + this.status = 'idle'; + this.input.detachBindings(); + this.radio.detachObservers(); + this.signal?.removeEventListener('abort', this.lifetimeAbort); + this.ui?.root?.remove(); + this.emit({ type: 'disposed' }); + } else if (!preserveStatus) this.setStatus('idle', msg('Voice off')); + } +} + +export function createGeminiSession(options) { + const controller = new GeminiSessionController(options); + return { + controller, + capabilities: { + costControls: false, + pushToTalk: true, + advisoryLimit: t( + 'Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.', + ), + }, + start: (settings) => controller.start(settings), + getStartOptions: () => ({ + pushToTalk: Boolean(controller.input.pushToTalkMode), + }), + stop: (settings) => controller.stop(settings), + sendText: (text) => controller.sendTextCommand(text), + sendMapEvent: (event) => controller.notifyMapEvent(event), + ignoreButtonClick: () => Boolean(controller.spaceKeyHeld), + bindControls: () => controller.input.bindPushToTalkShortcut(), + }; +} diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs new file mode 100644 index 000000000..b2f380b37 --- /dev/null +++ b/src/voice/geminiSession.test.mjs @@ -0,0 +1,3400 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { geminiSocketUrl } from './geminiSession.js'; +import { createVoiceSession } from './session.js'; +import { createPointerTracker } from './pointerContext.js'; +import { withToolCatalog } from './gevRealtime.js'; +import { + harness, + tick, + deferred, +} from '../testSupport/geminiNativeHarness.mjs'; + +const endpoint = + 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained'; + +const credential = { + token: 'auth_tokens/test', + model: 'gemini-3.8-live', + websocketUrl: endpoint, + config: { generationConfig: { responseModalities: ['AUDIO'] } }, +}; + +const manualCredential = { + ...credential, + config: { + ...credential.config, + realtimeInputConfig: { automaticActivityDetection: { disabled: true } }, + }, +}; + +test('ephemeral credentials are sent only to the fixed constrained Google endpoint', () => { + const url = new URL(geminiSocketUrl(credential)); + assert.equal(url.searchParams.get('access_token'), credential.token); + for (const websocketUrl of [ + endpoint + '?key=foo', + endpoint.replace('googleapis.com', 'example.com'), + endpoint.replace('wss:', 'ws:'), + endpoint.replace('Constrained', ''), + endpoint.replace('wss://', 'wss://user@'), + ]) + assert.throws( + () => geminiSocketUrl({ ...credential, websocketUrl }), + /endpoint/, + ); + assert.throws( + () => + geminiSocketUrl({ ...credential, token: 'AIza-not-an-ephemeral-token' }), + /ephemeral/, + ); +}); + +test('PTT locks manual activity and serializes cold and repeated held turns', async () => { + const h = harness(); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.spaceKeyHeld = true; + await h.c.start({ pushToTalk: true }); + assert.equal(h.requests[0].body, '{"inputMode":"push-to-talk"}'); + const ws = h.sockets[0]; + assert.deepEqual(ws.sent[0].setup.realtimeInputConfig, { + automaticActivityDetection: { disabled: true }, + }); + const audio = h.captures[0]; + const pcm = { data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' }; + audio.callbacks.onAudioStart(audio.metadata); // A duplicate notification cannot reopen. + audio.callbacks.onAudio(pcm, audio.metadata); + h.c.input.setMicrophoneEnabled(false); + audio.callbacks.onAudioEnd(audio.metadata); + audio.callbacks.onAudio(pcm, audio.metadata); // Unbracketed/stale PCM is not forwarded. + assert.equal(h.tracks[0].enabled, false); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio(pcm, audio.metadata); + h.c.input.setMicrophoneEnabled(false); + assert.deepEqual(ws.sent.slice(1), [ + { realtimeInput: { activityStart: {} } }, + { realtimeInput: { audio: pcm } }, + { realtimeInput: { activityEnd: {} } }, + { realtimeInput: { activityStart: {} } }, + { realtimeInput: { audio: pcm } }, + { realtimeInput: { activityEnd: {} } }, + ]); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.input.pushToTalkMode, true); + assert.equal(h.tracks[0].enabled, false); + h.c.stop(); +}); + +test('ordinary microphone keeps its default setup and automatic stream-end message', async () => { + const h = harness(); + await h.c.start(); + assert.equal(h.requests[0].body, '{}'); + assert.deepEqual(h.sockets[0].sent, [ + { setup: { ...credential.config, model: 'models/gemini-3.8-live' } }, + ]); + h.c.input.setMicrophoneEnabled(false); + assert.deepEqual(h.sockets[0].sent.at(-1), { + realtimeInput: { audioStreamEnd: true }, + }); + h.c.stop(); +}); + +for (const phase of ['token', 'setup', 'microphone']) { + test(`PTT release during ${phase} startup stays muted until a later hold`, async () => { + const pending = deferred(); + const h = harness({ + ...(phase === 'token' ? { fetchImpl: () => pending.promise } : {}), + ...(phase === 'setup' ? { autoSetup: false } : {}), + ...(phase === 'microphone' + ? { getUserMedia: () => pending.promise } + : {}), + }); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.spaceKeyHeld = true; + const start = h.c.start({ pushToTalk: true }); + await tick(); + assert.equal(h.media.length, 0); + assert.equal( + h.sockets[0]?.sent.filter((value) => value.realtimeInput).length || 0, + 0, + ); + h.c.input.pushToTalkKeyHeld = false; + h.c.input.spaceKeyHeld = false; + h.c.input.setMicrophoneEnabled(false); + if (phase === 'token') pending.resolve(Response.json(manualCredential)); + else if (phase === 'setup') h.sockets[0].message({ setupComplete: {} }); + else pending.resolve(h.makeStream()); + await start; + assert.equal(h.c.status, 'listening'); + assert.equal(h.tracks[0].enabled, false); + assert.equal(h.media[0].enabled, false); + assert.equal( + h.sockets[0].sent.length, + 1, + 'no phantom activity for a released gesture', + ); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.setMicrophoneEnabled(true); + assert.deepEqual(h.sockets[0].sent.at(-1), { + realtimeInput: { activityStart: {} }, + }); + h.c.input.setMicrophoneEnabled(false); + assert.deepEqual(h.sockets[0].sent.at(-1), { + realtimeInput: { activityEnd: {} }, + }); + h.c.stop(); + }); +} + +test('a token config that disagrees with the gesture fails before socket creation and releases consented microphone', async () => { + for (const [pushToTalk, supplied] of [ + [true, credential], + [false, manualCredential], + ]) { + const h = harness({ + fetchImpl: async () => Response.json(supplied), + }); + await h.c.start({ pushToTalk }); + assert.equal(h.c.status, 'error'); + assert.match(h.c.lastError, /input mode/); + assert.equal(h.sockets.length, 0); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.captures[0].closed, true); + } +}); + +test('old PTT callbacks cannot send to a stopped or replacement session', async () => { + const h = harness(); + h.c.input.pushToTalkKeyHeld = true; + await h.c.start({ pushToTalk: true }); + const old = h.captures[0].callbacks; + const socket = h.sockets[0]; + h.c.stop(); + const oldCount = socket.sent.length; + await h.c.start(); + const replacementCount = h.sockets[1].sent.length; + old.onAudioStart(h.captures[0].metadata); + old.onAudio( + { data: 'AAAA', mimeType: 'audio/pcm;rate=16000' }, + h.captures[0].metadata, + ); + old.onAudioEnd(h.captures[0].metadata); + assert.equal(socket.sent.length, oldCount); + assert.equal(h.sockets[1].sent.length, replacementCount); + assert.equal(h.c.status, 'listening'); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.tracks[1].enabled, true); + h.c.stop({ removeUi: true }); + const latest = h.captures[1].callbacks; + latest.onAudioStart(h.captures[1].metadata); + latest.onAudioEnd(h.captures[1].metadata); + assert.equal(h.sockets[1].sent.length, replacementCount); +}); + +test('session waits for setup, captures PCM, plays all parts, emits transcripts and releases every resource', async () => { + const h = harness(); + await h.c.start(); + assert.equal(h.c.status, 'listening'); + assert.equal(h.media.length, 1); + const ws = h.sockets[0]; + assert.equal(ws.sent[0].setup.model, 'models/gemini-3.8-live'); + h.captures[0].callbacks.onAudio( + { + data: 'AAAA', + mimeType: 'audio/pcm;rate=16000', + }, + h.captures[0].metadata, + ); + assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AAAA'); + ws.message({ + serverContent: { + inputTranscription: { text: 'Hello' }, + outputTranscription: { text: 'Hi' }, + modelTurn: { + parts: [ + { inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } }, + { inlineData: { data: 'AQAB', mimeType: 'audio/pcm;rate=24000' } }, + ], + }, + turnComplete: true, + }, + }); + assert.equal(h.captures[0].played.length, 2); + assert.equal( + h.events.some((event) => event.type === 'completion'), + false, + 'server turn completion waits for queued audio to drain', + ); + assert.ok( + h.events.some( + (event) => + event.type === 'transcript' && + event.role === 'assistant' && + event.final && + event.text === 'Hi', + ), + ); + assert.equal(h.c.getDiagnostics().completedTurns, 1); + h.captures[0].drain(); + assert.equal( + h.events.some( + (event) => event.type === 'completion' && event.status === 'completed', + ), + true, + ); + h.c.stop(); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(ws.closed, true); + assert.equal(h.captures[0].closed, true); + assert.equal(h.state.ducked, false); + assert.equal(h.c.status, 'idle'); +}); + +test('goAway deadline tears down pending work and an explicit restart creates a fresh conversation', async () => { + const pending = deferred(); + let issued = 0; + let pendingOptions; + const h = harness({ + fetchImpl: async () => + Response.json({ + ...credential, + token: `auth_tokens/session-${++issued}`, + }), + runAction: async (name, _args, options) => { + if (name === 'control_radio') + return { + ok: true, + action: 'control_radio', + radioAction: 'play', + radioPlaybackRequested: true, + }; + pendingOptions = options; + return pending.promise; + }, + }); + await h.c.start(); + establishToolInput(h); + const oldSocket = h.sockets[0]; + const oldAudio = h.captures[0]; + const oldCallbacks = oldAudio.callbacks; + oldSocket.message({ + toolCall: { + functionCalls: [ + { id: 'radio', name: 'control_radio', args: { action: 'play' } }, + { id: 'pending', name: 'get_current_view_state', args: {} }, + ], + }, + }); + await tick(); + await tick(); + assert.ok(h.c.radio.pendingRadioPlaybackResult); + assert.equal(h.c.pendingTools, 1); + oldSocket.message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + }, + }); + assert.equal(oldAudio.played.length, 1); + let sentBeforeExpiry = oldSocket.sent.length; + + oldSocket.message({ goAway: { timeLeft: '5s' } }); + sentBeforeExpiry = oldSocket.sent.length; // Microphone stream-end is a reliable lifecycle marker. + assert.equal(h.c.status, 'executing'); + assert.equal(pendingOptions.signal.aborted, false); + assert.equal(h.c.sendTextCommand('Do not admit a new turn'), false); + h.advance(5000); + assert.equal(h.c.status, 'error'); + assert.equal( + h.c.lastError, + 'Gemini Live connection is expiring. Start voice again for a new conversation.', + ); + assert.equal( + h.events.at(-1).recovery, + 'Start voice again for a fresh token and a new conversation.', + ); + assert.equal(oldSocket.closed, true); + assert.equal(oldAudio.closed, true); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(pendingOptions.signal.aborted, true); + + oldCallbacks.onAudioStart(h.captures[0].metadata); + oldCallbacks.onAudio( + { data: 'AQAB', mimeType: 'audio/pcm;rate=16000' }, + h.captures[0].metadata, + ); + oldCallbacks.onAudioEnd(h.captures[0].metadata); + oldCallbacks.onDrain(); + oldCallbacks.onError(new Error('late retired audio')); + oldSocket.message({ serverContent: { turnComplete: true } }); + oldSocket.error(); + oldSocket.remoteClose(); + pending.resolve({ ok: true }); + await tick(); + await tick(); + assert.equal(oldSocket.sent.length, sentBeforeExpiry); + assert.equal(oldAudio.played.length, 1); + assert.equal( + oldSocket.sent.filter((entry) => entry.toolResponse).length, + 1, + 'only the completed Radio result was sent before expiry', + ); + assert.equal(h.state.playing, 0); + + await h.c.start(); + assert.equal(issued, 2); + assert.equal(h.sockets.length, 2); + assert.equal( + new URL(h.sockets[1].url).searchParams.get('access_token'), + 'auth_tokens/session-2', + ); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.completedTurns, 0); + assert.equal(h.captures[1].played.length, 0); + assert.equal(h.tracks[1].readyState, 'live'); + h.c.stop(); +}); + +test('pre-setup rejection or close releases startup and asks for a fresh session', async () => { + for (const terminal of ['rejected', 'closed']) { + const h = harness({ autoSetup: false }); + const start = h.c.start(); + await tick(); + if (terminal === 'rejected') + h.sockets[0].message({ error: { message: 'credential rejected' } }); + else h.sockets[0].remoteClose(); + await start; + assert.equal(h.c.status, 'error', terminal); + assert.equal(h.tracks[0].readyState, 'ended', terminal); + assert.equal(h.captures[0].closed, true, terminal); + assert.equal(h.sockets[0].closed, true, terminal); + assert.match(h.c.lastError, /Start voice again/, terminal); + assert.match(h.events.at(-1).recovery, /fresh session/, terminal); + assert.equal(h.sockets.length, 1, 'no automatic retry'); + } +}); + +test('network close after setup releases media and starts no replacement', async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].remoteClose(); + assert.equal(h.c.status, 'error'); + assert.equal( + h.c.lastError, + 'Gemini Live connection ended. Start voice again for a new conversation.', + ); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.captures[0].closed, true); + assert.equal(h.sockets.length, 1); + assert.doesNotMatch(h.events.at(-1).recovery, /microphone|API key/i); +}); + +test('OFF during unresolved setup blocks late setup, error and close callbacks', async () => { + const h = harness({ autoSetup: false }); + const start = h.c.start(); + await tick(); + const socket = h.sockets[0]; + h.c.stop(); + const eventCount = h.events.length; + socket.message({ setupComplete: {} }); + socket.message({ error: { message: 'late rejection' } }); + socket.error(); + socket.remoteClose(); + await start; + assert.equal(h.c.status, 'idle'); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.captures[0].closed, true); + assert.equal(h.events.length, eventCount); +}); + +test('a disposed pending microphone is stopped and cannot update the replacement UI', async () => { + const microphone = deferred(); + const h = harness({ getUserMedia: () => microphone.promise }); + const start = h.c.start(); + await tick(); + h.c.stop({ removeUi: true }); + const eventCount = h.events.length; + microphone.resolve(h.makeStream()); + await start; + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.events.length, eventCount); + assert.equal(h.media.length, 0); +}); + +test('late token rejection after stop cannot replace a successor state', async () => { + const response = deferred(); + const h = harness({ fetchImpl: () => response.promise }); + const start = h.c.start(); + h.c.stop({ removeUi: true }); + const count = h.events.length; + response.resolve(Response.json({ error: 'late rejection' }, { status: 503 })); + await start; + assert.equal(h.events.length, count); + assert.equal(h.sockets.length, 0); +}); + +test('tool cancellation covers queued calls and interrupted results never return to the socket', async () => { + const first = deferred(); + const names = []; + const h = harness({ + runAction: async (name) => { + names.push(name); + return first.promise; + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'a', name: 'one', args: {} }, + { id: 'b', name: 'two', args: {} }, + ], + }, + }); + await tick(); + ws.message({ toolCallCancellation: { ids: ['b'] } }); + first.resolve({ ok: true }); + await h.c.toolQueue; + assert.deepEqual(names, ['one']); + assert.equal(h.c.pendingTools, 0); + assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1); + const second = deferred(); + h.c.runAction = () => second.promise; + ws.message({ + toolCall: { functionCalls: [{ id: 'c', name: 'three', args: {} }] }, + }); + await tick(); + ws.message({ serverContent: { interrupted: true } }); + second.resolve({ ok: true }); + await tick(); + assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1); + assert.equal(h.captures[0].cleared, true); + h.c.stop(); +}); + +test('wire voiceActivity.type start aborts pending work and discards old playback', async () => { + const pending = deferred(); + let signal; + const h = harness({ + runAction: (_name, _args, options) => { + signal = options.signal; + return pending.promise; + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, + }); + ws.message({ + toolCall: { + functionCalls: [ + { id: 'old', name: 'fly_to_location', args: { query: 'Austin' } }, + ], + }, + }); + await tick(); + const turn = h.c.turn; + assert.equal(signal.aborted, false); + assert.equal(h.captures[0].pending, true); + ws.message({ + voiceActivity: { type: 'ACTIVITY_START', audioOffset: '5.240s' }, + }); + assert.equal(signal.aborted, true); + assert.ok(h.c.turn > turn); + assert.equal(h.c.pendingTools, 0); + assert.equal(h.captures[0].cleared, true); + assert.equal(h.c.status, 'listening'); + pending.resolve({ ok: true }); + await tick(); + assert.equal( + ws.sent.some((entry) => entry.toolResponse), + false, + ); + h.c.stop(); +}); + +test('unknown or malformed voice activity cannot revoke the current turn', async () => { + const h = harness(); + await h.c.start(); + const turn = h.c.turn; + for (const voiceActivity of [ + null, + {}, + { type: 'TYPE_UNSPECIFIED' }, + { type: 'unknown' }, + { type: 1 }, + ]) { + h.sockets[0].message({ voiceActivity }); + assert.equal(h.c.turn, turn); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.narration.state().pendingTimers, 0); + } + h.c.stop(); +}); + +for (const pushToTalk of [false, true]) { + test(`wire voiceActivity.type end respects ${pushToTalk ? 'manual' : 'automatic'} turn ownership`, async () => { + const h = harness(); + await h.c.start({ pushToTalk }); + if (!pushToTalk) + h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); + h.c.narration.cancel(); + h.sockets[0].message({ + voiceActivity: { type: 'ACTIVITY_END', audioOffset: '9.080s' }, + }); + assert.equal(h.c.narration.state().pendingTimers, pushToTalk ? 0 : 1); + assert.equal(h.c.status, 'listening'); + h.c.stop(); + assert.equal(h.timers.size, 0); + }); +} + +test('radio handoff waits for a post-tool reply and actual playback drain', async () => { + const h = harness({ + runAction: async () => ({ + ok: true, + action: 'control_radio', + radioAction: 'play', + radioPlaybackRequested: true, + }), + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'radio', name: 'control_radio', args: { action: 'play' } }, + ], + }, + }); + await h.c.toolQueue; + ws.message({ serverContent: { turnComplete: true } }); + assert.equal( + h.state.playing, + 0, + 'the tool turn alone must not release Radio', + ); + ws.message({ + serverContent: { + modelTurn: { + parts: [ + { inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } }, + ], + }, + turnComplete: true, + }, + }); + assert.equal( + h.state.playing, + 0, + 'queued assistant audio still owns the speaker', + ); + h.captures[0].drain(); + await tick(); + assert.equal(h.state.playing, 1); + assert.equal(h.state.ducked, false); + assert.equal(h.c.status, 'idle'); +}); + +test('explicit Radio stop cancels a handoff while confirmation is playing', async () => { + const h = harness({ + runAction: async () => ({ + ok: true, + action: 'control_radio', + radioPlaybackRequested: true, + }), + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'r', name: 'control_radio', args: { action: 'play' } }, + ], + }, + }); + await h.c.toolQueue; + ws.message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + turnComplete: true, + }, + }); + h.radioControl({ action: 'stop', origin: 'user' }); + h.captures[0].drain(); + await tick(); + assert.equal(h.state.playing, 0); + assert.equal(h.c.status, 'listening'); + h.c.stop(); +}); + +test('eligible entity context sends a bounded fresh image before its function result', async () => { + const h = harness({ + runAction: async () => ({ + ok: true, + action: 'get_entity_context', + scene: { basemap: { viewScale: 'local' } }, + }), + captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=', + }); + await h.c.start(); + establishToolInput(h); + h.sockets[0].message({ + toolCall: { + functionCalls: [{ id: 'view', name: 'get_entity_context', args: {} }], + }, + }); + await h.c.toolQueue; + const sent = h.sockets[0].sent; + const imageIndex = sent.findIndex((entry) => entry.realtimeInput?.video); + assert.ok(imageIndex > 0); + assert.ok(imageIndex < sent.findIndex((entry) => entry.toolResponse)); + h.c.stop(); +}); + +test('socket pressure drops PCM without failing the active session and delivery recovers', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0], + audio = h.captures[0], + before = ws.sent.length; + ws.bufferedAmount = 300000; + audio.callbacks.onAudio( + { data: 'AA==', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + assert.equal(ws.sent.length, before); + assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1); + assert.equal(h.c.status, 'listening'); + assert.equal(h.tracks[0].readyState, 'live'); + ws.bufferedAmount = 0; + audio.callbacks.onAudio( + { data: 'AQ==', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AQ=='); + h.c.stop(); +}); + +test('passive map context never interrupts spoken output and is delivered at the next explicit turn', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ + serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, + }); + const before = ws.sent.length; + const event = { + type: 'map_annotation_outline', + label: 'Austin', + status: 'resolved', + }; + assert.equal(h.c.notifyMapEvent(event), true); + event.label = 'changed after enqueue'; + assert.equal( + ws.sent.length, + before, + 'passive context sends no clientContent while speech is active', + ); + assert.equal(h.captures[0].pending, true); + h.c.sendTextCommand('What did you outline?'); + const parts = ws.sent.at(-1).clientContent.turns[0].parts; + assert.equal(JSON.parse(parts[0].text).events[0].label, 'Austin'); + assert.equal(parts[1].text, 'What did you outline?'); + assert.equal(h.c.pendingMapEvents.length, 0); + for (let i = 0; i < 30; i++) h.c.notifyMapEvent({ index: i }); + assert.equal(h.c.pendingMapEvents.length, 16); + assert.equal(h.c.notifyMapEvent({ text: 'x'.repeat(16000) }), false); + h.c.stop(); + assert.equal(h.c.pendingMapEvents.length, 0); +}); + +test('passive context joins the next tool response and survives a failed explicit send', async () => { + const h = harness(); + await h.c.start(); + h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'London' }); + h.sockets[0].readyState = 0; + assert.equal(h.c.sendTextCommand('Continue'), false); + assert.equal(h.c.pendingMapEvents.length, 1); + h.sockets[0].readyState = 1; + h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); + h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_END' } }); + h.sockets[0].message({ + serverContent: { inputTranscription: { text: 'Continue' } }, + }); + h.sockets[0].message({ + toolCall: { + functionCalls: [{ id: 'context', name: 'get_entity_context', args: {} }], + }, + }); + await h.c.toolQueue; + assert.equal( + h.sockets[0].sent.at(-1).toolResponse.functionResponses[0].response + .mapEvents[0].label, + 'London', + ); + assert.equal(h.c.pendingMapEvents.length, 0); + h.c.stop(); +}); + +test('a delayed permission prompt does not mint a token or consume the network deadline', async () => { + const microphone = deferred(); + const h = harness({ + startupTimeoutMs: 40, + getUserMedia: () => microphone.promise, + }); + const start = h.c.start(); + await new Promise((resolve) => setTimeout(resolve, 80)); + assert.equal(h.c.status, 'connecting'); + assert.equal(h.requests.length, 0); + assert.equal(h.sockets.length, 0); + microphone.resolve(h.makeStream()); + await start; + assert.equal(h.c.status, 'listening'); + assert.equal(h.requests.length, 1); + h.c.stop(); +}); + +test('Stop during a permission prompt releases a late grant without touching the successor', async () => { + const microphone = deferred(); + const h = harness({ getUserMedia: () => microphone.promise }); + const start = h.c.start(); + await tick(); + h.c.stop(); + await start; + const lateStream = h.makeStream(); + h.c.getUserMedia = async () => h.makeStream(); + await h.c.start(); + microphone.resolve(lateStream); + await tick(); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.tracks[1].readyState, 'live'); + assert.equal(h.c.status, 'listening'); + h.c.stop(); +}); + +test('stalled audio initialization also obeys the startup deadline', async () => { + const initialization = deferred(); + const h = harness({ startupTimeoutMs: 40 }); + const createAudio = h.c.createAudio; + h.c.createAudio = (callbacks) => { + const audio = createAudio(callbacks); + audio.initialize = () => initialization.promise; + return audio; + }; + await Promise.all([ + h.c.start(), + new Promise((resolve) => setTimeout(resolve, 80)), + ]); + assert.equal(h.c.status, 'error'); + assert.equal(h.captures[0].closed, true); + assert.equal(h.sockets.length, 0); + initialization.resolve(); + await tick(); + assert.equal(h.c.status, 'error'); +}); + +for (const activity of [ + 'activity-start', + 'push-to-talk', + 'interrupted', + 'user-text', +]) { + test(`new ${activity} revokes a pending Radio preflight after the old spoken turn ended`, async () => { + const ready = deferred(); + const h = harness({ + runAction: async () => ({ + ok: true, + action: 'control_radio', + radioPlaybackRequested: true, + }), + }); + h.c.radio.radioLayer.playForVoice = () => ready.promise; + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'radio', name: 'control_radio', args: { action: 'play' } }, + ], + }, + }); + await h.c.toolQueue; + ws.message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + turnComplete: true, + }, + }); + h.captures[0].drain(); + assert.equal(h.c.radio.radioHandoffInFlight, true); + if (activity === 'activity-start') + ws.message({ + voiceActivity: { + type: 'ACTIVITY_START', + audioOffset: '12s', + }, + }); + else if (activity === 'interrupted') + ws.message({ serverContent: { interrupted: true } }); + else if (activity === 'user-text') h.c.sendTextCommand('Actually stop'); + else h.c.input.setMicrophoneEnabled(true); + ready.resolve(true); + await tick(); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.stream.getTracks()[0].readyState, 'live'); + assert.equal(h.state.ducked, true); + assert.ok(h.state.stopped > 0); + h.c.stop(); + }); +} + +test('uncorrelated late transcription preserves its own successful Radio action and preflight', async () => { + const tool = deferred(); + const ready = deferred(); + const h = harness({ runAction: () => tool.promise }); + h.c.radio.radioLayer.playForVoice = () => ready.promise; + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'radio-late-transcript', + name: 'control_radio', + args: { action: 'play' }, + }, + ], + }, + }); + await tick(); + const handoffEpoch = h.c.radio.radioHandoffEpoch; + ws.message({ + serverContent: { + inputTranscription: { text: 'Play the radio' }, + interimInputTranscription: { text: 'Play the radio' }, + }, + }); + assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch); + tool.resolve({ + ok: true, + action: 'control_radio', + radioPlaybackRequested: true, + }); + await h.c.toolQueue; + const response = ws.sent.find((message) => message.toolResponse).toolResponse + .functionResponses[0].response; + assert.equal(response.ok, true); + assert.equal(response.radioPlaybackRequested, true); + ws.message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + turnComplete: true, + }, + }); + h.captures[0].drain(); + assert.equal(h.c.radio.radioHandoffInFlight, true); + ws.message({ serverContent: { inputTranscription: { text: ' please' } } }); + assert.equal(h.c.radio.radioHandoffInFlight, true); + ready.resolve(true); + await tick(); + assert.equal(h.c.status, 'idle'); + assert.equal(h.state.ducked, false); +}); + +function radioPlaybackResult() { + return { + ok: true, + action: 'control_radio', + radioAction: 'play', + radioPlaybackRequested: true, + }; +} + +function establishToolInput(h, text = 'Run the requested action') { + assert.equal(h.c.sendTextCommand(text), true); +} + +function requestTool( + ws, + id, + name = 'control_radio', + args = { action: 'play' }, +) { + ws.message({ toolCall: { functionCalls: [{ id, name, args }] } }); +} + +function drainRadioReply(h) { + h.sockets[0].message({ + serverContent: { + modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, + turnComplete: true, + }, + }); + h.captures[0].drain(); +} + +test('cancelling one queued Radio call preserves active and queued same-feature siblings', async () => { + const first = deferred(); + const executions = []; + const h = harness({ + runAction: (name, args, options) => { + executions.push({ args, options }); + return args.station === 'first' ? first.promise : radioPlaybackResult(); + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + for (const station of ['first', 'cancelled', 'last']) + requestTool(ws, station, 'control_radio', { action: 'select', station }); + await tick(); + const epoch = h.c.radio.radioHandoffEpoch; + ws.message({ toolCallCancellation: { ids: ['cancelled'] } }); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + assert.equal(executions[0].options.isCurrent(), true); + first.resolve(radioPlaybackResult()); + await h.c.toolQueue; + assert.deepEqual( + executions.map((entry) => entry.args.station), + ['first', 'last'], + ); + assert.ok(executions.every((entry) => !entry.options.signal.aborted)); + assert.equal(h.c.pendingTools, 0); + const newer = h.c.radio.pendingRadioPlaybackResult; + ws.message({ toolCallCancellation: { ids: ['first', 'unknown'] } }); + assert.equal(h.c.radio.pendingRadioPlaybackResult, newer); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + h.c.stop(); +}); + +test('matching active Radio cancellation aborts its work and permits its sibling to finish', async () => { + const first = deferred(); + const executions = []; + const h = harness({ + runAction: (name, args, options) => { + executions.push(options); + return args.action === 'play' ? first.promise : radioPlaybackResult(); + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'old'); + requestTool(ws, 'new', 'control_radio', { action: 'select' }); + await tick(); + const epoch = h.c.radio.radioHandoffEpoch; + ws.message({ toolCallCancellation: { ids: ['old'] } }); + assert.equal(executions[0].signal.aborted, true); + assert.equal(executions[0].isCurrent(), false); + await h.c.toolQueue; + const pending = h.c.radio.pendingRadioPlaybackResult; + assert.ok(pending); + assert.equal(executions[1].isCurrent(), true); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + first.resolve(radioPlaybackResult()); + await tick(); + assert.equal(h.c.radio.pendingRadioPlaybackResult, pending); + assert.deepEqual( + ws.sent + .flatMap((m) => m.toolResponse?.functionResponses || []) + .map((r) => r.id), + ['new'], + ); + h.c.stop(); +}); + +test('matching prepared cancellation preserves a stronger sibling reservation and signal', async () => { + const stronger = deferred(); + let strongerOptions; + const h = harness({ + runAction: (name, args, options) => { + if (args.action === 'stop') { + strongerOptions = options; + return stronger.promise; + } + return radioPlaybackResult(); + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'play'); + await h.c.toolQueue; + requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); + await tick(); + const reservations = [...h.c.radio.radioToolHandoffReservations.keys()]; + assert.equal(reservations.length, 1); + const epoch = h.c.radio.radioHandoffEpoch; + ws.message({ toolCallCancellation: { ids: ['play', 'unknown'] } }); + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(strongerOptions.signal.aborted, false); + assert.deepEqual( + [...h.c.radio.radioToolHandoffReservations.keys()], + reservations, + ); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + stronger.resolve({ ok: false, error: 'Refused fixture stop' }); + await h.c.toolQueue; + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + drainRadioReply(h); + await tick(); + assert.equal(h.state.playing, 0); + assert.equal(h.c.status, 'listening'); + h.c.stop(); +}); + +test('cancelled stronger call releases only its reservation and retains prepared playback', async () => { + const stronger = deferred(); + const h = harness({ + runAction: (name, args) => + args.action === 'stop' ? stronger.promise : radioPlaybackResult(), + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'play'); + await h.c.toolQueue; + const prepared = h.c.radio.pendingRadioPlaybackResult; + requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); + await tick(); + const separateReservation = h.c.radio.reserveRadioToolHandoff(); + ws.message({ toolCallCancellation: { ids: ['stop'] } }); + await h.c.toolQueue; + assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); + assert.deepEqual( + [...h.c.radio.radioToolHandoffReservations.keys()], + [separateReservation], + ); + stronger.resolve({ ok: true, action: 'control_radio' }); + await tick(); + assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); + h.c.radio.settleRadioToolHandoffReservation(separateReservation); + drainRadioReply(h); + await tick(); + assert.equal(h.state.playing, 1); + assert.equal(h.c.status, 'idle'); +}); + +test('matching preflight cancellation and late completion cannot revoke a newer reused-result handoff', async () => { + const sharedResult = radioPlaybackResult(); + const starts = []; + const stops = []; + let playingAttempt = null; + const h = harness({ runAction: () => sharedResult }); + h.c.radio.radioLayer.playForVoice = ({ attemptId }) => { + const ready = deferred(); + starts.push({ attemptId, ready }); + playingAttempt = attemptId; + return ready.promise; + }; + h.c.radio.radioLayer.stopPlayback = ({ attemptId }) => { + stops.push(attemptId); + if (playingAttempt === attemptId) playingAttempt = null; + }; + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'old'); + await h.c.toolQueue; + drainRadioReply(h); + const oldResult = h.c.radio.radioHandoffInFlightResult; + establishToolInput(h, 'Play the radio again'); + requestTool(ws, 'new'); + await h.c.toolQueue; + const newerResult = h.c.radio.pendingRadioPlaybackResult; + assert.notEqual(oldResult, newerResult); + assert.notEqual(newerResult, sharedResult); + const epoch = h.c.radio.radioHandoffEpoch; + ws.message({ toolCallCancellation: { ids: ['old'] } }); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + assert.equal(h.c.radio.pendingRadioPlaybackResult, newerResult); + assert.equal(h.c.radio.radioHandoffInFlight, false); + assert.deepEqual(stops, [starts[0].attemptId]); + drainRadioReply(h); + assert.equal(starts.length, 2); + const newerAttempt = starts[1].attemptId; + starts[0].ready.resolve(true); + await tick(); + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt); + assert.equal(playingAttempt, newerAttempt); + assert.ok(stops.every((id) => id === starts[0].attemptId)); + ws.message({ toolCallCancellation: { ids: ['old', 'unknown'] } }); + assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt); + starts[1].ready.resolve(true); + await tick(); + assert.equal(h.c.status, 'idle'); + assert.equal(playingAttempt, newerAttempt); +}); + +test('cancelled preflight rejection cannot emit a stale Radio failure correction', async () => { + let rejectReady; + const ready = new Promise((_, reject) => { + rejectReady = reject; + }); + const corrections = []; + const h = harness({ runAction: () => radioPlaybackResult() }); + h.c.radio.radioLayer.playForVoice = () => ready; + h.c.radio.queueResponseCreate = (text) => corrections.push(text); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'old'); + await h.c.toolQueue; + drainRadioReply(h); + ws.message({ toolCallCancellation: { ids: ['old'] } }); + rejectReady(new Error('late obsolete stream rejection')); + await tick(); + assert.deepEqual(corrections, []); + assert.equal(h.c.status, 'listening'); + h.c.stop(); +}); + +test('completed query and unknown cancellation IDs are inert and cannot poison future calls', async () => { + const executions = []; + const h = harness({ + runAction: (name, args) => { + executions.push(args.action); + return args.action === 'status' + ? { ok: true, action: name } + : radioPlaybackResult(); + }, + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'query', 'control_radio', { action: 'status' }); + await h.c.toolQueue; + requestTool(ws, 'owner'); + await h.c.toolQueue; + const prepared = h.c.radio.pendingRadioPlaybackResult; + const epoch = h.c.radio.radioHandoffEpoch; + ws.message({ toolCallCancellation: { ids: ['query', 'future'] } }); + assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); + assert.equal(h.c.radio.radioHandoffEpoch, epoch); + requestTool(ws, 'future', 'control_radio', { action: 'select' }); + await h.c.toolQueue; + assert.deepEqual(executions, ['status', 'play', 'select']); + h.c.stop(); +}); + +for (const replacement of ['user-activity', 'stop']) { + test(`${replacement} still revokes all prepared and preflight Radio ownership`, async () => { + const ready = deferred(); + const h = harness({ runAction: () => radioPlaybackResult() }); + h.c.radio.radioLayer.playForVoice = () => ready.promise; + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'old'); + await h.c.toolQueue; + drainRadioReply(h); + h.c.radio.setPendingPlayback(radioPlaybackResult()); + assert.ok(h.c.radio.pendingRadioPlaybackResult); + assert.equal(h.c.radio.radioHandoffInFlight, true); + if (replacement === 'user-activity') h.c.noteUserActivity(); + else h.c.stop(); + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.equal(h.c.radio.radioHandoffInFlight, false); + assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); + assert.ok(h.state.stopped > 0); + ready.resolve(true); + await tick(); + assert.equal(h.c.status, replacement === 'stop' ? 'idle' : 'listening'); + if (replacement === 'user-activity') { + assert.equal(h.c.stream.getTracks()[0].readyState, 'live'); + h.c.stop(); + } + }); +} + +test('successful stronger Radio stop still invalidates the prepared group', async () => { + const h = harness({ + runAction: (name, args) => + args.action === 'stop' + ? { ok: true, action: name, radioAction: 'stop' } + : radioPlaybackResult(), + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0]; + requestTool(ws, 'play'); + await h.c.toolQueue; + const epoch = h.c.radio.radioHandoffEpoch; + requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); + await h.c.toolQueue; + assert.equal(h.c.radio.pendingRadioPlaybackResult, null); + assert.ok(h.c.radio.radioHandoffEpoch > epoch); + drainRadioReply(h); + await tick(); + assert.equal(h.state.playing, 0); + assert.equal(h.c.status, 'listening'); + h.c.stop(); +}); + +test('slow Gemini tools reuse shared progress timing without injecting a conversation turn', async () => { + const pending = deferred(); + let options; + const h = harness({ + runAction: (_name, _args, next) => { + options = next; + next.progress({ step: 'search', label: 'Austin' }); + return pending.promise; + }, + }); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Fly to Austin'); + const ws = h.sockets[0]; + requestTool(ws, 'flight', 'fly_to_location', { query: 'Austin' }); + await tick(); + assert.equal( + options.callId, + 'flight', + 'shared card and narration use one call identity', + ); + assert.equal(h.c.narration.state().tools, 1); + h.advance(699); + assert.equal(h.earcon.playing, false); + h.advance(1); + assert.equal(h.earcon.playing, true); + h.advance(7500); + assert.equal( + ws.sent.length, + 2, + 'progress never sends clientContent or an early tool response', + ); + assert.equal(h.c.narration.state().stepLines, 0); + assert.equal(h.c.narration.state().stillSpoken, false); + pending.resolve({ ok: true, action: 'fly_to_location' }); + await h.c.toolQueue; + assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'flight'); + assert.equal(h.c.narration.state().tools, 0); + h.advance(3000); + assert.equal(h.earcon.playing, false, 'reply grace bounds the final earcon'); + assert.equal(h.timers.size, 0); + h.c.stop(); +}); + +test('fast Gemini tools retract pending progress and Radio tools do not schedule lines', async () => { + for (const name of ['fly_to_location', 'control_radio']) { + const h = harness(); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Run command'); + requestTool( + h.sockets[0], + 'fast', + name, + name === 'control_radio' ? { action: 'status' } : {}, + ); + await h.c.toolQueue; + h.advance(20000); + assert.equal(h.earcon.starts, 0); + assert.equal(h.timers.size, 0); + assert.equal( + h.sockets[0].sent.filter((message) => message.clientContent).length, + 1, + ); + h.c.stop(); + } +}); + +test('Gemini model preamble and playback take priority over the working earcon', async () => { + const pending = deferred(); + const h = harness({ runAction: () => pending.promise }); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Fly to Austin'); + const ws = h.sockets[0]; + ws.message({ + serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, + }); + requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' }); + await tick(); + assert.equal(h.c.narration.state().preambleSpoken, true); + h.advance(1500); + assert.equal(h.earcon.starts, 0, 'queued model audio owns the speaker'); + h.captures[0].drain(); + h.advance(700); + assert.equal(h.earcon.playing, true); + ws.message({ + serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, + }); + assert.equal( + h.earcon.playing, + false, + 'new model audio immediately stops the tone', + ); + h.c.stop(); + pending.resolve({ ok: true }); + await tick(); +}); + +test('PTT release starts shared progress timing and a new hold cancels it', async () => { + const pending = deferred(); + let options; + const h = harness({ + runAction: (_name, _args, next) => { + options = next; + return pending.promise; + }, + }); + h.c.input.pushToTalkKeyHeld = true; + await h.c.start({ pushToTalk: true }); + assert.equal(h.timers.size, 0); + h.c.input.pushToTalkKeyHeld = false; + h.c.input.setMicrophoneEnabled(false); + h.sockets[0].message({ + serverContent: { inputTranscription: { text: 'fly to Austin' } }, + }); + requestTool(h.sockets[0], 'held', 'fly_to_location', { query: 'Austin' }); + await tick(); + h.advance(700); + assert.equal(h.earcon.playing, true); + h.c.input.pushToTalkKeyHeld = true; + assert.equal(h.c.input.mayClaimSpeaker(), true); + assert.equal(h.c.input.bargeIn(), true); + h.c.input.setMicrophoneEnabled(true); + assert.equal(h.earcon.playing, false); + assert.ok(h.c.retiredBoundaryWait, 'interruption retains its recovery bound'); + assert.deepEqual([...h.timers.keys()], [h.c.retiredBoundaryWait.timer]); + assert.equal(options.signal.aborted, true); + assert.equal(h.captures[0].cleared, true); + assert.equal(h.c.narration.state().preambleSpoken, false); + pending.resolve({ ok: true }); + await tick(); + assert.equal( + h.sockets[0].sent.some((message) => message.toolResponse), + false, + ); + h.c.stop(); +}); + +test('open microphone and Radio speaker ownership suppress Gemini earcons', async () => { + for (const mode of ['open-mic', 'held-ptt', 'radio']) { + const pending = deferred(); + const h = harness({ runAction: () => pending.promise }); + await h.c.start({ pushToTalk: mode !== 'open-mic' }); + h.c.sendTextCommand('Fly to Austin'); + requestTool(h.sockets[0], 'slow', 'fly_to_location', { query: 'Austin' }); + await tick(); + if (mode === 'held-ptt') h.c.input.pushToTalkKeyHeld = true; + if (mode === 'radio') { + h.c.radio.setPendingPlayback(radioPlaybackResult()); + assert.equal(h.c.input.mayClaimSpeaker(), false); + assert.equal( + h.c.input.bargeIn(), + false, + 'a hold cannot steal a Radio handoff', + ); + assert.equal(h.c.pendingTools, 1); + } + h.advance(20000); + assert.equal(h.earcon.starts, 0, mode); + h.c.stop(); + pending.resolve({ ok: true }); + await tick(); + } +}); + +test('Gemini progress is bounded and all terminal paths release narration ownership', async () => { + for (const end of ['stop', 'goAway', 'destroy', 'interruption']) { + const pending = deferred(); + const h = harness({ runAction: () => pending.promise }); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Fly to Austin'); + const ws = h.sockets[0]; + requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' }); + await tick(); + h.advance(700); + assert.equal(h.earcon.playing, true); + h.advance(15000); + assert.equal( + h.earcon.playing, + false, + 'working tone never runs beyond the shared limit', + ); + const disposals = h.earcon.disposals; + if (end === 'goAway') ws.message({ goAway: { timeLeft: '0s' } }); + else if (end === 'interruption') + ws.message({ serverContent: { interrupted: true } }); + else h.c.stop({ removeUi: end === 'destroy' }); + assert.equal(h.c.narration.state().tools, 0, end); + const retirementTimer = h.c.retiredBoundaryWait?.timer; + assert.deepEqual( + [...h.timers.keys()], + end === 'interruption' ? [retirementTimer] : [], + 'only the interrupted reply recovery timer may remain', + ); + if (end === 'interruption') assert.ok(h.c.retiredBoundaryWait); + assert.equal(h.earcon.playing, false, end); + if (end !== 'interruption') + assert.equal(h.earcon.disposals, disposals + 1, end); + pending.resolve({ ok: true }); + await tick(); + h.advance(20000); + assert.equal( + ws.sent.some((message) => message.toolResponse), + false, + end, + ); + assert.equal(h.timers.size, 0, end); + h.c.stop(); + } +}); + +test('late tool and audio callbacks cannot change a successor Gemini narrator', async () => { + const first = deferred(); + const second = deferred(); + const options = []; + const h = harness({ + runAction: (_name, _args, next) => { + options.push(next); + return options.length === 1 ? first.promise : second.promise; + }, + }); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Fly to Austin'); + requestTool(h.sockets[0], 'reused', 'fly_to_location', { query: 'Austin' }); + await tick(); + const oldAudio = h.captures[0].callbacks; + h.c.stop(); + await h.c.start({ pushToTalk: true }); + h.c.sendTextCommand('Fly to Tokyo'); + requestTool(h.sockets[1], 'reused', 'fly_to_location', { query: 'Tokyo' }); + await tick(); + const before = h.c.narration.state(); + options[0].progress({ step: 'fly', label: 'Austin' }); + oldAudio.onAudioEnd(h.captures[0].metadata); + oldAudio.onDrain(); + first.resolve({ ok: true }); + await tick(); + assert.deepEqual(h.c.narration.state(), before); + assert.equal(h.c.pendingTools, 1); + h.advance(700); + assert.equal(h.earcon.playing, true); + second.resolve({ ok: true }); + await h.c.toolQueue; + assert.equal(h.c.narration.state().tools, 0); + h.c.stop(); + assert.equal(h.timers.size, 0); +}); + +test('Gemini tool progress reaches the shared card event contract with its call id', async () => { + const pending = deferred(); + const events = []; + let options; + let h; + const session = createVoiceSession({ + runner: (_name, _args, next) => { + options = next; + next.progress({ step: 'search', label: 'Austin' }); + return pending.promise; + }, + createAdapter: (shared) => { + h = harness(shared); + return { + start: (settings) => h.c.start(settings), + stop: (settings) => h.c.stop(settings), + sendText: (text) => h.c.sendTextCommand(text), + sendMapEvent: (event) => h.c.notifyMapEvent(event), + }; + }, + }); + session.subscribe((event) => events.push(event)); + await session.start({ pushToTalk: true }); + establishToolInput(h); + requestTool(h.sockets[0], 'shared-call', 'fly_to_location', { + query: 'Austin', + }); + await tick(); + assert.equal( + events.find((event) => event.type === 'action-call').callId, + 'shared-call', + ); + assert.deepEqual( + events.find((event) => event.type === 'progress'), + { + type: 'progress', + name: 'fly_to_location', + callId: 'shared-call', + step: 'search', + label: 'Austin', + }, + ); + pending.resolve({ ok: true, action: 'fly_to_location' }); + await h.c.toolQueue; + assert.equal( + events.find((event) => event.type === 'action-result').callId, + 'shared-call', + ); + session.stop(); + const progressCount = events.filter( + (event) => event.type === 'progress', + ).length; + options.progress({ step: 'fly', label: 'Stale Austin' }); + assert.equal( + events.filter((event) => event.type === 'progress').length, + progressCount, + ); + assert.equal(h.timers.size, 0); + session.destroy(); +}); + +function pointerHarness() { + let clock = 1000; + const pointer = createPointerTracker({ + element: { + clientWidth: 1000, + clientHeight: 500, + addEventListener() {}, + }, + now: () => clock, + setTimer: () => 1, + clearTimer() {}, + pick: (x) => ({ + entity: { + layerId: 'flights', + id: String(x), + label: `Aircraft ${x}`, + kind: 'aircraft', + lat: 30, + lon: -97, + }, + }), + }); + return { + pointer, + advance: (ms) => { + clock += ms; + }, + }; +} + +const replyAudio = (data) => ({ + modelTurn: { + parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }], + }, +}); + +test('B48 pointer: a cold held PTT turn freezes keydown and sends quiet context before activity', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + pointer.moveTo(300, 200); + h.c.input.holdPointer(); + pointer.moveTo(700, 200); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.spaceKeyHeld = true; + await h.c.start({ pushToTalk: true }); + assert.equal(pointer.activeSnapshot().entity.id, '300'); + assert.deepEqual( + h.events.filter((event) => event.type === 'pointer').at(-1).pointer, + { + kind: 'this', + label: 'Aircraft 300', + screenPx: { x: 300, y: 200 }, + }, + ); + const start = h.sockets[0].sent.find( + (value) => value.realtimeInput?.activityStart, + ); + const context = h.sockets[0].sent.find((value) => value.clientContent); + assert.equal( + JSON.parse(context.clientContent.turns[0].parts[0].text).target.label, + 'Aircraft 300', + ); + assert.equal(context.clientContent.turns[0].role, 'user'); + assert.equal(context.clientContent.turnComplete, false); + assert.deepEqual(start, { realtimeInput: { activityStart: {} } }); + assert.equal( + h.sockets[0].sent.indexOf(context) < h.sockets[0].sent.indexOf(start), + true, + ); + assert.equal( + h.c.replyPending, + false, + 'context and an empty hold request no reply', + ); + assert.equal( + h.sockets[0].sent.some( + (value) => value.clientContent?.turnComplete === true, + ), + false, + ); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + // A delayed server onset must not replace the released gesture's referent. + h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); + assert.equal(pointer.activeSnapshot().entity.id, '300'); + h.sockets[0].message({ + serverContent: { inputTranscription: { text: 'what is this' } }, + }); + h.sockets[0].message({ + toolCall: { + functionCalls: [ + { + id: 'pointed', + name: 'get_entity_context', + args: { scope: 'pointer' }, + }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(h.calls[0].args.scope, 'pointer'); + assert.equal(pointer.activeSnapshot().entity.id, '300'); + h.c.stop(); + assert.equal(pointer.activeSnapshot(), null); + pointer.destroy(); +}); + +test('B48 pointer: text uses recent availability, clears stale context and sends one turn', async () => { + const { pointer, advance } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start(); + pointer.moveTo(400, 200); + h.c.sendTextCommand('what is this?'); + assert.equal(pointer.activeSnapshot().entity.id, '400'); + let parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts; + assert.deepEqual(JSON.parse(parts[0].text), { + type: 'pointer_context', + at: 'text', + pointing: 'aircraft', + }); + assert.equal(parts.at(-1).text, 'what is this?'); + assert.equal( + h.sockets[0].sent.filter((value) => value.clientContent).length, + 1, + ); + assert.equal( + h.events.filter((event) => event.type === 'pointer').at(-1).pointer, + null, + ); + h.c.runAction = async () => { + assert.equal(pointer.activeSnapshot().entity.id, '400'); + assert.equal(h.c.announcePointer(), true); + assert.equal(h.c.announcePointer(), false); + return { ok: true }; + }; + h.sockets[0].message({ + toolCall: { + functionCalls: [ + { + id: 'implicit-pointer', + name: 'get_entity_context', + args: { scope: 'pointer' }, + }, + ], + }, + }); + await h.c.toolQueue; + assert.equal( + h.events.filter((event) => event.type === 'pointer').at(-1).pointer.label, + 'Aircraft 400', + ); + h.sockets[0].message({ serverContent: { turnComplete: true } }); + advance(5001); + h.c.sendTextCommand('what is here?'); + assert.equal(pointer.activeSnapshot(), null); + parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts; + assert.deepEqual(JSON.parse(parts[0].text), { + type: 'pointer_context', + at: 'text', + target: 'none', + }); + assert.equal(parts.at(-1).text, 'what is here?'); + h.c.stop(); + pointer.destroy(); +}); + +test('B48 pointer: automatic speech snapshots once per onset and late input transcript keeps it', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start(); + pointer.moveTo(450, 200); + const ws = h.sockets[0]; + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + const turn = h.c.turn; + assert.equal(pointer.activeSnapshot().entity.id, '450'); + assert.deepEqual(JSON.parse(ws.sent.at(-1).realtimeInput.text), { + type: 'pointer_context', + at: 'speech_start', + pointing: 'aircraft', + }); + pointer.moveTo(750, 200); + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ + serverContent: { inputTranscription: { text: 'what is this?' } }, + }); + assert.equal( + h.c.turn, + turn, + 'duplicate onset does not revoke the current turn', + ); + assert.equal(pointer.activeSnapshot().entity.id, '450'); + assert.equal( + ws.sent.some((value) => value.clientContent), + false, + ); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + ws.message({ serverContent: { turnComplete: true } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + assert.equal(pointer.activeSnapshot().entity.id, '750'); + h.c.stop(); + pointer.destroy(); +}); + +for (const route of ['push-to-talk', 'user-text']) { + for (const idle of ['first', 'completed']) { + test(`B48 fence: ${route} permits the ${idle} idle reply without an interruption marker`, async () => { + const h = harness(); + await h.c.start({ pushToTalk: route === 'push-to-talk' }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + if (idle === 'completed') { + ws.message({ + serverContent: { ...replyAudio('previous'), turnComplete: true }, + }); + audio.drain(); + } + if (route === 'user-text') h.c.sendTextCommand('first live request'); + else { + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + h.c.input.setMicrophoneEnabled(false); + ws.message({ + serverContent: { inputTranscription: { text: 'what is in view' } }, + }); + } + ws.message({ + toolCall: { + functionCalls: [ + { id: 'valid-first-call', name: 'get_entity_context', args: {} }, + ], + }, + }); + await h.c.toolQueue; + ws.message({ + serverContent: { ...replyAudio('legitimate'), turnComplete: true }, + }); + assert.equal(h.calls.length, 1); + assert.equal(audio.played.at(-1).data, 'legitimate'); + h.c.stop(); + }); + } +} + +test('B48 fence: an empty PTT hold creates no retired boundary for the next command', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + h.c.input.setMicrophoneEnabled(false); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + h.captures[0].callbacks.onAudio( + { + data: 'AAAA', + mimeType: 'audio/pcm;rate=16000', + }, + h.captures[0].metadata, + ); + h.c.input.setMicrophoneEnabled(false); + h.sockets[0].message({ + serverContent: { ...replyAudio('first-real'), turnComplete: true }, + }); + assert.equal(h.captures[0].played.at(-1).data, 'first-real'); + h.c.stop(); +}); + +test('B48 fence: a coalesced interrupted boundary keeps successor speech and its pointer', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + pointer.moveTo(600, 200); + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + const turn = h.c.turn; + ws.message({ + voiceActivity: { type: 'ACTIVITY_START' }, + serverContent: { + ...replyAudio('boundary-old'), + interrupted: true, + turnComplete: true, + inputTranscription: { text: 'new pointed question' }, + outputTranscription: { text: 'old answer' }, + }, + }); + assert.equal(h.c.turn, turn); + assert.equal(pointer.activeSnapshot().entity.id, '600'); + assert.equal(h.captures[0].played.length, 1); + assert.equal( + h.events.some((event) => event.type === 'completion'), + false, + ); + assert.equal( + h.events.some( + (event) => event.role === 'user' && event.text === 'new pointed question', + ), + true, + ); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + ws.message({ + serverContent: { ...replyAudio('successor'), turnComplete: true }, + }); + assert.equal(h.captures[0].played.at(-1).data, 'successor'); + h.c.stop(); + pointer.destroy(); +}); + +for (const route of ['push-to-talk', 'user-text']) { + test(`B48 queue: rapid ${route} keeps only the latest pointer/input and serializes reply boundaries`, async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + ws.message({ serverContent: replyAudio('old') }); + const submit = (x) => { + pointer.moveTo(x, 200); + if (route === 'user-text') return h.c.sendTextCommand(`request ${x}`); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.beginPointerTurn('keydown'); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { + data: String(x), + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + }; + submit(300); // This start/command must reach the provider to cause interruption. + const sentFirst = ws.sent.length; + submit(500); + submit(700); // Newer intent replaces the still-unsent second turn. + assert.equal( + ws.sent.length, + sentFirst, + 'later input waits without opening another server generation', + ); + assert.equal(pointer.activeSnapshot().entity.id, '700'); + const latestTurn = h.c.turn; + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + const flushed = ws.sent.slice(sentFirst); + if (route === 'user-text') { + assert.equal(flushed.length, 1); + assert.equal( + flushed[0].clientContent.turns[0].parts.at(-1).text, + 'request 700', + ); + } else { + assert.deepEqual( + flushed.map((value) => + value.clientContent ? ['context'] : Object.keys(value.realtimeInput), + ), + [['context'], ['activityStart'], ['audio'], ['activityEnd']], + ); + assert.equal( + JSON.parse(flushed[0].clientContent.turns[0].parts[0].text).target + .label, + 'Aircraft 700', + ); + assert.equal(flushed[0].clientContent.turnComplete, false); + assert.equal(flushed[2].realtimeInput.audio.data, '700'); + } + ws.message({ + serverContent: { + ...replyAudio('first-successor-retired'), + outputTranscription: { text: 'stale second answer' }, + }, + }); + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'queued-old-call', + name: 'fly_to_location', + args: { query: 'stale' }, + }, + ], + }, + }); + assert.equal(audio.played.length, 1); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.turn, latestTurn); + ws.message({ + serverContent: { inputTranscription: { text: 'what is this' } }, + }); + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'latest-call', + name: 'get_entity_context', + args: { scope: 'pointer' }, + }, + ], + }, + }); + if (route === 'push-to-talk') { + await tick(); + h.advance(5_000); + } + await h.c.toolQueue; + assert.equal(h.calls.length, 1); + assert.equal(pointer.activeSnapshot().entity.id, '700'); + ws.message({ + serverContent: { ...replyAudio('latest'), turnComplete: true }, + }); + assert.equal(audio.played.at(-1).data, 'latest'); + h.c.stop(); + pointer.destroy(); + }); +} + +test('B48 queue: wait timeout fails closed and Stop releases queued input without replay', async () => { + for (const exit of ['timeout', 'stop']) { + const h = harness({ inputBoundaryTimeoutMs: 10 }); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('sent successor'); + h.c.sendTextCommand('queued latest'); + const sent = ws.sent.length; + if (exit === 'stop') h.c.stop(); + else h.advance(25); + assert.equal(ws.closed, true); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'error'); + assert.equal(h.c.deferredInput, null); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal( + ws.sent.length, + sent, + 'released queue cannot replay on a late boundary', + ); + h.c.stop(); + } +}); + +test('B48 queue: 256 KiB input bound fails closed without flushing partial queued speech', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + h.captures[0].callbacks.onAudio( + { + data: 'first', + mimeType: 'audio/pcm;rate=16000', + }, + h.captures[0].metadata, + ); + h.c.input.setMicrophoneEnabled(false); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + const sent = ws.sent.length; + h.captures[0].callbacks.onAudio( + { + data: 'A'.repeat(256 * 1024), + mimeType: 'audio/pcm;rate=16000', + }, + h.captures[0].metadata, + ); + assert.equal(h.c.status, 'error'); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(ws.closed, true); + assert.equal(h.c.deferredInput, null); + assert.equal(ws.sent.length, sent); + h.c.stop(); +}); + +test('B48 fence: duplicate automatic activity end cannot retire an idle next command', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + ws.message({ + serverContent: { ...replyAudio('finished'), turnComplete: true }, + }); + h.captures[0].drain(); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + h.c.sendTextCommand('new idle turn'); + ws.message({ + serverContent: { ...replyAudio('new idle answer'), turnComplete: true }, + }); + assert.equal(h.captures[0].played.at(-1).data, 'new idle answer'); + h.c.stop(); +}); + +test('B48 queue: a boundary before deferred PTT worklet start still fences the submitted successor', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('first successor'); + h.c.input.bargeIn(); // Audio-start callback is deliberately delayed. + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + ws.message({ serverContent: replyAudio('first-successor-stale') }); + assert.equal(h.captures[0].played.length, 1); + h.c.input.setMicrophoneEnabled(true); + h.captures[0].callbacks.onAudio( + { + data: 'latest', + mimeType: 'audio/pcm;rate=16000', + }, + h.captures[0].metadata, + ); + h.c.input.setMicrophoneEnabled(false); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + ws.message({ + serverContent: { ...replyAudio('latest'), turnComplete: true }, + }); + assert.equal(h.captures[0].played.at(-1).data, 'latest'); + h.c.stop(); +}); + +for (const latest of ['push-to-talk', 'user-text']) { + test(`B48 queue: ${latest} can replace the other unsent input mode without stale pointer context`, async () => { + const { pointer, advance } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + ws.message({ serverContent: replyAudio('old') }); + pointer.moveTo(300, 200); + h.c.sendTextCommand('first successor'); + const speak = (x) => { + pointer.moveTo(x, 200); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.beginPointerTurn('keydown'); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { + data: String(x), + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + }; + if (latest === 'push-to-talk') { + pointer.moveTo(500, 200); + h.c.sendTextCommand('discard queued text'); + speak(700); + } else { + speak(500); + advance(5001); + h.c.sendTextCommand('latest text without pointer'); + } + const sent = ws.sent.length; + ws.message({ serverContent: { turnComplete: true } }); // Natural completion races the claim. + const frames = ws.sent.slice(sent); + if (latest === 'push-to-talk') { + assert.equal( + frames.some((message) => message.clientContent?.turnComplete === true), + false, + ); + assert.equal(frames[0].clientContent.turnComplete, false); + assert.deepEqual(frames[1], { realtimeInput: { activityStart: {} } }); + assert.equal( + JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label, + 'Aircraft 700', + ); + } else { + assert.equal(frames.length, 1); + assert.deepEqual( + JSON.parse(frames[0].clientContent.turns[0].parts[0].text), + { type: 'pointer_context', at: 'text', target: 'none' }, + ); + assert.equal(pointer.activeSnapshot(), null); + } + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + ws.message({ + serverContent: { ...replyAudio('latest'), turnComplete: true }, + }); + assert.equal(audio.played.at(-1).data, 'latest'); + h.c.stop(); + pointer.destroy(); + }); +} + +test('R51: typed override closes forwarded PTT before its queued text and suppresses the old flush', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + ws.message({ serverContent: replyAudio('retired A') }); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { + data: 'spoken B', + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + const before = ws.sent.length; + h.c.sendTextCommand('latest typed C'); + assert.equal( + ws.sent.at(-1).realtimeInput?.activityEnd !== undefined, + true, + 'the forwarded speech must end when typed intent supersedes it', + ); + assert.equal(ws.sent.length, before + 1); + audio.callbacks.onAudio( + { + data: 'retired buffered B', + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + assert.equal( + ws.sent.length, + before + 1, + 'late old PCM/end cannot extend or duplicate closed activity', + ); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + const flushed = ws.sent.slice(before); + assert.equal(flushed.length, 2); + assert.equal(flushed[0].realtimeInput?.activityEnd !== undefined, true); + assert.equal( + flushed[1].clientContent.turns[0].parts.at(-1).text, + 'latest typed C', + ); + h.c.stop(); +}); + +test('R51: a delayed old worklet start cannot reclaim a typed turn; a fresh hold still works', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + h.c.sendTextCommand('current text'); + const before = ws.sent.length; + // This is the prior gesture's callback, delayed until after typed intent. + audio.callbacks.onAudioStart(audio.metadata); + audio.callbacks.onAudio( + { + data: 'old gesture', + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + audio.callbacks.onAudioEnd(audio.metadata); + assert.equal(ws.sent.length, before); + ws.message({ + serverContent: { ...replyAudio('text completed'), turnComplete: true }, + }); + audio.drain(); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { + data: 'fresh hold', + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + h.c.input.setMicrophoneEnabled(false); + const frames = ws.sent.slice(before); + assert.equal(frames[0].realtimeInput.activityStart !== undefined, true); + assert.equal(frames[1].realtimeInput.audio.data, 'fresh hold'); + assert.equal(frames[2].realtimeInput.activityEnd !== undefined, true); + h.c.stop(); +}); + +test('R51: replaced unsent PTT contributes no unmatched end to the forwarded stream', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + ws.message({ serverContent: replyAudio('retired A') }); + h.c.sendTextCommand('submitted B'); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { + data: 'unsent C', + mimeType: 'audio/pcm;rate=16000', + }, + audio.metadata, + ); + h.c.sendTextCommand('newest typed D'); + const before = ws.sent.length; + h.c.input.setMicrophoneEnabled(false); + assert.equal(ws.sent.length, before); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + const flushed = ws.sent.slice(before); + assert.equal(flushed.length, 1); + assert.equal( + flushed[0].clientContent.turns[0].parts.at(-1).text, + 'newest typed D', + ); + h.c.stop(); +}); + +test('R51-2: FIFO old capture messages stay retired after typed replacement and a fresh claim', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + const audio = h.captures[0]; + // Delay only the worklet message port; gate/gesture callbacks still run now. + const port = []; + const callbacks = audio.callbacks; + const enable = audio.setMicrophoneEnabled.bind(audio); + audio.setMicrophoneEnabled = (value, metadata) => { + if (value && !audio.enabled) + port.push(() => callbacks.onAudioStart(metadata)); + if (!value && audio.enabled) { + const ending = audio.metadata; + port.push(() => callbacks.onAudioEnd(ending)); + } + audio.enabled = value; + if (value) audio.metadata = metadata; + }; + ws.message({ serverContent: replyAudio('outstanding A') }); + const claim = (x) => { + pointer.moveTo(x, 200); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.beginPointerTurn('keydown'); + h.c.input.setMicrophoneEnabled(true); + return audio.metadata; + }; + const b = claim(300); + port.push(() => + callbacks.onAudio( + { data: 'retired B', mimeType: 'audio/pcm;rate=16000' }, + b, + ), + ); + h.c.sendTextCommand('typed C'); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + const d = claim(700); + port.push(() => + callbacks.onAudio( + { data: 'current D', mimeType: 'audio/pcm;rate=16000' }, + d, + ), + ); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + const before = ws.sent.length; + // Allowed port FIFO: start B / PCM B / end B / start D / PCM D / end D. + for (const deliver of port) deliver(); + assert.equal(ws.sent.length, before); + const queued = h.c.deferredInput.messages; + assert.deepEqual( + queued.map((message) => + message.clientContent + ? 'context' + : message.realtimeInput.audio?.data || + (message.realtimeInput.activityStart ? 'start' : 'end'), + ), + ['context', 'start', 'current D', 'end'], + ); + assert.equal( + JSON.parse(queued[0].clientContent.turns[0].parts[0].text).target.label, + 'Aircraft 700', + ); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.deepEqual(ws.sent.slice(before), queued); + assert.equal(JSON.stringify(ws.sent).includes('captureId'), false); + audio.setMicrophoneEnabled = enable; + h.c.stop(); +}); + +test('R51-2: a normal release keeps its delayed final PCM and end, without relabelling duplicate gates or tool aborts', async () => { + const h = harness(); + h.c.input.pushToTalkKeyHeld = true; + await h.c.start({ pushToTalk: true }); + const audio = h.captures[0]; + const metadata = audio.metadata; + const ws = h.sockets[0]; + h.c.input.setMicrophoneEnabled(true); + assert.equal(audio.metadata.captureId, metadata.captureId); + h.c.abortTools(); // Radio cleanup is not a new physical capture. + const end = audio.callbacks.onAudioEnd; + let released; + audio.callbacks.onAudioEnd = (identity) => { + released = identity; + }; + h.c.input.setMicrophoneEnabled(false); + assert.equal(released.captureId, metadata.captureId); + const before = ws.sent.length; + audio.callbacks.onAudio( + { data: 'released tail', mimeType: 'audio/pcm;rate=16000' }, + metadata, + ); + end(released); + assert.deepEqual(ws.sent.slice(before), [ + { + realtimeInput: { + audio: { data: 'released tail', mimeType: 'audio/pcm;rate=16000' }, + }, + }, + { realtimeInput: { activityEnd: {} } }, + ]); + audio.callbacks.onAudioEnd = end; + h.c.stop(); +}); + +test('R51-2: a forwarded old capture closes once before fresh FIFO start and cannot close the new activity', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const audio = h.captures[0]; + const ws = h.sockets[0]; + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + const b = audio.metadata; + audio.callbacks.onAudio( + { data: 'forwarded B', mimeType: 'audio/pcm;rate=16000' }, + b, + ); + const start = audio.callbacks.onAudioStart; + const end = audio.callbacks.onAudioEnd; + const port = []; + audio.callbacks.onAudioStart = (identity) => port.push(() => start(identity)); + audio.callbacks.onAudioEnd = (identity) => port.push(() => end(identity)); + h.c.input.setMicrophoneEnabled(false); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + const d = audio.metadata; + const before = ws.sent.length; + for (const deliver of port) deliver(); // end B, start D in port FIFO. + assert.deepEqual(ws.sent.slice(before), [ + { realtimeInput: { activityEnd: {} } }, + { realtimeInput: { activityStart: {} } }, + ]); + assert.equal(h.c.forwardedCaptureId, d.captureId); + end(b); // A duplicate old end cannot close D. + assert.equal(ws.sent.length, before + 2); + assert.equal(h.c.forwardedActivityOpen, true); + audio.callbacks.onAudio( + { data: 'current D', mimeType: 'audio/pcm;rate=16000' }, + d, + ); + audio.callbacks.onAudioStart = start; + audio.callbacks.onAudioEnd = end; + h.c.input.setMicrophoneEnabled(false); + assert.equal(ws.sent.at(-1).realtimeInput.activityEnd !== undefined, true); + assert.equal(h.c.forwardedActivityOpen, false); + h.c.stop(); +}); + +test('R51-2: absent metadata fails closed and Stop/error restart never reuse capture identity', async () => { + const h = harness(); + let previous = 0; + let old; + let oldMetadata; + for (const terminate of ['stop', 'error', 'stop']) { + h.c.input.pushToTalkKeyHeld = true; + await h.c.start({ pushToTalk: true }); + const audio = h.captures.at(-1); + const ws = h.sockets.at(-1); + assert.ok(audio.metadata.captureId > previous); + previous = audio.metadata.captureId; + const before = ws.sent.length; + for (const captureId of [undefined, null, 0, -1, 1.5, '1', Infinity]) { + const metadata = captureId === undefined ? undefined : { captureId }; + audio.callbacks.onAudioStart(metadata); + audio.callbacks.onAudio( + { data: 'must reject', mimeType: 'audio/pcm;rate=16000' }, + metadata, + ); + audio.callbacks.onAudioEnd(metadata); + } + if (old) { + old.onAudioStart(oldMetadata); + old.onAudio( + { data: 'old epoch', mimeType: 'audio/pcm;rate=16000' }, + oldMetadata, + ); + old.onAudioEnd(oldMetadata); + } + assert.equal(ws.sent.length, before); + assert.equal(h.c.forwardedActivityOpen, true); + old = audio.callbacks; + oldMetadata = audio.metadata; + if (terminate === 'error') h.c.fail(new Error('capture failed')); + else h.c.stop(); + assert.equal(h.c.captureOwner, null); + assert.equal(h.c.captureEnabled, false); + assert.equal(h.c.forwardedCaptureId, null); + } +}); + +for (const queued of ['empty', 'text']) { + test(`R52: ${queued} flush keeps the original retirement wait bound`, async () => { + const h = harness({ inputBoundaryTimeoutMs: 40 }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('first submitted successor'); + if (queued === 'empty') h.c.input.bargeIn(); + else h.c.sendTextCommand('newest queued text'); + // A physical empty hold no longer submits/retires a successor. The first + // submitted text still owns the reply when the old native boundary arrives. + if (queued === 'empty') { + assert.equal(h.c.deferredInput, null); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.retiredReplyPending, false); + assert.equal(h.c.replyPending, true); + h.advance(60); + assert.equal(h.c.status, 'listening'); + assert.equal(ws.closed, undefined); + h.c.stop(); + return; + } + const deadline = h.c.deferredInput.deadline; + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.deferredInput, null); + assert.equal(h.c.retiredReplyPending, true); + h.advance(deadline + 20); + assert.equal( + h.c.status, + 'error', + 'missing genuine boundary cannot leave voice waiting indefinitely', + ); + assert.equal(ws.closed, true); + assert.equal(h.tracks[0].readyState, 'ended'); + h.c.stop(); + }); +} + +test('R52: replacement cannot extend a flushed retirement deadline and Stop cancels it', async () => { + for (const exit of ['timeout', 'stop', 'boundary']) { + const h = harness({ inputBoundaryTimeoutMs: 60 }); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('first successor'); + h.c.sendTextCommand('queued successor'); + const deadline = h.c.deferredInput.deadline; + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + h.advance(15); + h.c.sendTextCommand('replacement while successor retired'); + assert.equal(h.c.deferredInput.deadline, deadline); + if (exit === 'stop') h.c.stop(); + if (exit === 'boundary') { + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + ws.message({ + serverContent: { ...replyAudio('latest'), turnComplete: true }, + }); + h.captures[0].drain(); + } + h.advance(deadline + 20); + assert.equal( + h.c.status, + exit === 'timeout' ? 'error' : exit === 'stop' ? 'idle' : 'listening', + ); + assert.equal(Boolean(ws.closed), exit !== 'boundary'); + h.c.stop(); + } +}); + +test('PTT pointer context is quiet, ordered once, and only PCM creates a pending reply', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + pointer.moveTo(300, 200); + h.c.input.holdPointer(); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.spaceKeyHeld = true; + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0]; + const [context, start] = ws.sent.slice(1); + assert.deepEqual(Object.keys(context), ['clientContent']); + assert.equal(context.clientContent.turnComplete, false); + assert.equal(context.clientContent.turns.length, 1); + assert.equal(context.clientContent.turns[0].role, 'user'); + assert.equal( + JSON.parse(context.clientContent.turns[0].parts[0].text).target.label, + 'Aircraft 300', + ); + assert.deepEqual(start, { realtimeInput: { activityStart: {} } }); + assert.equal(h.c.replyPending, false); + assert.equal(audio.played.length, 0); + assert.equal(h.calls.length, 0); + audio.callbacks.onAudioStart(audio.metadata); + assert.equal( + ws.sent.length, + 3, + 'duplicate start does not resend quiet context', + ); + audio.callbacks.onAudio( + { data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + assert.equal(h.c.replyPending, true); + assert.equal( + ws.sent.some((message) => message.realtimeInput?.text), + false, + ); + h.c.stop(); + pointer.destroy(); +}); + +test('PTT pointer context rejected by the channel fails closed without opening activity', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + pointer.moveTo(300, 200); + h.c.input.holdPointer(); + const ws = h.sockets[0], + before = ws.sent.length; + h.c.channel.readyState = 'connecting'; + // The adapter emits one admitted start, not a retryable channel-open event. + h.c.input.setMicrophoneEnabled(true); + assert.equal(ws.sent.length, before); + assert.equal(h.c.forwardedActivityOpen, false); + assert.equal(h.c.pointerSentKey, null); + assert.equal(h.c.captureOwner, null); + assert.equal(h.c.userTurnPending, false); + assert.equal(h.c.status, 'error'); + assert.equal(ws.closed, true); + assert.equal(h.tracks[0].readyState, 'ended'); + h.c.stop(); + pointer.destroy(); +}); + +test('quiet pointer context rejects stale or absent capture metadata without touching the new pointer', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + pointer.moveTo(300, 200); + h.c.input.holdPointer(); + h.c.input.setMicrophoneEnabled(true); + const old = h.captures[0].metadata, + context = h.c.captureOwner.context; + h.c.input.setMicrophoneEnabled(false); + pointer.moveTo(700, 200); + h.c.input.holdPointer(); + h.c.input.setMicrophoneEnabled(true); + const before = h.sockets[0].sent.length, + key = h.c.pointerSentKey; + assert.equal(h.c.sendPointerContext(context, old), false); + assert.equal(h.c.sendPointerContext(context), false); + assert.equal(h.c.sendPointerContext(context, { captureId: 0 }), false); + assert.equal(h.sockets[0].sent.length, before); + assert.equal(h.c.pointerSentKey, key); + assert.equal(pointer.activeSnapshot().entity.id, '700'); + h.c.stop(); + pointer.destroy(); +}); + +test('replacing a deferred quiet PTT context preserves its deadline and only flushes the latest owner', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0]; + ws.message({ serverContent: replyAudio('old A') }); + const claim = (x, pcm) => { + pointer.moveTo(x, 200); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.beginPointerTurn('keydown'); + h.c.input.setMicrophoneEnabled(true); + if (pcm) + audio.callbacks.onAudio( + { data: pcm, mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + h.c.input.releasePointerHold(); + h.c.input.setMicrophoneEnabled(false); + }; + claim(300, 'submitted B'); + claim(500); + const deadline = h.c.deferredInput.deadline; + const sent = ws.sent.length; + claim(700); + assert.equal(h.c.deferredInput.deadline, deadline); + assert.equal(h.c.deferredInput.kind, 'push-to-talk'); + assert.equal(h.c.deferredInput.captureId, h.c.captureOwner.captureId); + assert.equal(h.c.deferredInput.messages.length, 3); + assert.equal( + JSON.parse( + h.c.deferredInput.messages[0].clientContent.turns[0].parts[0].text, + ).target.label, + 'Aircraft 700', + ); + assert.equal( + JSON.stringify(h.c.deferredInput.messages).includes('Aircraft 500'), + false, + ); + assert.equal(ws.sent.length, sent); + // The genuine retired boundary may coalesce content; it must discard it + // while releasing only the latest quiet context and its activity markers. + ws.message({ + serverContent: { + ...replyAudio('retired coalesced A'), + interrupted: true, + turnComplete: true, + }, + }); + const flushed = ws.sent.slice(sent); + assert.equal(flushed.length, 3); + assert.equal(flushed[0].clientContent.turnComplete, false); + assert.deepEqual(flushed[1], { realtimeInput: { activityStart: {} } }); + assert.deepEqual(flushed[2], { realtimeInput: { activityEnd: {} } }); + assert.equal(audio.played.at(-1).data, 'old A'); + assert.equal( + h.c.replyPending, + false, + 'flushed context without PCM asks for no reply', + ); + assert.equal(h.c.retiredBoundaryWait.deadline, deadline); + assert.equal(h.c.retiredReplyPending, true); + assert.equal(h.c.pointerSentKey, h.c.captureOwner.context.key); + h.c.stop(); + pointer.destroy(); +}); + +test('projected PCM bytes and optional viewport frames drop while reliable tool results wait', async () => { + const h = harness({ + captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=', + }); + await h.c.start(); + establishToolInput(h); + const ws = h.sockets[0], + audio = h.captures[0], + before = ws.sent.length; + ws.bufferedAmount = 262140; + audio.callbacks.onAudio( + { data: 'AA==', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + ws.message({ + toolCall: { + functionCalls: [ + { id: 'image-pressure', name: 'get_entity_context', args: {} }, + ], + }, + }); + h.c.runAction = async () => ({ + ok: true, + action: 'get_entity_context', + scene: { basemap: { viewScale: 'local' } }, + }); + await h.c.toolQueue; + assert.equal(h.c.status, 'listening'); + assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1); + assert.equal(h.c.getDiagnostics().droppedImages, 1); + assert.equal(ws.sent.length, before); + assert.equal(h.c.controlQueue.length, 1); + ws.bufferedAmount = 0; + h.advance(25); + assert.equal( + ws.sent.at(-1).toolResponse.functionResponses[0].id, + 'image-pressure', + ); + assert.match( + ws.sent.at(-1).toolResponse.functionResponses[0].response.responseGuidance, + /briefly|request|read|view/i, + ); + assert.equal(h.c.controlQueue.length, 0); + h.c.stop(); +}); + +test('congested PTT markers remain ordered and forwarding flags commit only at wire delivery', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0], + before = ws.sent.length; + ws.bufferedAmount = 300000; + h.c.input.setMicrophoneEnabled(true); + const capture = audio.metadata; + audio.callbacks.onAudio( + { data: 'discarded PCM', mimeType: 'audio/pcm;rate=16000' }, + capture, + ); + h.c.input.setMicrophoneEnabled(false); + assert.equal(h.c.forwardedActivityOpen, false); + assert.equal(h.c.controlQueue.length, 2); + assert.equal(ws.sent.length, before); + assert.equal(h.c.replyPending, false); + ws.bufferedAmount = 0; + h.advance(25); + assert.deepEqual(ws.sent.slice(before), [ + { realtimeInput: { activityStart: {} } }, + { realtimeInput: { activityEnd: {} } }, + ]); + assert.equal(h.c.forwardedActivityOpen, false); + h.c.stop(); +}); + +test('saturation then typed override and a fresh hold cannot flush retired pointer or capture markers', async () => { + const { pointer } = pointerHarness(); + const h = harness({ pointer }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0], + audio = h.captures[0], + before = ws.sent.length; + ws.bufferedAmount = 300000; + pointer.moveTo(300, 200); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + const old = audio.metadata; + h.c.input.setMicrophoneEnabled(false); + h.c.sendTextCommand('unsent typed override'); + assert.equal( + h.c.replyPending, + false, + 'queued text creates no phantom retired reply', + ); + pointer.moveTo(700, 200); + h.c.input.holdPointer(); + h.c.input.bargeIn(); + h.c.input.setMicrophoneEnabled(true); + h.c.input.setMicrophoneEnabled(false); + audio.callbacks.onAudioStart(old); + audio.callbacks.onAudioEnd(old); + assert.equal( + h.c.pointerSentKey, + null, + 'queued context is not consumed before wire delivery', + ); + ws.bufferedAmount = 0; + h.advance(25); + const frames = ws.sent.slice(before); + assert.equal(JSON.stringify(frames).includes('Aircraft 300'), false); + assert.equal(JSON.stringify(frames).includes('unsent typed override'), false); + assert.equal( + JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label, + 'Aircraft 700', + ); + assert.deepEqual(frames.slice(1), [ + { realtimeInput: { activityStart: {} } }, + { realtimeInput: { activityEnd: {} } }, + ]); + assert.equal(h.c.forwardedActivityOpen, false); + h.c.stop(); + pointer.destroy(); +}); + +for (const cancellation of ['new-turn', 'server-call']) { + test(`a queued tool response is revoked by ${cancellation} before pressure clears`, async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0], + before = ws.sent.length; + ws.bufferedAmount = 300000; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'queued-old', name: 'get_current_view_state', args: {} }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(h.c.controlQueue.length, 1); + if (cancellation === 'new-turn') h.c.sendTextCommand('new owned input'); + else ws.message({ toolCallCancellation: { ids: ['queued-old'] } }); + ws.bufferedAmount = 0; + h.advance(25); + assert.equal( + ws.sent.slice(before).some((frame) => frame.toolResponse), + false, + ); + h.c.stop(); + }); +} + +test('reliable pressure has an absolute deadline and Stop releases its poll without successor effects', async () => { + for (const mode of ['deadline', 'stop']) { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + ws.bufferedAmount = 300000; + h.c.send({ realtimeInput: { text: 'reliable authored context' } }); + h.advance(4975); + assert.equal(h.c.status, 'listening'); + if (mode === 'stop') { + h.c.stop(); + await h.c.start(); + } + h.advance(25); + assert.equal(h.c.status, mode === 'deadline' ? 'error' : 'listening'); + assert.equal(h.c.controlQueue.length, 0); + assert.equal(h.timers.size, 0); + h.c.stop(); + } +}); + +test('reliable queue is byte and message bounded without partially sending oversized frames', async () => { + for (const kind of ['bytes', 'messages']) { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0], + before = ws.sent.length; + ws.bufferedAmount = 300000; + if (kind === 'bytes') { + assert.throws( + () => h.c.send({ realtimeInput: { text: 'x'.repeat(1024 * 1024) } }), + /queue limit/, + ); + } else { + for (let i = 0; i < 64; i++) + assert.equal(h.c.send({ realtimeInput: { text: String(i) } }), true); + assert.throws( + () => h.c.send({ realtimeInput: { text: 'overflow' } }), + /queue limit/, + ); + } + assert.equal(ws.sent.length, before); + h.c.stop(); + assert.equal(h.timers.size, 0); + } +}); + +test('goAway permits the current tool/reply/audio drain and denies successor input', async () => { + const tool = deferred(); + const h = harness({ runAction: () => tool.promise }); + await h.c.start(); + const ws = h.sockets[0], + audio = h.captures[0]; + h.c.sendTextCommand('read the view'); + ws.message({ + toolCall: { + functionCalls: [{ id: 'current', name: 'get_entity_context', args: {} }], + }, + }); + await tick(); + ws.message({ goAway: { timeLeft: '5s' } }); + assert.equal(h.c.goAway.deadline, 5000); + assert.equal(h.tracks[0].enabled, false); + assert.equal(h.c.sendTextCommand('successor is disallowed'), false); + assert.equal(h.c.bargeIn(), false); + h.c.input.setMicrophoneEnabled(true); + assert.equal(h.tracks[0].enabled, false); + tool.resolve({ ok: true, action: 'get_entity_context' }); + await h.c.toolQueue; + assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'current'); + ws.message({ + serverContent: { + ...replyAudio('accepted reply'), + outputTranscription: { text: 'Current result' }, + turnComplete: true, + }, + }); + assert.equal(h.c.isActive(), true); + assert.equal(audio.pending, true); + audio.drain(); + assert.equal(h.c.status, 'error'); + assert.equal(ws.closed, true); + assert.equal(h.timers.size, 0); + assert.equal( + h.events.findLast((event) => event.type === 'completion').audioDrained, + true, + ); + h.c.stop(); +}); + +test('goAway chooses the earliest bounded deadline and Stop cancels it for a fresh epoch', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('playing') }); + ws.message({ goAway: { timeLeft: '1.250s' } }); + h.advance(250); + ws.message({ goAway: { timeLeft: '500s' } }); + assert.equal(h.c.goAway.deadline, 1250); + ws.message({ goAway: { timeLeft: '0.5s' } }); + assert.equal(h.c.goAway.deadline, 750); + h.c.stop(); + await h.c.start(); + h.advance(1000); + assert.equal(h.c.status, 'listening'); + assert.equal(h.sockets.length, 2); + assert.equal(h.c.goAway, null); + h.c.stop(); +}); + +for (const duration of ['bad', '-1s', 'Infinitys', '10000000s', null]) { + test(`malformed goAway timeLeft ${duration} closes safely without an unbounded timer`, async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].message({ serverContent: replyAudio('playing') }); + h.sockets[0].message({ goAway: { timeLeft: duration } }); + assert.equal(h.c.status, 'error'); + assert.equal(h.timers.size, 0); + }); +} + +test('close diagnostics preserve bounded code/category but never provider reason or credential text', async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].onclose({ + code: 1008, + reason: 'auth_tokens/private-secret user-private-text', + }); + assert.deepEqual(h.c.getDiagnostics().close, { + code: 1008, + category: 'policy', + }); + assert.doesNotMatch( + JSON.stringify(h.c.getDiagnostics()), + /private-secret|user-private-text/, + ); + await h.c.start(); + h.sockets[1].onclose({ code: Infinity, reason: 'private' }); + assert.deepEqual(h.c.getDiagnostics().close, { + code: null, + category: 'connection', + }); + h.c.stop(); +}); + +test('goAway never flushes an unsent successor at the retired response boundary', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('retired previous') }); + h.c.sendTextCommand('already forwarded successor'); + h.c.sendTextCommand('unsent newest successor'); + assert.ok(h.c.deferredInput); + const before = ws.sent.length; + ws.message({ goAway: { timeLeft: '5s' } }); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal( + ws.sent.slice(before).some((message) => message.clientContent), + false, + ); + assert.equal(h.c.status, 'error'); + assert.equal(h.c.deferredInput, null); + assert.equal(h.timers.size, 0); + h.c.stop(); +}); + +test('permission rejection opens no token/socket and initialized audio is released', async () => { + const h = harness({ + getUserMedia: async () => { + throw new DOMException('not allowed', 'NotAllowedError'); + }, + }); + await h.c.start(); + assert.equal(h.c.status, 'error'); + assert.equal(h.requests.length, 0); + assert.equal(h.sockets.length, 0); + assert.equal(h.captures[0].closed, true); + assert.match(h.c.lastError, /permission was denied/); +}); + +test('consented microphone stays muted through the network phase and its deadline releases tracks', async () => { + const pending = deferred(); + const h = harness({ startupTimeoutMs: 20, fetchImpl: () => pending.promise }); + const start = h.c.start(); + await tick(); + assert.equal(h.tracks[0].enabled, false); + await Promise.all([start, new Promise((resolve) => setTimeout(resolve, 40))]); + assert.equal(h.c.status, 'error'); + assert.equal(h.tracks[0].readyState, 'ended'); + assert.equal(h.captures[0].closed, true); + pending.resolve(Response.json(credential)); + await tick(); + assert.equal(h.sockets.length, 0); + h.c.stop(); +}); + +test('valid huge goAway duration is clamped to a bounded timer', async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].message({ serverContent: replyAudio('playing') }); + h.sockets[0].message({ goAway: { timeLeft: '999999s' } }); + assert.equal(h.c.goAway.deadline, 600000); + h.c.stop(); + assert.equal(h.timers.size, 0); +}); + +test('cancelling a pressure-queued response retains passive map context for the next owned text', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'Austin' }); + ws.bufferedAmount = 300000; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'old-map-envelope', name: 'get_current_view_state', args: {} }, + ], + }, + }); + await h.c.toolQueue; + assert.equal( + h.c.pendingMapEvents.length, + 1, + 'unsent data remains owned by the context queue', + ); + ws.message({ toolCallCancellation: { ids: ['old-map-envelope'] } }); + h.c.sendTextCommand('use the map context'); + ws.bufferedAmount = 0; + h.advance(25); + const frame = ws.sent.at(-1); + assert.equal( + JSON.parse(frame.clientContent.turns[0].parts[0].text).events[0].label, + 'Austin', + ); + assert.equal(h.c.pendingMapEvents.length, 0); + h.c.stop(); +}); + +test('an expired goAway deadline rejects late output before the timer callback can run', async () => { + let now = 0; + const h = harness({ now: () => now }); + await h.c.start(); + const ws = h.sockets[0]; + h.c.sendTextCommand('current request'); + ws.message({ serverContent: replyAudio('initial output') }); + ws.message({ goAway: { timeLeft: '1s' } }); + now = 1000; + ws.message({ + serverContent: { ...replyAudio('too late'), turnComplete: true }, + }); + assert.equal(h.c.status, 'error'); + assert.equal(h.captures[0].played.length, 1); + assert.equal( + h.events.some((event) => event.type === 'completion'), + false, + ); + assert.equal(h.timers.size, 0); +}); + +function guardedNativeHarness(overrides = {}) { + const mutations = []; + const observed = []; + const guarded = withToolCatalog( + async (name, args, options) => { + mutations.push({ name, args, options }); + return { ok: true, action: name }; + }, + async () => null, + ); + const h = harness({ + ...overrides, + runAction: async (name, args, options) => { + observed.push({ name, args, options }); + return guarded(name, args, options); + }, + }); + return { ...h, mutations, observed }; +} + +for (const stage of ['deferred', 'flushed']) { + test(`Sameh P3-1: ${stage} retirement deadline uses the injected clock and releases timers`, async () => { + const h = harness({ inputBoundaryTimeoutMs: 60 }); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('first successor'); + h.c.sendTextCommand('queued successor'); + assert.equal(h.c.deferredInput.deadline, 60); + h.advance(15); + h.c.sendTextCommand('replacement'); + assert.equal(h.c.deferredInput.deadline, 60); + if (stage === 'flushed') { + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + assert.equal(h.c.retiredBoundaryWait.deadline, 60); + } + h.advance(44); + assert.notEqual(h.c.status, 'error'); + h.advance(1); + assert.equal(h.c.status, 'error'); + assert.equal(ws.closed, true); + assert.equal(h.timers.size, 0); + } finally { + h.c.stop(); + } + }); +} + +for (const exit of ['stop', 'boundary']) { + test(`Sameh P3-1: ${exit} clears an injected retirement deadline before replacement`, async () => { + const h = harness({ inputBoundaryTimeoutMs: 60 }); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ serverContent: replyAudio('old') }); + h.c.sendTextCommand('first successor'); + h.c.sendTextCommand('queued successor'); + const deadline = h.c.deferredInput.deadline; + h.advance(15); + h.c.sendTextCommand('newest successor'); + assert.equal(h.c.deferredInput.deadline, deadline); + ws.message({ serverContent: { interrupted: true, turnComplete: true } }); + assert.equal(h.c.retiredBoundaryWait.deadline, deadline); + assert.equal( + [...h.timers.values()].some((timer) => timer.at === 60), + true, + 'retirement wait must belong to the injected timer registry', + ); + if (exit === 'stop') h.c.stop(); + else { + ws.message({ + serverContent: { interrupted: true, turnComplete: true }, + }); + ws.message({ serverContent: { turnComplete: true } }); + } + assert.equal( + [...h.timers.values()].some((timer) => timer.at === 60), + false, + ); + h.advance(100); + assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'listening'); + assert.equal(Boolean(ws.closed), exit === 'stop'); + } finally { + h.c.stop(); + } + }); +} + +for (const segments of [['use foo as my voice provider', 'and zoom out']]) { + test(`native advisory caption text ${JSON.stringify(segments)} releases tools`, async () => { + const h = guardedNativeHarness(); + try { + await h.c.start(); + const ws = h.sockets[0]; + ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); + for (const text of segments) + ws.message({ serverContent: { inputTranscription: { text } } }); + requestTool(ws, 'ordinary-port', 'zoom_to_globe', {}); + await h.c.toolQueue; + assert.equal(h.mutations.length, 1); + } finally { + h.c.stop(); + } + assert.equal(h.timers.size, 0); + }); +} + +const zoomCall = (id) => ({ + toolCall: { + functionCalls: [ + { id, name: 'adjust_camera_zoom', args: { direction: 'in' } }, + ], + }, +}); +const toolResponses = (ws, from = 0) => + ws.sent + .slice(from) + .flatMap((message) => message.toolResponse?.functionResponses || []); + +async function completedPttTurn() { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + h.captures[0].drain(); + await tick(); + return { h, ws }; +} + +test('push-to-talk refuses an unrequested tool call and every call in its reply', async () => { + const { h, ws } = await completedPttTurn(); + const sent = ws.sent.length; + ws.message(zoomCall('unrequested')); + ws.message(zoomCall('follow-up')); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + assert.deepEqual( + toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), + [ + ['unrequested', false], + ['follow-up', false], + ], + ); + ws.message({ serverContent: { turnComplete: true } }); + establishToolInput(h, 'Zoom in'); + // A refused ID stays spent; only the newly requested call runs. + ws.message(zoomCall('unrequested')); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + assert.equal(toolResponses(ws, sent).length, 3); + h.c.stop(); +}); + +test('push-to-talk refuses a call after unprompted speech that started the reply', async () => { + const { h, ws } = await completedPttTurn(); + const sent = ws.sent.length; + ws.message({ + serverContent: { outputTranscription: { text: 'Let me zoom in' } }, + }); + ws.message(zoomCall('after-speech')); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + assert.deepEqual( + toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), + [['after-speech', false]], + ); + h.c.stop(); +}); + +test('a refusal does not hold back the completion of the reply still playing', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + ws.message(zoomCall('unrequested')); + h.captures[0].drain(); + await tick(); + assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); + assert.deepEqual(h.calls, []); + h.c.stop(); +}); + +test('open microphone still runs a call Gemini makes without a local boundary', async () => { + const h = harness(); + await h.c.start(); + h.sockets[0].message(zoomCall('server-detected')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + h.c.stop(); +}); + +test('a late caption after a completed push-to-talk turn cannot retire the next request', async () => { + const { h, ws } = await completedPttTurn(); + ws.message({ serverContent: { outputTranscription: { text: ' done.' } } }); + establishToolInput(h, 'Zoom in'); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + h.advance(5_000); + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + assert.equal(h.c.lastError, null); + h.c.stop(); +}); + +test('an open-mic reply that starts with a caption is retired by newer typed input', async () => { + const h = harness(); + await h.c.start(); + const ws = h.sockets[0]; + establishToolInput(h); + ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); + h.captures[0].drain(); + await tick(); + ws.message({ serverContent: { outputTranscription: { text: 'Zooming' } } }); + establishToolInput(h, 'Replacement request'); + ws.message(zoomCall('earlier-reply')); + ws.message({ serverContent: { turnComplete: true } }); + await h.c.toolQueue; + assert.deepEqual(h.calls, []); + h.c.stop(); +}); + +test('a refused reply boundary that arrives after new push-to-talk input cannot close it', async () => { + const { h, ws } = await completedPttTurn(); + const audio = h.captures[0]; + ws.message(zoomCall('unrequested')); + h.c.input.pushToTalkKeyHeld = true; + h.c.input.setMicrophoneEnabled(true); + audio.callbacks.onAudio( + { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, + audio.metadata, + ); + h.c.input.pushToTalkKeyHeld = false; + h.c.input.setMicrophoneEnabled(false); + const completions = h.events.filter((e) => e.type === 'completion').length; + ws.message({ serverContent: { turnComplete: true } }); + assert.equal( + h.events.filter((e) => e.type === 'completion').length, + completions, + ); + ws.message(zoomCall('requested')); + await h.c.toolQueue; + assert.deepEqual( + h.calls.map((call) => call.name), + ['adjust_camera_zoom'], + ); + h.c.stop(); +}); + +test('push-to-talk admits a call made while an earlier requested tool is still running', async () => { + const running = deferred(); + const h = harness({ + runAction: async (name) => { + h.calls.push({ name }); + if (h.calls.length === 1) await running.promise; + return { ok: true, action: name }; + }, + }); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h, 'Zoom in'); + ws.message(zoomCall('first')); + await tick(); + ws.message({ serverContent: { turnComplete: true } }); + ws.message(zoomCall('second')); + running.resolve(); + await h.c.toolQueue; + assert.equal(h.calls.length, 2); + assert.equal( + toolResponses(ws).some(({ response }) => response.ok === false), + false, + ); + h.c.stop(); +}); + +test('a narration callback that submits a successor keeps the old packet tools from running', async () => { + const h = harness(); + await h.c.start({ pushToTalk: true }); + const ws = h.sockets[0]; + establishToolInput(h, 'Zoom in'); + const audible = h.c.narration.audible.bind(h.c.narration); + let submitted = false; + h.c.narration.audible = () => { + audible(); + if (submitted) return; + submitted = true; + assert.equal(h.c.sendTextCommand('New request'), true); + }; + ws.message({ + serverContent: replyAudio('old'), + toolCall: { + functionCalls: [ + { id: 'old', name: 'adjust_camera_zoom', args: { direction: 'in' } }, + ], + }, + }); + await h.c.toolQueue; + assert.equal(submitted, true); + assert.deepEqual(h.calls, []); + h.c.stop(); +}); diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index 26974ce60..bf3e5b8b8 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -1,6 +1,5 @@ import { layerSnapshot, - layerSnapshots, feedProvenanceEnvelope, } from '../data/layerSnapshot.js'; import { readLayerLifecycleSummary } from './layerSummary.js'; @@ -27,7 +26,6 @@ import { isContextRecordActive, } from '../data/contextStore.js'; import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js'; -import { contextModeWord } from '../contextModePolicy.js'; import { createAnalystEngine, normalizeAnalystSpec, @@ -48,6 +46,12 @@ import { unavailablePlaceSearch } from '../search/placeSearch.js'; import * as defaultAnnotationResolver from '../annotations/annotationResolver.js'; import { normalizeRadioCountryInput } from '../data/radioCountry.js'; import { TR3B_CLASS } from '../data/tr3bRegistry.js'; +import { + getWorldPacket, + withContextModeVocabulary, + activeContactsWindow, + TRACKABLE_FAMILIES, +} from '../agent/worldPacket.js'; const ALLOWED_STYLES = new Set([ 'normal', @@ -119,64 +123,6 @@ const CONTEXT_MODE_ALIASES = new Map([ ['space-missions', 'space-missions'], ['missions', 'space-missions'], ]); -/** - * Every model-readable field that carries a context-mode id, and what an - * absent value means for each. - * - * `mode` always names a mode, so nothing is 'off'. `entering` and `priorMode` - * are absent when there is no such mode at all β€” calling those 'off' would - * assert a state that does not exist. - */ -const CONTEXT_MODE_RESULT_FIELDS = Object.freeze([ - { field: 'mode', emptyAs: 'off' }, - { field: 'entering', emptyAs: null }, - { field: 'priorMode', emptyAs: null }, -]); - -/** Nested results that are themselves context-mode payloads the model reads. */ -const NESTED_CONTEXT_RESULT_FIELDS = Object.freeze([ - 'context', - 'contextRollback', -]); - -/** - * Report a context-mode payload in the tools' own vocabulary. - * - * `set_context_mode` accepts 'contacts' while the mode's internal id is - * 'flights'. Reporting the internal id back made the model read - * `mode:'flights'` as "Contacts is off" and refuse to answer from the Contacts - * window counts sitting in the very same payload (owner field session - * 2026-08-21). Secondary fields and nested transition/rollback results are - * translated too β€” one leaked internal id is enough to recreate the confusion, - * and a rollback result is exactly what the model reads when something went - * wrong. Each internal id is kept alongside as `Internal` for anything - * reasoning about layers. - * @param {object|null|undefined} state Any payload carrying context-mode fields. - * @returns {object|null|undefined} The same payload, modes translated. - */ -function withContextModeVocabulary(state) { - if (!state || typeof state !== 'object') return state; - let out = state; - const mutable = () => { - if (out === state) out = { ...state }; - return out; - }; - for (const { field, emptyAs } of CONTEXT_MODE_RESULT_FIELDS) { - if (!(field in state)) continue; - const internal = state[field] ?? null; - const target = mutable(); - target[field] = contextModeWord(internal, { emptyAs }); - target[`${field}Internal`] = internal; - } - for (const field of NESTED_CONTEXT_RESULT_FIELDS) { - const nested = state[field]; - if (!nested || typeof nested !== 'object') continue; - const translated = withContextModeVocabulary(nested); - if (translated !== nested) mutable()[field] = translated; - } - return out; -} - const COCKPIT_ACTION_ALIASES = new Map([ ['next', 'next'], ['previous', 'previous'], @@ -246,14 +192,6 @@ const STACK_ALIASES = new Map([ ['road map', 'osm'], ]); -/** Search order for track_entity across entity layer families. */ -const TRACKABLE_FAMILIES = [ - { layerId: 'flights', kind: 'aircraft' }, - { layerId: 'military', kind: 'aircraft' }, - { layerId: 'ais-live-vessels', kind: 'vessel' }, - { layerId: 'satellites', kind: 'satellite' }, -]; - const FRAME_TARGETS = new Map([ ['flights', 'flights'], ['planes', 'flights'], @@ -2553,26 +2491,6 @@ function runManagedVoiceNavigation( }; } -/** Gathers tracked/selected entities across layer families for read-back. */ -function collectTrackedEntities(dataManager) { - const tracked = []; - for (const family of TRACKABLE_FAMILIES) { - const module = dataManager.layers.get(family.layerId)?.module; - if (!module) continue; - try { - const info = - family.kind === 'vessel' - ? module.getSelectedInfo?.() - : module.getTrackedInfo?.(); - if (info) - tracked.push({ kind: family.kind, layerId: family.layerId, ...info }); - } catch { - // layer not ready - } - } - return tracked; -} - export async function getBasemapLabelContext( viewer, service = defaultGeospatial, @@ -3210,60 +3128,12 @@ function getCurrentViewState( dataManager, sceneDirector = null, ) { - const cartographic = Cesium.Cartographic.fromCartesian( - viewer.camera.positionWC, - ); - return { - ok: true, - action: 'get_current_view_state', - camera: { - latitude: Cesium.Math.toDegrees(cartographic.latitude), - longitude: Cesium.Math.toDegrees(cartographic.longitude), - heightM: cartographic.height, - }, - style: styleManager.activeStyle || 'normal', - context: - typeof styleManager.getContextModeState === 'function' - ? { - ...withContextModeVocabulary(styleManager.getContextModeState()), - // The numbers on the operator's Contacts panel, so a window/count - // question can be answered from what they are looking at. - ...(activeContactsWindow(dataManager) - ? { contactsWindow: activeContactsWindow(dataManager) } - : {}), - } - : null, - cockpit: - typeof styleManager.getCockpitState === 'function' - ? styleManager.getCockpitState() - : null, - controls: - typeof styleManager.getControlState === 'function' - ? styleManager.getControlState() - : null, - scenePlayback: sceneDirector?.getPlaybackStatus?.() || null, - tracked: collectTrackedEntities(dataManager), - // Enabled layers only: the full registry cost ~1k tokens per call and - // the model only reasons about what is on. - disabledLayerCount: dataManager.getAll().filter((layer) => !layer.enabled) - .length, - layers: dataManager - .getAll() - .filter((layer) => layer.enabled) - .map((layer) => ({ - id: layer.id, - name: layer.name, - enabled: layer.enabled, - count: layer.stats?.count || 0, - error: layer.stats?.error || null, - feedState: layerSnapshot(layer).feedState, - source: layerSnapshot(layer).source, - lastUpdate: layerSnapshot(layer).lastUpdate, - })), - feedProvenance: feedProvenanceEnvelope( - layerSnapshots(dataManager.getAll()).filter((layer) => layer.enabled), - ), - }; + return getWorldPacket({ + viewer, + styleManager, + dataManager, + sceneDirector, + }); } async function getEntityContext( @@ -4635,23 +4505,6 @@ function clampNumber(value, min, max, fallback) { * ("which of those is closest?") scoped to the session. */ -/** - * The Contacts panel's own counts, or null when Contacts has no subject. - * Read through the awareness snapshot the panel renders, so the two cannot - * diverge no matter which surface asks. - * @returns {object|null} Panel-equivalent window counts. - */ -function activeContactsWindow(dataManager) { - try { - const layer = dataManager?.layers?.get('military-awareness')?.module; - return ( - layer?.contactsWindowFromSnapshot?.(layer.getContextSnapshot?.()) ?? null - ); - } catch { - return null; - } -} - /** * Most records one layer hands a single query. Above every loaded feed seen * so far (flights ~8k, vessels ~12k, a 24 h global FIRMS day ~160k), so diff --git a/src/voice/gevRealtime.js b/src/voice/gevRealtime.js index 58efacad3..8364a5723 100644 --- a/src/voice/gevRealtime.js +++ b/src/voice/gevRealtime.js @@ -61,10 +61,8 @@ export function initGevVoiceCommands(options) { }); deixis.imageFrame = () => commands.retainedImageFrame?.() || null; deixis.onPointerUsed = () => commands.announcePointer?.(); - commands.session?.signal?.addEventListener( - 'abort', - () => pointer?.destroy(), - { once: true }, - ); + const lifetime = commands.signal ?? commands.session?.signal; + lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true }); + if (lifetime?.aborted) pointer?.destroy(); return commands; } diff --git a/src/voice/pointerContext.js b/src/voice/pointerContext.js index d54c0bd64..3f9d09b6d 100644 --- a/src/voice/pointerContext.js +++ b/src/voice/pointerContext.js @@ -182,6 +182,63 @@ export function pointerChip(snapshot) { }; } +/** Dedupe key of the clearing pointer_context item. */ +export const POINTER_NONE_KEY = 'none'; + +/** + * The voice-card event for a turn's pointer: a gesture reveals its target at + * once; open-mic and typed turns reveal it only through announcePointer. + * @param {object|null} snapshot + * @param {boolean} gesture + * @returns {{announced: object|null, pointer: object|null}} + */ +export function pointerTurnEvent(snapshot, gesture) { + const chip = gesture ? pointerChip(snapshot) : null; + return chip + ? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } } + : { announced: null, pointer: null }; +} + +/** + * The pointer_context item a turn should send, or null when there is nothing + * new. Without a gesture the model learns only what kind of thing is pointed + * at, never which. Losing a sent pointer produces one clearing item. + * @param {object|null} snapshot + * @param {{at: string, gesture: boolean, sentKey: string|null}} turn + * @returns {{item: object, key: string}|null} + */ +export function pointerTurnContext(snapshot, { at, gesture, sentKey }) { + const chip = pointerChip(snapshot); + const item = !chip + ? sentKey && sentKey !== POINTER_NONE_KEY + ? { type: 'pointer_context', at, target: 'none' } + : null + : gesture + ? pointerContextItem(snapshot) + : { + type: 'pointer_context', + at, + pointing: snapshot.entity?.kind || 'ground', + }; + if (!item) return null; + const { at: _at, ...content } = item; + const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY; + return key === sentKey ? null : { item, key }; +} + +/** + * The voice-card event revealing a pointer a tool resolved, once per snapshot. + * @param {object|null} snapshot + * @param {object|null} announced Snapshot already shown. + * @returns {object|null} + */ +export function pointerAnnouncement(snapshot, announced) { + const chip = pointerChip(snapshot); + return chip && snapshot !== announced + ? { ...chip, screenPx: snapshot.screenPx } + : null; +} + /** * Track the pointer over one element (the globe canvas). * @param {object} options @@ -330,7 +387,14 @@ export function createPointerTracker({ * (open mic only when it moved recently: `recentMs`). The result becomes * the turn's referent (null when stale). */ - beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) { + beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) { + // Delayed admitted audio keeps its capture's pointer without replacing a + // newer physical hold. Undefined preserves existing caller behavior. + if (capturedSnapshot !== undefined) { + const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot; + active = snapshot?.fresh ? snapshot : null; + return snapshot; + } const reuse = at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS; const snapshot = reuse ? held.snapshot : capture(at, { recentMs }); diff --git a/src/voice/pointerContext.test.mjs b/src/voice/pointerContext.test.mjs index 1d4f2e1fe..7d8d56d9c 100644 --- a/src/voice/pointerContext.test.mjs +++ b/src/voice/pointerContext.test.mjs @@ -6,8 +6,13 @@ import { POINTER_HOLD_MS, buildPointerSnapshot, createPointerTracker, + POINTER_NONE_KEY, formatPointerContext, + pointerAnnouncement, pointerChip, + pointerContextItem, + pointerTurnContext, + pointerTurnEvent, } from './pointerContext.js'; function harness({ pick } = {}) { @@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => { assert.equal(h.tracker.capture('keydown').fresh, false); }); +test('delayed speech uses its released capture without replacing a newer physical hold', () => { + const h = harness(); + h.move(700, 250); + const capturedA = h.tracker.hold(); + h.tracker.releaseHold(); + h.move(200, 100); + h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA }); + assert.equal(h.tracker.activeSnapshot(), capturedA); + const heldB = h.tracker.hold(); + h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA }); + assert.equal(h.tracker.activeSnapshot(), capturedA); + assert.equal(h.tracker.isHeld(), true); + assert.equal(h.tracker.beginTurn('speech_start'), heldB); + h.tracker.destroy(); +}); + +test('explicit stale captures clear the referent while a silent hold leaves it alone', () => { + const h = harness(); + h.move(700, 250); + const current = h.tracker.beginTurn('text'); + h.move(200, 100); + h.tracker.hold(); + h.tracker.releaseHold(); + assert.equal(h.tracker.activeSnapshot(), current); + for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) { + h.tracker.beginTurn('text'); + h.tracker.beginTurn('speech_start', { capturedSnapshot }); + assert.equal(h.tracker.activeSnapshot(), null); + } + h.tracker.destroy(); +}); + +test('a capture arriving after destruction cannot restore an active referent', () => { + const h = harness(); + h.move(700, 250); + const capturedSnapshot = h.tracker.hold(); + assert.equal(capturedSnapshot.fresh, true); + h.tracker.destroy(); + const result = h.tracker.beginTurn('speech_start', { capturedSnapshot }); + assert.equal(result.fresh, false); + assert.equal(h.tracker.activeSnapshot(), null); + assert.equal(h.tracker.isHeld(), false); + assert.equal(h.tracker.beginTurn('speech_start').fresh, false); +}); + test('format: the model item is compact inert JSON; labels are bounded data', () => { const snapshot = buildPointerSnapshot({ at: 'speech_start', @@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => { assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' }); assert.equal(pointerChip(null), null); }); + +test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => { + const snapshot = { + fresh: true, + at: 'keydown', + entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' }, + lat: 10, + lon: 20, + screen: { x: 5, y: 6 }, + screenPx: { x: 5, y: 6 }, + }; + const shown = pointerTurnEvent(snapshot, true); + assert.equal(shown.announced, snapshot); + assert.deepEqual(shown.pointer, { + kind: 'this', + label: 'UAL1', + screenPx: { x: 5, y: 6 }, + }); + assert.deepEqual(pointerTurnEvent(snapshot, false), { + announced: null, + pointer: null, + }); + + const gesture = pointerTurnContext(snapshot, { + at: 'keydown', + gesture: true, + sentKey: null, + }); + assert.deepEqual(gesture.item, pointerContextItem(snapshot)); + assert.equal( + pointerTurnContext(snapshot, { + at: 'keydown', + gesture: true, + sentKey: gesture.key, + }), + null, + ); + assert.deepEqual( + pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null }) + .item, + { type: 'pointer_context', at: 'text', pointing: 'aircraft' }, + ); + + const cleared = pointerTurnContext(null, { + at: 'text', + gesture: false, + sentKey: gesture.key, + }); + assert.deepEqual(cleared, { + item: { type: 'pointer_context', at: 'text', target: 'none' }, + key: POINTER_NONE_KEY, + }); + for (const sentKey of [null, POINTER_NONE_KEY]) + assert.equal( + pointerTurnContext(null, { at: 'text', gesture: false, sentKey }), + null, + ); + + assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer); + assert.equal(pointerAnnouncement(snapshot, snapshot), null); + assert.equal(pointerAnnouncement(null, null), null); +}); diff --git a/src/voice/providerCapabilities.js b/src/voice/providerCapabilities.js new file mode 100644 index 000000000..128785cc8 --- /dev/null +++ b/src/voice/providerCapabilities.js @@ -0,0 +1,21 @@ +export const VOICE_PROVIDERS = Object.freeze({ + openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name + gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name +}); + +/** Optional providers require an explicit host opt-in and an available factory. */ +export function supportedVoiceProviders( + requested = ['openai'], + factories = {}, +) { + if (!Array.isArray(requested)) + throw new TypeError('supportedProviders must be an array'); + const selected = [...new Set(requested)].filter( + (name) => + Object.hasOwn(VOICE_PROVIDERS, name) && + typeof factories[name] === 'function', + ); + if (!selected.length) + throw new Error('No supported voice provider is configured'); + return Object.freeze(selected); +} diff --git a/src/voice/providerCommands.js b/src/voice/providerCommands.js new file mode 100644 index 000000000..86855e3bc --- /dev/null +++ b/src/voice/providerCommands.js @@ -0,0 +1,161 @@ +import { supportedVoiceProviders } from './providerCapabilities.js'; + +const STORAGE_KEY = 'gev.voice.provider'; + +export function readVoiceProvider(storage, supported = ['openai', 'gemini']) { + let saved; + try { + saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY); + } catch { + /* Storage is optional. */ + } + return supported.includes(saved) + ? saved + : supported.includes('openai') + ? 'openai' + : supported[0]; +} + +/** Keep one public controller identity while replacing each provider's lifetime. */ +export function createProviderCommands(options, { bind, factories, storage }) { + const supportedProviders = supportedVoiceProviders( + options.supportedProviders ?? Object.keys(factories), + factories, + ); + window.__gevVoiceCommands?.stop?.({ removeUi: true }); + let provider = readVoiceProvider(storage, supportedProviders); + let current; + let disposed = false; + const lifetime = new AbortController(); + let mountGeneration = 0; + let voiceEnabled = false; + const stored = () => { + try { + return storage ?? globalThis.localStorage; + } catch { + return null; + } + }; + const stop = (settings = {}) => { + if (disposed) return; + voiceEnabled = false; + if (settings.removeUi) { + disposed = true; + options.signal?.removeEventListener('abort', destroy); + } + try { + current?.session.stop(settings); + } finally { + if (settings.removeUi) { + lifetime.abort(); + options.runner.dispose?.(); + } + } + }; + const destroy = () => stop({ removeUi: true }); + const selectProvider = (next) => { + if (disposed || !supportedProviders.includes(next) || next === provider) + return false; + const resume = voiceEnabled && current.session.isActive(); + // Carry the input mode, never a held key, microphone or conversation. + const pushToTalk = Boolean( + current.session.adapter.getStartOptions?.().pushToTalk, + ); + const generation = ++mountGeneration; + voiceEnabled = false; + current.session.destroy(); + // Disposal suppresses the retired adapter's idle event. Clear the shared + // conversation explicitly before the new mount can use these owners. + if (disposed || generation !== mountGeneration) return false; + options.pointer?.clear(); + options.referents?.clear(); + options.runner.resetConversation?.(); + if (disposed || generation !== mountGeneration) return false; + provider = next; + try { + stored()?.setItem(STORAGE_KEY, provider); + } catch { + /* Storage is optional. */ + } + mount(); + if (resume && !disposed && provider === next) + void current.session.start({ pushToTalk }); + globalThis.document?.getElementById('gev-voice-provider')?.focus(); + return true; + }; + + const handleSessionEvent = (event) => { + if (event?.type === 'state' && event.state === 'connecting') + voiceEnabled = true; + if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) { + voiceEnabled = false; + } + // The host still observes the mounted provider's neutral session events. + options.onSessionEvent?.(event); + }; + + const sendTextCommand = (text) => { + if (disposed) return false; + return current.session.sendText(text); + }; + + const api = { + // The shared action surface is independent of an active voice connection. + runner: options.runner, + supportedProviders, + signal: lifetime.signal, + get provider() { + return provider; + }, + get session() { + return current?.session; + }, + get status() { + return current?.status ?? current?.session.state ?? 'idle'; + }, + get state() { + return current?.session.state ?? 'idle'; + }, + get disposed() { + return disposed; + }, + start: (settings) => !disposed && current.session.start(settings), + stop, + destroy, + isActive: () => !disposed && current.session.isActive(), + sendTextCommand, + sendText: sendTextCommand, + notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event), + }; + // Existing diagnostic and controller accessors continue to reflect the active provider. + const facade = new Proxy(api, { + get(target, name, receiver) { + if (Reflect.has(target, name)) return Reflect.get(target, name, receiver); + const value = current?.[name]; + return typeof value === 'function' ? value.bind(current) : value; + }, + }); + function mount() { + voiceEnabled = false; + const generation = ++mountGeneration; + current = bind({ + ...options, + signal: lifetime.signal, + resetExisting: false, + disposeRunner: false, + provider, + supportedProviders, + onProviderChange: selectProvider, + onSessionEvent: (event) => { + if (!disposed && generation === mountGeneration) + handleSessionEvent(event); + }, + createSession: factories[provider], + }); + window.__gevVoiceCommands = facade; + } + mount(); + options.signal?.addEventListener('abort', destroy, { once: true }); + if (options.signal?.aborted) destroy(); + return facade; +} diff --git a/src/voice/providerCommands.test.mjs b/src/voice/providerCommands.test.mjs new file mode 100644 index 000000000..74d99699b --- /dev/null +++ b/src/voice/providerCommands.test.mjs @@ -0,0 +1,888 @@ +// Imports intentionally target the candidate, never the immutable reference. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + createProviderCommands, + readVoiceProvider, +} from './providerCommands.js'; +import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js'; +import { createVoiceCommands as createCommands } from './commands.js'; +import { createGevActionRunner } from './gevActions.js'; +import { createPointerTracker } from './pointerContext.js'; +import { createReferentRegistry } from './referents.js'; +import { describeUnsavedWork, t as translate } from '../i18n/index.js'; + +const PROVIDER_KEY = 'gev.voice.provider'; +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +function deferred() { + let resolve; + let reject; + const promise = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + return { promise, resolve, reject }; +} + +// Listener counts observe the actual binder's registrations/removals. +class Element extends EventTarget { + listeners = new Map(); + attributes = new Map(); + addEventListener(type, handler, options) { + if (!this.listeners.has(type)) this.listeners.set(type, new Set()); + this.listeners.get(type).add(handler); + super.addEventListener(type, handler, options); + } + removeEventListener(type, handler, options) { + this.listeners.get(type)?.delete(handler); + super.removeEventListener(type, handler, options); + } + listenerCount(type) { + return this.listeners.get(type)?.size || 0; + } + setAttribute(name, value) { + this.attributes.set(name, String(value)); + } + focus() { + this.focusCount = (this.focusCount || 0) + 1; + } +} + +for (const custom of [false, true]) { + test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => { + const prior = globalThis.window; + globalThis.window = new EventTarget(); + let controls; + t.after(() => { + controls?.session.destroy(); + globalThis.window = prior; + }); + const ui = { + root: { dataset: {}, remove() {} }, + button: new Element(), + status: {}, + detail: {}, + providerField: {}, + }; + let defaultCreated = 0; + let customCreated = 0; + const options = { + runner: async () => ({ ok: true }), + createControl: () => ui, + createController: ({ onSessionEvent }) => { + defaultCreated++; + return { + ui, + start({ pushToTalk }) { + this.pushToTalkMode = pushToTalk; + onSessionEvent({ type: 'state', state: 'listening' }); + }, + stop() {}, + syncCostUi() {}, + bindPushToTalkShortcut() {}, + sendTextCommand: (text) => `default:${text}`, + }; + }, + }; + if (custom) { + options.supportedProviders = ['gemini']; + options.createSession = ({ emit }) => { + customCreated++; + return { + capabilities: { pushToTalk: false }, + start() { + emit({ type: 'state', state: 'listening' }); + }, + stop() {}, + sendText: (text) => `custom:${text}`, + sendMapEvent: () => true, + }; + }; + } + controls = createCommands(options); + await controls.session.start({ pushToTalk: true }); + assert.equal(defaultCreated, custom ? 0 : 1); + assert.equal(customCreated, custom ? 1 : 0); + assert.equal( + controls.session.sendText('test'), + `${custom ? 'custom' : 'default'}:test`, + ); + assert.equal(ui.providerField.hidden, true); + assert.equal(ui.root.dataset.providerChoice, 'false'); + if (!custom) + assert.deepEqual(controls.session.adapter.getStartOptions(), { + pushToTalk: true, + }); + }); +} + +/** Real facade -> real binder -> real common session and real action runner. + * Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count + * calls to real pointer/referent/conversation cleanup; they do not replace it. + */ +function compositionFixture( + t, + { + supportedProviders = ['openai', 'gemini'], + factoryProviders = ['openai', 'gemini'], + savedProvider = null, + storageDenied = false, + gateStarts = () => false, + onSessionEvent = () => {}, + } = {}, +) { + const original = new Map( + ['window', 'document'].map((key) => [ + key, + Object.getOwnPropertyDescriptor(globalThis, key), + ]), + ); + const lifetime = new AbortController(); + const views = []; + const instances = []; + const mounts = []; + const events = []; + const order = []; + const annotations = new Set(); + const storageWrites = []; + const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 }; + const state = { height: 1000, zooms: 0, renders: 0 }; + let facade; + let pointer; + globalThis.window = new EventTarget(); + globalThis.document = { + getElementById: (id) => + id === 'gev-voice-provider' ? views.at(-1)?.providerSelect : null, + }; + t.after(async () => { + try { + facade?.destroy(); + pointer?.destroy(); + for (const instance of instances) + for (const gate of instance.gates) gate.resolve(); + await Promise.allSettled(instances.flatMap((instance) => instance.jobs)); + } finally { + for (const [key, descriptor] of original) { + if (descriptor) Object.defineProperty(globalThis, key, descriptor); + else delete globalThis[key]; + } + } + }); + + const canvas = new EventTarget(); + Object.assign(canvas, { clientWidth: 800, clientHeight: 600 }); + const position = { x: 6379137, y: 0, z: 0 }; + const viewer = { + clock: { onTick: { addEventListener: () => () => {} } }, + camera: { + positionWC: position, + positionCartographic: { height: state.height }, + moveEnd: { addEventListener() {} }, + cancelFlight() {}, + zoomIn(metres) { + position.x -= metres; + state.height -= metres; + this.positionCartographic.height = state.height; + state.zooms++; + }, + zoomOut(metres) { + position.x += metres; + state.height += metres; + this.positionCartographic.height = state.height; + state.zooms++; + }, + }, + scene: { + canvas, + requestRender() { + state.renders++; + }, + }, + }; + const dataManager = { layers: new Map() }; + pointer = createPointerTracker({ element: null, now: () => 1000 }); + const referents = createReferentRegistry(); + const runner = createGevActionRunner({ + viewer, + dataManager, + styleManager: {}, + deixis: { pointer, referents }, + }); + for (const [object, method, counter] of [ + [pointer, 'clear', 'pointer'], + [referents, 'clear', 'referents'], + [runner, 'resetConversation', 'conversation'], + [runner, 'dispose', 'dispose'], + ]) { + const originalMethod = object[method]; + object[method] = (...args) => { + counters[counter]++; + order.push(counter); + return originalMethod.apply(object, args); + }; + } + + const factories = Object.fromEntries( + factoryProviders.map((provider) => [ + provider, + (hooks) => { + const id = instances.length; + let generation = 0; + const instance = { + id, + provider, + hooks, + starts: [], + stops: [], + jobs: [], + gates: [], + acquired: 0, + microphoneEnabled: false, + bound: 0, + input: { pushToTalk: false, held: false }, + texts: [], + mapEvents: [], + }; + instances.push(instance); + order.push(`factory:${id}`); + const adapter = { + identity: `${provider}:${id}`, + capabilities: { + pushToTalk: true, + costControls: provider === 'openai', + }, + controller: { + providerDiagnostic() { + return this.session.adapter.identity; + }, + }, + start(settings = {}) { + const attempt = ++generation; + instance.starts.push({ ...settings }); + instance.input.pushToTalk = Boolean(settings.pushToTalk); + hooks.emit({ + type: 'state', + state: 'connecting', + detail: `adapter:${provider}`, + }); + const job = (async () => { + if (attempt !== generation || hooks.signal.aborted) return; + if (gateStarts(instance)) { + const gate = deferred(); + instance.gates.push(gate); + await gate.promise; + } + // Adapters own cancellation of their asynchronous acquisition. + if (attempt !== generation || hooks.signal.aborted) return; + instance.acquired++; + instance.microphoneEnabled = + !instance.input.pushToTalk || instance.input.held; + hooks.emit({ + type: 'state', + state: 'listening', + detail: provider, + }); + })(); + instance.jobs.push(job); + return job; + }, + stop(settings = {}) { + generation++; + instance.stops.push({ ...settings }); + order.push(`stop:${id}`); + instance.input.pushToTalk = instance.input.held = false; + instance.microphoneEnabled = false; + }, + getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }), + bindControls() { + instance.bound++; + }, + sendText(text) { + instance.texts.push(text); + return `${provider}:${text}`; + }, + sendMapEvent(event) { + instance.mapEvents.push(event); + return true; + }, + }; + instance.adapter = adapter; + return adapter; + }, + ]), + ); + + function createControl() { + const id = views.length; + order.push(`control:${id}`); + const providerSelect = new Element(); + providerSelect.ownerDocument = { createElement: () => ({}) }; + providerSelect.replaceChildren = (...options) => { + providerSelect.options = options; + }; + const view = { + root: { + dataset: {}, + classList: { remove() {} }, + removeCount: 0, + remove() { + this.removeCount++; + order.push(`remove:${id}`); + }, + }, + button: new Element(), + providerSelect, + providerField: {}, + status: {}, + detail: {}, + errorDetail: {}, + tierButton: {}, + costValue: {}, + helpDetail: {}, + card: {}, + }; + views.push(view); + return view; + } + function passiveView() { + return { + events: [], + outlines: [], + destroyed: 0, + handle(event) { + this.events.push(event); + }, + outline(event) { + this.outlines.push(event); + }, + destroy() { + this.destroyed++; + }, + }; + } + + facade = createProviderCommands( + { + runner, + pointer, + referents, + dataManager, + signal: lifetime.signal, + supportedProviders, + createControl, + createCard() { + const card = passiveView(); + views.at(-1).cardView = card; + return card; + }, + createReticle() { + const reticle = passiveView(); + views.at(-1).reticle = reticle; + return reticle; + }, + annotations: { + onOutlineEvent(callback) { + annotations.add(callback); + return () => annotations.delete(callback); + }, + }, + onSessionEvent(event) { + events.push(event); + onSessionEvent(event, facade); + }, + }, + { + factories, + storage: { + getItem() { + if (storageDenied) throw Error('Storage denied'); + return savedProvider; + }, + setItem(key, value) { + if (storageDenied) throw Error('Storage denied'); + storageWrites.push([key, value]); + savedProvider = value; + }, + }, + bind(options) { + // Keep the real callback so stale-event suppression tests exercise the + // facade's mount generation as well as the common session's disposal. + const mount = { options, callback: options.onSessionEvent }; + mounts.push(mount); + mount.controls = bindVoiceCommands(options); + return mount.controls; + }, + }, + ); + return { + facade, + runner, + pointer, + referents, + lifetime, + instances, + views, + mounts, + events, + order, + annotations, + counters, + storageWrites, + state, + select(provider) { + const select = views.at(-1).providerSelect; + select.value = provider; + select.dispatchEvent(new Event('change')); + }, + seedConversation() { + pointer.beginTurn('text', { + capturedSnapshot: { fresh: true, at: 'text' }, + }); + referents.recordResult('frame_overhead', { + ok: true, + display: { title: 'One aircraft' }, + referents: [ + { n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' }, + ], + }); + assert.ok(pointer.activeSnapshot()); + assert.equal(referents.list().length, 1); + }, + }; +} + +test('provider preference uses only the supplied host capability list', () => { + const stored = (value) => ({ getItem: () => value }); + assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai'); + assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini'); + assert.equal( + readVoiceProvider(stored('unknown'), ['gemini', 'openai']), + 'openai', + ); + assert.equal( + readVoiceProvider(stored('gemini'), ['openai', 'gemini']), + 'gemini', + ); + assert.equal( + readVoiceProvider( + { + getItem() { + throw Error('denied'); + }, + }, + ['gemini'], + ), + 'gemini', + ); +}); + +for (const scenario of [ + { + title: 'OpenAI-only host ignores saved Gemini', + requested: ['openai'], + saved: 'gemini', + expected: ['openai'], + provider: 'openai', + }, + { + title: 'Gemini-only host falls back from saved OpenAI', + requested: ['gemini'], + saved: 'openai', + expected: ['gemini'], + provider: 'gemini', + }, + { + title: 'dual host restores Gemini', + requested: ['openai', 'gemini'], + saved: 'gemini', + expected: ['openai', 'gemini'], + provider: 'gemini', + }, + { + title: 'unknown, duplicate and unavailable factories are excluded', + requested: ['unknown', 'gemini', 'openai', 'gemini'], + factories: ['gemini'], + saved: 'openai', + expected: ['gemini'], + provider: 'gemini', + }, +]) { + test(scenario.title, (t) => { + const f = compositionFixture(t, { + supportedProviders: scenario.requested, + factoryProviders: scenario.factories, + savedProvider: scenario.saved, + }); + assert.deepEqual(f.facade.supportedProviders, scenario.expected); + assert.equal(Object.isFrozen(f.facade.supportedProviders), true); + assert.equal(f.facade.provider, scenario.provider); + assert.deepEqual( + f.views[0].providerSelect.options.map((option) => option.value), + scenario.expected, + ); + assert.equal( + f.views[0].providerField.hidden, + scenario.expected.length === 1, + ); + assert.equal( + f.views[0].root.dataset.providerChoice, + String(scenario.expected.length > 1), + ); + assert.equal(f.instances.length, 1); + assert.equal( + f.instances[0].starts.length, + 0, + 'discovery must not start a provider', + ); + assert.equal(f.facade.state, 'idle'); + }); +} + +test('invalid or empty host support fails before mounting a control', () => { + let mounts = 0; + const dependencies = { + bind() { + mounts++; + }, + factories: { openai() {} }, + }; + assert.throws( + () => + createProviderCommands({ supportedProviders: 'gemini' }, dependencies), + /must be an array/, + ); + assert.throws( + () => + createProviderCommands( + { supportedProviders: ['gemini', 'unknown'] }, + dependencies, + ), + /No supported voice provider/, + ); + assert.equal(mounts, 0); +}); + +test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => { + const f = compositionFixture(t); + const firstSession = f.facade.session; + const outerSignal = f.facade.signal; + const first = f.instances[0]; + // PTT controls can activate the adapter directly; the facade must learn + // activity from its common events, not only from facade.start(). + first.input.held = true; + await first.adapter.start({ pushToTalk: true }); + assert.equal(first.microphoneEnabled, true); + assert.equal(f.facade.isActive(), true); + f.seedConversation(); + const before = { ...f.counters }; + f.select('gemini'); + await settle(); + const second = f.instances[1]; + assert.equal(window.__gevVoiceCommands, f.facade); + assert.equal(f.facade.runner, f.runner); + assert.equal(f.facade.signal, outerSignal); + assert.equal(outerSignal.aborted, false); + assert.equal(firstSession.disposed, true); + assert.equal(first.hooks.signal.aborted, true); + assert.deepEqual(first.stops, [{ removeUi: true }]); + assert.equal(first.input.held, false); + assert.equal(first.microphoneEnabled, false); + assert.deepEqual(second.starts, [{ pushToTalk: true }]); + assert.equal(second.input.held, false); + assert.equal( + second.microphoneEnabled, + false, + 'a held key must never migrate to the new adapter', + ); + assert.equal(f.facade.state, 'listening'); + assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1')); + assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1')); + assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1')); + assert.equal(f.views[0].button.listenerCount('click'), 0); + assert.equal(f.views[0].providerSelect.listenerCount('change'), 0); + assert.equal(f.views[1].providerSelect.listenerCount('change'), 1); + assert.equal(f.views[1].providerSelect.focusCount, 1); + assert.equal(f.views[1].tierButton.hidden, true); + assert.equal(f.views[1].costValue.hidden, true); + assert.equal(f.facade.providerDiagnostic(), 'gemini:1'); + assert.equal(f.pointer.activeSnapshot(), null); + assert.deepEqual(f.referents.list(), []); + for (const key of ['pointer', 'referents', 'conversation']) + assert.equal( + f.counters[key], + before[key] + 1, + `one explicit replacement reset: ${key}`, + ); + assert.equal( + f.counters.dispose, + 0, + 'provider replacement keeps the page runner', + ); + assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]); + assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello'); + assert.deepEqual(first.texts, []); + assert.deepEqual(second.texts, ['hello']); + const result = await f.facade.runner('adjust_camera_zoom', { + direction: 'in', + amount: 'little', + }); + assert.equal( + result.ok, + true, + 'the actual shared action runner remains usable after switching', + ); + assert.equal(f.state.zooms, 1); +}); + +for (const initialState of ['idle', 'error']) { + test(`${initialState} selection never autostarts; manual start still works`, async (t) => { + const f = compositionFixture(t, { storageDenied: true }); + if (initialState === 'error') { + await f.facade.start({ pushToTalk: true }); + f.instances[0].adapter.stop({ preserveStatus: true }); + f.instances[0].hooks.emit({ + type: 'state', + state: 'error', + detail: 'Fixture failure', + }); + } + f.select('gemini'); + await settle(); + assert.equal( + f.facade.provider, + 'gemini', + 'denied storage cannot block a supported selection', + ); + assert.equal(f.facade.state, 'idle'); + assert.deepEqual(f.instances[1].starts, []); + f.select('gemini'); + f.select('unknown'); + assert.equal( + f.instances.length, + 2, + 'same and unsupported selections cannot remount', + ); + assert.equal(f.facade.provider, 'gemini'); + await f.facade.start({ pushToTalk: false }); + assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]); + assert.equal(f.instances[1].microphoneEnabled, true); + }); +} + +for (const outcome of ['resolve', 'reject']) { + test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => { + const f = compositionFixture(t, { + gateStarts: (instance) => instance.id === 0, + }); + const oldStart = f.facade.start({ pushToTalk: false }); + const old = f.instances[0]; + assert.equal(f.facade.state, 'connecting'); + f.select('gemini'); + await settle(); + assert.equal(f.facade.state, 'listening'); + const current = f.instances[1]; + const eventCount = f.events.length; + const currentStops = current.stops.length; + const currentCardCount = f.views[1].cardView.events.length; + old.gates[0][outcome](new Error('Late retired startup failure')); + await oldStart; + for (const state of ['idle', 'error', 'connecting']) { + old.hooks.emit({ type: 'state', state, detail: 'retired adapter' }); + f.mounts[0].callback({ + type: 'state', + state, + detail: 'retired forward callback', + }); + } + assert.equal(f.events.length, eventCount); + assert.equal(f.views[1].cardView.events.length, currentCardCount); + assert.equal(f.facade.state, 'listening'); + assert.equal(old.acquired, 0); + assert.equal(current.stops.length, currentStops); + assert.equal(current.starts.length, 1); + f.facade.stop(); + f.mounts[0].callback({ type: 'state', state: 'connecting' }); + f.select('openai'); + assert.deepEqual( + f.instances[2].starts, + [], + 'a retired connecting callback cannot revive voice intent', + ); + assert.equal(f.facade.state, 'idle'); + }); +} + +test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => { + const f = compositionFixture(t, { + gateStarts: (instance) => instance.id === 1, + }); + await f.facade.start(); + f.select('gemini'); + const replacement = f.instances[1]; + assert.equal(f.facade.state, 'connecting'); + f.facade.stop(); + replacement.gates[0].resolve(); + await Promise.all(replacement.jobs); + assert.equal(replacement.acquired, 0); + assert.equal(f.facade.state, 'idle'); + f.select('openai'); + assert.deepEqual(f.instances[2].starts, []); + await f.facade.start(); + assert.equal( + f.facade.state, + 'listening', + 'a later explicit start remains healthy', + ); +}); + +for (const emitsIdle of [false, true]) { + test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => { + const f = compositionFixture(t); + if (emitsIdle) { + const instance = f.instances[0]; + const originalStop = instance.adapter.stop; + instance.adapter.stop = function (settings = {}) { + originalStop.call(this, settings); + if (!settings.preserveStatus && !settings.removeUi) + instance.hooks.emit({ type: 'state', state: 'idle' }); + }; + } + for (const end of ['stop', 'error']) { + await f.facade.start(); + f.seedConversation(); + const before = { ...f.counters }; + const generation = f.referents.generation(); + if (end === 'stop') f.facade.stop(); + else { + f.instances[0].adapter.stop({ preserveStatus: true }); + f.instances[0].hooks.emit({ + type: 'state', + state: 'error', + detail: 'Fixture failure', + }); + } + assert.equal(f.pointer.activeSnapshot(), null); + assert.deepEqual(f.referents.list(), []); + assert.equal(f.referents.generation(), generation + 1); + for (const key of ['pointer', 'referents', 'conversation']) + assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`); + } + }); +} + +test('outer destruction disposes the runner once and removes each provider binding', async (t) => { + const priorUnsaved = describeUnsavedWork(); + const f = compositionFixture(t); + await f.facade.start(); + f.select('gemini'); + await settle(); + assert.equal(f.annotations.size, 1); + assert.ok( + describeUnsavedWork().includes(translate('the active voice session')), + ); + const oldCardEvents = f.views[0].cardView.events.length; + for (const callback of f.annotations) callback({ id: 'fixture-outline' }); + assert.equal(f.views[0].cardView.outlines.length, 0); + assert.equal(f.views[1].cardView.outlines.length, 1); + assert.equal(f.instances[1].mapEvents.length, 1); + f.facade.destroy(); + f.facade.destroy(); + f.facade.stop({ removeUi: true }); + f.lifetime.abort(); + assert.equal(f.counters.dispose, 1); + assert.equal(f.facade.signal.aborted, true); + assert.equal(f.facade.disposed, true); + assert.equal(f.annotations.size, 0); + assert.deepEqual(describeUnsavedWork(), priorUnsaved); + const finalEvents = f.events.length; + const starts = f.instances.map((instance) => instance.starts.length); + for (const [index, view] of f.views.entries()) { + assert.equal(view.root.removeCount, 1); + assert.equal(view.cardView.destroyed, 1); + assert.equal(view.reticle.destroyed, 1); + assert.equal(view.button.listenerCount('click'), 0); + assert.equal(view.providerSelect.listenerCount('change'), 0); + assert.equal(f.instances[index].hooks.signal.aborted, true); + view.button.dispatchEvent(new Event('click')); + view.providerSelect.value = 'openai'; + view.providerSelect.dispatchEvent(new Event('change')); + f.instances[index].hooks.emit({ type: 'state', state: 'connecting' }); + f.mounts[index].callback({ type: 'state', state: 'connecting' }); + } + assert.equal(f.events.length, finalEvents); + assert.equal(f.views[0].cardView.events.length, oldCardEvents); + assert.deepEqual( + f.instances.map((instance) => instance.starts.length), + starts, + ); + assert.equal(f.instances.length, 2); + assert.equal(f.facade.start(), false); + assert.equal(f.facade.sendText('after destroy'), false); + assert.equal(f.facade.notifyMapEvent({}), false); + const result = await f.facade.runner('adjust_camera_zoom', { + direction: 'in', + amount: 'little', + }); + assert.equal(result.cancelled, true); + assert.equal( + f.state.zooms, + 0, + 'the actual runner cannot move the camera after outer disposal', + ); +}); + +for (const action of ['stop', 'destroy']) { + for (const phase of ['common', 'adapter']) { + test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => { + let intercepted = 0; + const f = compositionFixture(t, { + onSessionEvent(event, facade) { + if ( + facade?.provider !== 'gemini' || + event.type !== 'state' || + event.state !== 'connecting' + ) + return; + const matching = + phase === 'common' + ? event.detail === undefined + : event.detail === 'adapter:gemini'; + if (matching && intercepted++ === 0) facade[action](); + }, + }); + await f.facade.start({ pushToTalk: true }); + f.select('gemini'); + await settle(); + const replacement = f.instances[1]; + assert.equal(intercepted, 1); + assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1); + assert.equal(replacement.acquired, 0); + assert.equal(replacement.microphoneEnabled, false); + assert.equal(f.facade.isActive(), false); + assert.equal(f.facade.state, 'idle'); + assert.equal( + f.events.filter( + (event) => event.state === 'listening' && event.detail === 'gemini', + ).length, + 0, + ); + assert.equal(f.facade.disposed, action === 'destroy'); + assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0); + if (action === 'stop') { + f.select('openai'); + assert.deepEqual( + f.instances[2].starts, + [], + 'Stop cleared the intent to resume', + ); + await f.facade.start({ pushToTalk: false }); + assert.equal(f.facade.state, 'listening'); + } else { + f.select('openai'); + assert.equal(f.instances.length, 2); + assert.equal(f.facade.start(), false); + } + }); + } +} diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index 86586ea01..06c504b20 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js'; import { createNarrationScheduler } from './narration.js'; import { createEarcon } from './earcon.js'; import { + POINTER_NONE_KEY, POINTER_OPEN_MIC_RECENT_MS, - pointerChip, - pointerContextItem, + pointerAnnouncement, + pointerTurnContext, + pointerTurnEvent, } from './pointerContext.js'; -/** Dedupe key of the clearing pointer_context item. */ -const POINTER_NONE_KEY = 'none'; - import { shouldPauseRadioForVoice } from './realtimeProtocol.js'; import { postDebugLog } from './realtimeDiagnostics.js'; @@ -287,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade { } catch { snapshot = null; } - const chip = pointerChip(snapshot); - this.pointerAnnounced = gesture && chip ? snapshot : null; - this.emitSessionEvent({ - type: 'pointer', - pointer: - gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null, + const event = pointerTurnEvent(snapshot, gesture); + this.pointerAnnounced = event.announced; + this.emitSessionEvent({ type: 'pointer', pointer: event.pointer }); + const context = pointerTurnContext(snapshot, { + at, + gesture, + sentKey: this.pointerSentKey, }); - const item = !chip - ? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY - ? { type: 'pointer_context', at, target: 'none' } - : null - : gesture - ? pointerContextItem(snapshot) - : // What kind of thing, never which: no label or coordinates. - { - type: 'pointer_context', - at, - pointing: snapshot.entity?.kind || 'ground', - }; - if (!item) return snapshot; - const { at: _at, ...content } = item; - const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY; - if (key === this.pointerSentKey) return snapshot; - if (this._turns.notifyMapEvent(item)) { - this.pointerSentKey = key; + if (!context) return snapshot; + if (this._turns.notifyMapEvent(context.item)) { + this.pointerSentKey = context.key; + const chip = context.key !== POINTER_NONE_KEY; this.debugLog('pointer.context', { at, target: chip ? snapshot.target : 'none', @@ -327,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade { */ announcePointer() { const snapshot = this.pointer?.activeSnapshot?.() || null; - const chip = pointerChip(snapshot); - if (!chip || snapshot === this.pointerAnnounced) return false; + const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced); + if (!pointer) return false; this.pointerAnnounced = snapshot; - this.emitSessionEvent({ - type: 'pointer', - pointer: { ...chip, screenPx: snapshot.screenPx }, - }); + this.emitSessionEvent({ type: 'pointer', pointer }); return true; } diff --git a/src/voice/realtimeRadio.js b/src/voice/realtimeRadio.js index b4cd1dfa6..4c5212d6e 100644 --- a/src/voice/realtimeRadio.js +++ b/src/voice/realtimeRadio.js @@ -247,7 +247,9 @@ export class RealtimeRadio { handoffChannel?.readyState === 'open', }, ); - const stillCurrent = handoffEpoch === this.radioHandoffEpoch; + const stillCurrent = + handoffEpoch === this.radioHandoffEpoch && + this.radioHandoffAttemptId === handoffAttemptId; if (this.radioHandoffAttemptId === handoffAttemptId) { this.radioHandoffInFlight = false; this.radioHandoffAttemptId = null; @@ -266,6 +268,25 @@ export class RealtimeRadio { } } + /** Revoke one call's prepared/preflight result, preserving other owners. */ + cancelPlaybackResult(result) { + if (!result) return; + if (this.pendingRadioPlaybackResult === result) { + this.pendingRadioPlaybackResult = null; + this.radioHandoffDeferredByReservation = false; + } + if (this.radioHandoffInFlightResult !== result) return; + const attemptId = this.radioHandoffAttemptId; + const shouldStopPlayback = this.radioHandoffInFlight; + this.radioHandoffInFlightResult = null; + this.radioHandoffInFlight = false; + this.radioHandoffAttemptId = null; + // Keep the group epoch and sibling reservations; release ownership before + // Stop can synchronously notify playback observers. + if (shouldStopPlayback) + this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId }); + } + /** Invalidate delayed Radio work inside the requested authority scope. */ abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) { for (const [controller, metadata] of this.activeRadioToolControllers) { diff --git a/src/voice/realtimeRadio.test.mjs b/src/voice/realtimeRadio.test.mjs new file mode 100644 index 000000000..4e29325ee --- /dev/null +++ b/src/voice/realtimeRadio.test.mjs @@ -0,0 +1,120 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { GevRealtimeController } from './realtimeController.js'; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const playback = (id) => ({ + ok: true, + action: 'control_radio', + radioAction: 'play', + radioPlaybackRequested: true, + id, +}); + +function fixture(t) { + const attempts = [], + stopped = [], + corrections = [], + states = []; + const controller = new GevRealtimeController({ + ui: { + root: { + dataset: {}, + classList: { remove() {} }, + querySelectorAll: () => [], + remove() {}, + }, + status: {}, + detail: {}, + errorDetail: {}, + }, + runner: async () => ({ ok: false }), + debugSink: null, + onSessionEvent: (event) => states.push(event), + radioLayer: { + setVoiceDucked() {}, + playForVoice: ({ attemptId }) => + new Promise((resolve, reject) => + attempts.push({ attemptId, resolve, reject }), + ), + stopPlayback: (options) => stopped.push(options), + }, + }); + controller.queueResponseCreate = (message) => corrections.push(message); + controller.dc = { readyState: 'open', send() {}, close() {} }; + t.after(() => controller.stop({ removeUi: true })); + return { controller, attempts, stopped, corrections, states }; +} + +test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => { + const h = fixture(t), + c = h.controller; + const a = playback('A'); + c.pendingRadioPlaybackResult = a; + const pending = c.startPendingRadioHandoff(); + const reservation = c.reserveRadioToolHandoff(); + assert.equal(c.pendingRadioPlaybackResult, a); + assert.equal(c.radioHandoffInFlight, false); + h.attempts[0].reject(new Error('obsolete frozen preflight')); + await pending; + assert.deepEqual(h.corrections, []); + assert.equal( + h.states.some((event) => event.detail === 'Radio did not start'), + false, + ); + c.settleRadioToolHandoffReservation(reservation, { commit: false }); + await tick(); + assert.equal(h.attempts.length, 2); + assert.equal(c.radioHandoffInFlightResult, a); + assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId); + h.attempts[1].reject(new Error('current preflight failed')); + await tick(); + assert.equal(h.corrections.length, 1); + assert.equal(c.status, 'listening'); +}); + +test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => { + const h = fixture(t), + c = h.controller; + c.pendingRadioPlaybackResult = playback('A'); + const old = c.startPendingRadioHandoff(); + c.cancelRadioHandoff(); + h.attempts[0].reject(new Error('cancelled preflight')); + await old; + assert.deepEqual(h.corrections, []); + c.pendingRadioPlaybackResult = playback('B'); + const current = c.startPendingRadioHandoff(); + h.attempts[1].resolve(false); + await current; + assert.equal(h.corrections.length, 1); +}); + +test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => { + const h = fixture(t), + c = h.controller; + const a = playback('A'), + b = playback('B'); + c.pendingRadioPlaybackResult = a; + const old = c.startPendingRadioHandoff(); + const siblingEpoch = c.radioHandoffEpoch; + c.pendingRadioPlaybackResult = b; + c._radio.cancelPlaybackResult(a); + c._radio.cancelPlaybackResult(a); + c._radio.cancelPlaybackResult(playback('unknown')); + assert.equal(c.radioHandoffEpoch, siblingEpoch); + assert.equal(c.pendingRadioPlaybackResult, b); + assert.equal(h.stopped.length, 1); + h.attempts[0].reject(new Error('late cancelled preflight')); + await old; + assert.deepEqual(h.corrections, []); + const current = c.startPendingRadioHandoff(); + const reservation = c.reserveRadioToolHandoff(); + c._radio.cancelPlaybackResult(b); + assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true); + h.attempts[1].resolve(false); + await current; + assert.deepEqual(h.corrections, []); + c.settleRadioToolHandoffReservation(reservation, { commit: false }); + await tick(); + assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed'); +}); diff --git a/src/voice/realtimeSession.js b/src/voice/realtimeSession.js index 0220961f8..0d9eca872 100644 --- a/src/voice/realtimeSession.js +++ b/src/voice/realtimeSession.js @@ -16,6 +16,7 @@ export function createRealtimeSession({ controller, capabilities: { costControls: true, pushToTalk: true }, start: (settings) => controller.start(settings), + getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }), stop: (settings) => controller.stop(settings), sendText: (text) => controller.sendTextCommand(text), sendMapEvent: (event) => controller.notifyMapEvent(event), diff --git a/src/voice/realtimeTurns.js b/src/voice/realtimeTurns.js index 6905cc17e..cbc8d5dbc 100644 --- a/src/voice/realtimeTurns.js +++ b/src/voice/realtimeTurns.js @@ -137,7 +137,7 @@ export class RealtimeTurns { content: [{ type: 'input_text', text: cleanText }], }, }; - this.sendRealtimeEvent(itemEvent, 'client.user_text'); + if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return; this.requestUserTextResponse(); } diff --git a/src/voice/realtimeTypedSend.test.mjs b/src/voice/realtimeTypedSend.test.mjs new file mode 100644 index 000000000..6a32172c0 --- /dev/null +++ b/src/voice/realtimeTypedSend.test.mjs @@ -0,0 +1,268 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { GevRealtimeController } from './realtimeController.js'; + +function fixture({ pointer = null, onUserSend = () => {} } = {}) { + const sent = []; + const executed = []; + const rejectedTexts = new Set(); + let rejectResponse = false; + const controller = new GevRealtimeController({ + runner: async (name) => { + executed.push(name); + return { ok: true }; + }, + backend: {}, + debugSink: null, + pointer, + ui: { + root: { + dataset: {}, + classList: { remove() {} }, + querySelectorAll: () => [], + }, + status: {}, + detail: {}, + }, + }); + controller.dc = { + readyState: 'open', + send(body) { + const value = JSON.parse(body); + const text = value.item?.role === 'user' && value.item.content?.[0]?.text; + if (value.item?.role === 'user') onUserSend(); + if ( + rejectedTexts.has(text) || + (rejectResponse && value.type === 'response.create') + ) + throw new Error('Synthetic channel rejection'); + sent.push(value); + }, + close() { + this.readyState = 'closed'; + }, + }; + controller.status = 'listening'; + controller._turns.narration = null; + const event = (payload) => + controller.handleRealtimeEvent({ data: JSON.stringify(payload) }); + const created = (id) => event({ type: 'response.created', response: { id } }); + const done = (id) => + event({ + type: 'response.done', + response: { id, status: 'completed', output: [] }, + }); + const creates = () => + sent.filter( + (x) => + x.type === 'response.create' && + x.response?.metadata?.gev !== 'progress', + ); + const texts = () => + sent + .filter((x) => x.item?.role === 'user') + .map((x) => x.item.content[0].text); + return { + controller, + sent, + executed, + rejectedTexts, + event, + created, + done, + creates, + texts, + rejectResponses(value) { + rejectResponse = value; + }, + }; +} + +test('rejected idle text schedules no answer, and a successful successor answers once', async () => { + const f = fixture(); + try { + f.rejectedTexts.add('rejected'); + assert.equal(f.controller.sendTextCommand('rejected'), undefined); + assert.equal(f.creates().length, 0); + assert.equal(f.controller.pendingUserTextResponse, false); + assert.equal(f.controller.responseCreatePending, false); + assert.equal(f.controller.sendTextCommand('successor'), undefined); + assert.deepEqual(f.texts(), ['successor']); + assert.equal(f.creates().length, 1); + await f.created('successor'); + await f.done('successor'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('rejected text behind an active response cannot queue an answer', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.queueResponseCreate('Old tool follow-up'); + assert.equal( + f.controller.pendingResponseInstructions, + 'Old tool follow-up', + ); + const runningTool = new AbortController(); + f.controller._turns.activeToolAbortControllers.add(runningTool); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.pendingUserTextResponse, false); + assert.equal(f.controller.pendingResponseInstructions, null); + assert.equal(f.controller._turns.isSupersededResponse('active'), true); + assert.equal(runningTool.signal.aborted, true); + const staleCall = { + type: 'response.function_call_arguments.done', + response_id: 'active', + call_id: 'stale-call', + name: 'fly_to_location', + arguments: '{"query":"Paris"}', + }; + await f.event(staleCall); + await f.event(staleCall); + assert.deepEqual(f.executed, []); + const outputs = f.sent.filter( + (x) => x.item?.type === 'function_call_output', + ); + assert.equal(outputs.length, 1); + assert.equal(outputs[0].item.call_id, 'stale-call'); + assert.equal(JSON.parse(outputs[0].item.output).superseded, true); + await f.done('active'); + assert.equal(f.creates().length, 0); + } finally { + f.controller.stop(); + } +}); + +test('rejected text still prepares narration and pointer context before sending', () => { + const steps = []; + const f = fixture({ + pointer: { + beginTurn() { + steps.push('pointer'); + return null; + }, + clear() {}, + }, + onUserSend() { + steps.push('send'); + }, + }); + f.controller._turns.narration = { + cancel() { + steps.push('cancel'); + }, + userTurnEnded() { + steps.push('user-turn-ended'); + }, + }; + try { + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']); + assert.equal(f.creates().length, 0); + } finally { + f.controller.stop(); + } +}); + +test('earlier accepted deferred input survives a newer failed send and answers once', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.deepEqual(f.texts(), ['accepted']); + assert.equal(f.controller.pendingUserTextResponse, true); + assert.equal(f.creates().length, 0); + await f.done('active'); + assert.equal(f.creates().length, 1); + await f.done('active'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('rejected text preserves an outstanding create without adding deferred input', async () => { + const f = fixture(); + try { + f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.responseCreatePending, true); + assert.equal(f.controller.pendingUserTextResponse, false); + await f.created('accepted'); + await f.done('accepted'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +for (const acceptedFirst of [false, true]) { + test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => { + const f = fixture(); + try { + assert.equal( + f.controller._turns.speakProgressLine('Checking the map.'), + true, + ); + await f.event({ + type: 'response.created', + response: { id: 'progress', metadata: { gev: 'progress' } }, + }); + if (acceptedFirst) f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.pendingUserTextResponse, acceptedFirst); + await f.event({ + type: 'response.done', + response: { + id: 'progress', + status: 'cancelled', + metadata: { gev: 'progress' }, + }, + }); + assert.equal(f.creates().length, acceptedFirst ? 1 : 0); + } finally { + f.controller.stop(); + } + }); +} + +test('accepted text with a rejected response.create releases the pending latch for a successor', async () => { + const f = fixture(); + try { + f.rejectResponses(true); + f.controller.sendTextCommand('accepted without response'); + assert.equal(f.controller.responseCreatePending, false); + assert.equal(f.creates().length, 0); + f.rejectResponses(false); + f.controller.sendTextCommand('successor'); + assert.deepEqual(f.texts(), ['accepted without response', 'successor']); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('successful deferred typed burst still retains both items and coalesces one answer', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.sendTextCommand('first'); + f.controller.sendTextCommand('second'); + assert.deepEqual(f.texts(), ['first', 'second']); + assert.equal(f.creates().length, 0); + await f.done('active'); + await f.done('active'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); diff --git a/src/voice/session.js b/src/voice/session.js index f55b7657e..21400f04c 100644 --- a/src/voice/session.js +++ b/src/voice/session.js @@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) { }, async start(options) { if (disposed || !['idle', 'error'].includes(state)) return; - emit({ type: 'state', state: 'connecting' }); const attempt = ++startAttempt; + emit({ type: 'state', state: 'connecting' }); + if (disposed || attempt !== startAttempt) return; try { await adapter.start(options); } catch (error) { @@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) { stop(options = {}) { if (options.removeUi) return dispose(); if (disposed) return; - startAttempt++; + const attempt = ++startAttempt; cancelActions(); try { adapter.stop(options); } finally { - if (!options.preserveStatus) + if ( + !options.preserveStatus && + !disposed && + attempt === startAttempt && + state !== 'idle' + ) emit({ type: 'state', state: 'idle', detail: 'Voice off' }); } }, diff --git a/src/voice/session.test.mjs b/src/voice/session.test.mjs index 9bd016f20..8e4d37385 100644 --- a/src/voice/session.test.mjs +++ b/src/voice/session.test.mjs @@ -6,6 +6,7 @@ import { voiceCardView, } from './voiceCardPresentation.js'; import { createVoiceSession } from './session.js'; +import { createRealtimeSession } from './realtimeSession.js'; import { realtimeSessionEvent } from './realtimeEvents.js'; import { createVoiceCommands } from './sessionCommands.js'; import { describeUnsavedWork } from '../i18n/index.js'; @@ -441,3 +442,189 @@ test('a late rejection cannot settle a newer turn card', async () => { ); f.session.destroy(); }); + +function startReentrancyFixture() { + let starts = 0; + let stops = 0; + const settings = []; + const session = createVoiceSession({ + runner: async () => ({ ok: true }), + createAdapter: ({ emit }) => ({ + async start(options) { + starts++; + settings.push(options); + emit({ type: 'state', state: 'listening' }); + }, + stop() { + stops++; + }, + sendText() {}, + sendMapEvent() {}, + }), + }); + return { session, settings, starts: () => starts, stops: () => stops }; +} + +for (const action of ['stop', 'destroy']) { + test(`initial connecting observer ${action} prevents adapter acquisition`, async () => { + const f = startReentrancyFixture(); + const unsubscribe = f.session.subscribe((event) => { + if (event.type === 'state' && event.state === 'connecting') { + unsubscribe(); + f.session[action](); + } + }); + await f.session.start(); + assert.equal( + f.starts(), + 0, + 'cancelled attempt must not call adapter.start', + ); + assert.equal(f.stops(), 1); + assert.equal(f.session.state, 'idle'); + if (action === 'stop') { + await f.session.start(); + assert.equal(f.starts(), 1, 'later explicit Start remains usable'); + assert.equal(f.session.state, 'listening'); + } + f.session.destroy(); + }); +} + +test('preserveStatus Stop still prevents adapter acquisition', async () => { + const f = startReentrancyFixture(); + const unsubscribe = f.session.subscribe((event) => { + if (event.type === 'state' && event.state === 'connecting') { + unsubscribe(); + f.session.stop({ preserveStatus: true }); + } + }); + await f.session.start(); + assert.equal(f.starts(), 0); + assert.equal( + f.session.state, + 'connecting', + 'preserved state is not ownership', + ); + f.session.stop(); + await f.session.start(); + assert.equal(f.starts(), 1); + f.session.destroy(); +}); + +test('Stop and explicit reentrant successor start acquire only for the successor', async () => { + const f = startReentrancyFixture(); + let successor; + const unsubscribe = f.session.subscribe((event) => { + if (event.type === 'state' && event.state === 'connecting') { + unsubscribe(); + f.session.stop(); + successor = f.session.start(); + } + }); + await f.session.start(); + await successor; + assert.equal(f.starts(), 1); + assert.equal(f.session.state, 'listening'); + f.session.destroy(); +}); + +test('ordinary start forwards caller settings once and remains stoppable', async () => { + const f = startReentrancyFixture(); + const settings = { pushToTalk: true }; + await f.session.start(settings); + assert.equal(f.settings.length, 1); + assert.equal(f.settings[0], settings); + assert.equal(f.starts(), 1); + assert.equal(f.session.state, 'listening'); + f.session.stop(); + assert.equal(f.session.state, 'idle'); + f.session.destroy(); +}); + +for (const emitsIdle of [false, true]) { + for (const outcome of ['normal', 'preserveStatus', 'throw']) { + test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => { + let adapter; + const calls = []; + const events = []; + const failure = new Error('Controlled stop failure'); + const session = createVoiceSession({ + runner: async () => ({}), + createAdapter: ({ emit }) => + (adapter = { + async start() { + emit({ type: 'state', state: 'listening' }); + }, + stop(settings = {}) { + calls.push({ receiver: this, settings }); + if (settings.removeUi) return; + if (emitsIdle && !settings.preserveStatus) + emit({ type: 'state', state: 'idle' }); + if (outcome === 'throw') throw failure; + return 'adapter-only return'; + }, + sendText() {}, + sendMapEvent() {}, + }), + }); + t.after(() => session.destroy()); + session.subscribe((event) => events.push(event)); + await session.start(); + events.length = 0; + const settings = + outcome === 'preserveStatus' ? { preserveStatus: true } : {}; + if (outcome === 'throw') + assert.throws( + () => session.stop(settings), + (error) => error === failure, + ); + else assert.equal(session.stop(settings), undefined); + assert.equal(calls[0].receiver, adapter); + assert.equal(calls[0].settings, settings); + assert.equal( + session.state, + outcome === 'preserveStatus' ? 'listening' : 'idle', + ); + assert.equal( + events.filter( + (event) => event.type === 'state' && event.state === 'idle', + ).length, + outcome === 'preserveStatus' ? 0 : 1, + ); + }); + } +} + +test('actual default OpenAI adapter stop publishes one common idle', (t) => { + const events = []; + const ui = { + root: { + dataset: {}, + querySelectorAll: () => [], + classList: { remove() {} }, + remove() {}, + }, + status: {}, + detail: {}, + }; + const runner = async () => ({}); + const session = createVoiceSession({ + runner, + createAdapter: (hooks) => + createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }), + }); + t.after(() => session.destroy()); + session.subscribe((event) => events.push(event)); + // Exercise the real stop path without acquiring microphone/WebRTC resources. + session.adapter.controller.setStatus('listening', 'Controlled active state'); + events.length = 0; + session.stop(); + assert.equal( + events.filter((event) => event.type === 'state' && event.state === 'idle') + .length, + 1, + ); + assert.equal(session.state, 'idle'); + assert.equal(session.adapter.controller.status, 'idle'); +}); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index 05bbcc501..9431555df 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -2,6 +2,7 @@ import { createVoiceControl, voiceStatusLabel } from './control.js'; import { createVoiceSession } from './session.js'; import { VoiceCardControls } from './voiceCard.js'; import { PointerReticle } from './pointerReticle.js'; +import { VOICE_PROVIDERS } from './providerCapabilities.js'; import { msg, registerUnsavedWork, t } from '../i18n/index.js'; function createDefaultReticle(pointer) { @@ -25,12 +26,18 @@ export function createVoiceCommands({ signal, debugSink, createControl = createVoiceControl, + resetExisting = true, + disposeRunner = true, + provider, + supportedProviders, + onProviderChange, + onSessionEvent, createCard = (options) => new VoiceCardControls(options), pointer = null, referents = null, createReticle = createDefaultReticle, }) { - window.__gevVoiceCommands?.stop?.({ removeUi: true }); + if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true }); const ui = createControl({ reset: true }); const session = createVoiceSession({ runner, @@ -50,6 +57,34 @@ export function createVoiceCommands({ }); const adapter = session.adapter; const capabilities = adapter.capabilities || {}; + const providerHandler = () => onProviderChange?.(ui.providerSelect.value); + if (ui.providerSelect && onProviderChange) { + if (supportedProviders && ui.providerSelect.replaceChildren) { + const doc = ui.providerSelect.ownerDocument; + ui.providerSelect.replaceChildren( + ...supportedProviders.map((name) => { + const option = doc.createElement('option'); + option.value = name; + option.textContent = VOICE_PROVIDERS[name].label; + return option; + }), + ); + } + ui.providerSelect.value = provider; + ui.providerSelect.title = [ + t('Changing provider reconnects voice if it is on.'), + capabilities.advisoryLimit, + ] + .filter(Boolean) + .join(' '); + ui.providerSelect.addEventListener('change', providerHandler); + ui.root.dataset.provider = provider; + } + const hasProviderChoice = Boolean( + onProviderChange && supportedProviders?.length > 1, + ); + if (ui.providerField) ui.providerField.hidden = !hasProviderChoice; + ui.root.dataset.providerChoice = String(hasProviderChoice); if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls; if (ui.costValue) ui.costValue.hidden = !capabilities.costControls; if (!capabilities.pushToTalk) { @@ -60,6 +95,7 @@ export function createVoiceCommands({ // Retain the existing controller's inspection surface for browser tools. const controls = adapter.controller || session; controls.session = session; + controls.ui = ui; const updateStatus = session.subscribe((event) => { if (event.type !== 'state') return; ui.root.dataset.status = event.state; @@ -101,6 +137,10 @@ export function createVoiceCommands({ }); controls.pointer = pointer; controls.referents = referents; + // Forward last: a host observer may synchronously replace or stop this mount. + const forwardEvents = onSessionEvent + ? session.subscribe(onSessionEvent) + : null; const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => { card?.outline(event); session.sendMapEvent({ type: 'map_annotation_outline', ...event }); @@ -115,34 +155,26 @@ export function createVoiceCommands({ else void session.start({ pushToTalk: false }); }; ui.button.addEventListener('click', buttonHandler); - session.signal.addEventListener( - 'abort', - () => { - ui.button.removeEventListener('click', buttonHandler); - unregisterUnsavedWork(); - annotationUnsubscribe?.(); - updateStatus(); - updateCard?.(); - updatePointer(); - reticle?.destroy(); - card?.destroy(); - runner.dispose?.(); - ui.root.remove(); - }, - { once: true }, - ); - if (session.disposed) { + let cleaned = false; + const cleanup = () => { + if (cleaned) return; + cleaned = true; ui.button.removeEventListener('click', buttonHandler); + ui.providerSelect?.removeEventListener('change', providerHandler); unregisterUnsavedWork(); annotationUnsubscribe?.(); updateStatus(); updateCard?.(); updatePointer(); + forwardEvents?.(); reticle?.destroy(); card?.destroy(); - runner.dispose?.(); + if (disposeRunner) runner.dispose?.(); ui.root.remove(); - } else adapter.bindControls?.(); + }; + session.signal.addEventListener('abort', cleanup, { once: true }); + if (session.disposed) cleanup(); + else adapter.bindControls?.(); window.__gevVoiceCommands = controls; return controls; }