From b55663e3ce74156c31d243f1fe4352a0828627f1 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu Date: Mon, 28 Sep 2026 22:52:35 -0500 Subject: [PATCH] =?UTF-8?q?feat(voice):=20on-device=20voice=20tier=20?= =?UTF-8?q?=E2=80=94=20Gemma=204,=20Moonshine=20and=20Kokoro=20in=20the=20?= =?UTF-8?q?browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 5 + DATA_SOURCES.md | 14 + README.md | 2 + THIRD_PARTY_NOTICES.md | 25 +- build/onDeviceRuntime.js | 170 +++ build/vite.js | 24 +- package-lock.json | 1231 ++++++++++++++++- package.json | 3 + scripts/check-import-directions.mjs | 8 +- scripts/format-scope.json | 23 +- scripts/module-analysis.mjs | 13 +- scripts/package-boundaries.json | 54 +- server/providers/openai/tools.js | 2 +- server/standalone/vite.config.js | 1 + src/app/tools.js | 7 +- src/radioMarkup.test.mjs | 2 +- src/tooling/importDirections.test.mjs | 6 + src/tooling/viteBuild.test.mjs | 14 +- src/ui/styles/local-voice.css | 100 ++ src/ui/styles/voice-engine.css | 142 ++ .../voice/actionDescriptions.js | 5 +- src/voice/gevRealtime.js | 44 +- src/voice/local/gemmaToolCalls.js | 230 +++ src/voice/local/localAudio.js | 228 +++ src/voice/local/localEngine.js | 368 +++++ src/voice/local/localPrompt.js | 22 + src/voice/local/localRadio.js | 64 + src/voice/local/localRadio.test.mjs | 148 ++ src/voice/local/localRuntime.test.mjs | 106 ++ src/voice/local/localStreaming.test.mjs | 43 + src/voice/local/localVoice.test.mjs | 424 ++++++ src/voice/local/localVoiceControls.js | 166 +++ src/voice/local/localVoicePreferences.js | 117 ++ src/voice/local/localVoicePresentation.js | 139 ++ src/voice/local/localVoiceStore.js | 58 + src/voice/local/localVoiceUi.test.mjs | 418 ++++++ src/voice/local/localWebFlag.js | 10 + src/voice/local/localWebProvider.js | 71 + src/voice/local/localWebProvider.test.mjs | 144 ++ src/voice/local/localWebSession.js | 772 +++++++++++ src/voice/local/localWorkflow.test.mjs | 245 ++++ src/voice/local/modelCache.test.mjs | 217 +++ src/voice/local/modelCatalog.js | 215 +++ src/voice/local/naturalVoice.test.mjs | 224 +++ src/voice/local/naturalVoiceSetting.js | 17 + src/voice/local/onDeviceRuntime.test.mjs | 72 + src/voice/local/replies.js | 323 +++++ src/voice/local/sha256.js | 120 ++ src/voice/local/speechOutput.js | 145 ++ src/voice/local/speechText.js | 122 ++ src/voice/local/streamTee.js | 105 ++ src/voice/local/toolset.js | 546 ++++++++ src/voice/local/workerClient.js | 73 + src/voice/local/workers/llm.worker.js | 360 +++++ src/voice/local/workers/modelCache.js | 180 +++ src/voice/local/workers/phonemizerRuntime.js | 119 ++ src/voice/local/workers/runtimeAssets.js | 17 + src/voice/local/workers/stt.worker.js | 78 ++ src/voice/local/workers/tts.worker.js | 106 ++ src/voice/sessionCommands.js | 18 + src/voice/voiceEngine.js | 50 + src/voice/voiceEngine.test.mjs | 209 +++ src/voice/voiceEngineControls.js | 185 +++ src/voice/voiceEnginePresentation.js | 145 ++ style.css | 1 + 65 files changed, 9257 insertions(+), 58 deletions(-) create mode 100644 build/onDeviceRuntime.js create mode 100644 src/ui/styles/local-voice.css create mode 100644 src/ui/styles/voice-engine.css rename server/providers/openai/toolDescriptions.js => src/voice/actionDescriptions.js (99%) create mode 100644 src/voice/local/gemmaToolCalls.js create mode 100644 src/voice/local/localAudio.js create mode 100644 src/voice/local/localEngine.js create mode 100644 src/voice/local/localPrompt.js create mode 100644 src/voice/local/localRadio.js create mode 100644 src/voice/local/localRadio.test.mjs create mode 100644 src/voice/local/localRuntime.test.mjs create mode 100644 src/voice/local/localStreaming.test.mjs create mode 100644 src/voice/local/localVoice.test.mjs create mode 100644 src/voice/local/localVoiceControls.js create mode 100644 src/voice/local/localVoicePreferences.js create mode 100644 src/voice/local/localVoicePresentation.js create mode 100644 src/voice/local/localVoiceStore.js create mode 100644 src/voice/local/localVoiceUi.test.mjs create mode 100644 src/voice/local/localWebFlag.js create mode 100644 src/voice/local/localWebProvider.js create mode 100644 src/voice/local/localWebProvider.test.mjs create mode 100644 src/voice/local/localWebSession.js create mode 100644 src/voice/local/localWorkflow.test.mjs create mode 100644 src/voice/local/modelCache.test.mjs create mode 100644 src/voice/local/modelCatalog.js create mode 100644 src/voice/local/naturalVoice.test.mjs create mode 100644 src/voice/local/naturalVoiceSetting.js create mode 100644 src/voice/local/onDeviceRuntime.test.mjs create mode 100644 src/voice/local/replies.js create mode 100644 src/voice/local/sha256.js create mode 100644 src/voice/local/speechOutput.js create mode 100644 src/voice/local/speechText.js create mode 100644 src/voice/local/streamTee.js create mode 100644 src/voice/local/toolset.js create mode 100644 src/voice/local/workerClient.js create mode 100644 src/voice/local/workers/llm.worker.js create mode 100644 src/voice/local/workers/modelCache.js create mode 100644 src/voice/local/workers/phonemizerRuntime.js create mode 100644 src/voice/local/workers/runtimeAssets.js create mode 100644 src/voice/local/workers/stt.worker.js create mode 100644 src/voice/local/workers/tts.worker.js create mode 100644 src/voice/voiceEngine.js create mode 100644 src/voice/voiceEngine.test.mjs create mode 100644 src/voice/voiceEngineControls.js create mode 100644 src/voice/voiceEnginePresentation.js diff --git a/.env.example b/.env.example index a28f375d4..ce8c09ffd 100644 --- a/.env.example +++ b/.env.example @@ -72,6 +72,11 @@ OPENAI_REALTIME_TRANSCRIBE_MODEL=gpt-4o-mini-transcribe # Set to 1 to keep them while debugging; delete the log when you are done. GEV_VOICE_LOG_CONTENT= OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano +# On-device voice (Cloud / On-device choice in the mic control) runs models +# in the browser. Its natural voice downloads an open-source speech component +# (GPL) to the browser at runtime; set to off to leave natural voice out and +# speak with the browser's on-device voices (or text) instead. +# GEV_NATURAL_VOICE=off # Optional: opt-in per-IP rate limit for the OpenAI cost endpoints # (/api/realtime/token + /api/openai/hud-summary), in requests per minute. # DEFAULT IS UNLIMITED — unset (or 0) means no throttling, unchanged behavior. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ab059cebb..2c55ba565 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -100,6 +100,20 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Cockpit regional briefing.** `/api/regional-brief` rounds aircraft coordinates into 0.1° cache cells, caches results for five minutes, and resolves physical regions from bundled public-domain Natural Earth polygons without reverse-geocoding requests. Google News RSS is queried with the resolved locality/region first; GDELT is used only when that RSS query fails or is empty. Google's published Google News terms restrict that source to personal, noncommercial use, so commercial deployments must disable/replace it or obtain separate permission; GDELT permits commercial dataset use with citation. The Data attribution popover identifies the active headline sources; article links retain publisher attribution. Headlines are location-query matches, not verified incidents, risk rankings, or evidence that a location is safe. Empty, partial, stale, and unavailable source states remain distinct. Open-Meteo supplies current conditions independently of the news source. `WX OFF` disables cockpit weather rendering only; the Local Info briefing still fetches its source-backed weather values and displays the required linked Open-Meteo credit. - **Dynamic weather presentation.** While cockpit mode is active, `/api/weather-effects` requests current Open-Meteo observations for the aircraft/camera location, rounds coordinates into 0.1° cache cells, caches results for five minutes, and may retain a stale observation for up to 30 minutes during a transient outage. WMO condition code selects the visual family; observed cloud cover, precipitation, visibility, wind speed, and wind direction bound its strength and motion. Missing or expired weather renders no synthetic atmospheric effect, and normal globe view never renders the weather overlay. + +### On-device voice models (downloaded only when voice runs on this device) + +Downloaded by the browser from Hugging Face at pinned revisions and kept in its cache; not stored in this repo. + +| Model | Revision | License | +| --- | --- | --- | +| Gemma 4 E2B / E4B / 12B / 26B A4B web builds (`litert-community/gemma-4-*-it-litert-lm`) | per model in `src/voice/local/modelCatalog.js`, SHA-256 checked | Apache-2.0 | +| Moonshine Base (`onnx-community/moonshine-base-ONNX`) | `b1e9b6aa` | MIT | +| Whisper base.en / large-v3-turbo ONNX (`onnx-community/whisper-base.en`, `onnx-community/whisper-large-v3-turbo`) | `51eefc0a` / `360ebcde` | MIT | +| Kokoro-82M and voice vectors (`onnx-community/Kokoro-82M-v1.0-ONNX`) | `1939ad2a` | Apache-2.0 | + +Natural voice's phonemizer is fetched the same way, from the public npm CDN (see [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md)). + --- ## Bundled snapshots diff --git a/README.md b/README.md index f6a3ae886..171cab50c 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,8 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain — an > Voice needs an **OpenAI key**. Without one the entire app still runs — the mic button just reports voice is unavailable. The same key drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +> **No key? Run voice on this device.** Open the tray next to the mic and choose **ON-DEVICE**: Gemma 4, Moonshine and Kokoro run in the browser (WebGPU; current Chrome or Edge) after a one-time download, with no key and no per-use cost. The tray checks your hardware: 16 GB+ Apple Silicon works best with Gemma 4 12B (6 GB download); 8–16 GB with Gemma 4 E4B (3 GB). `?voice=local-web` is a shortcut. + Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: - **🧠 It knows what it's looking at.** The agent pulls live scene context before answering — including coordinates, street names, active layers, and view scale. Ask _"what city is this?"_ mid-flight and it knows. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 071299fba..ef094a8d1 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,8 +1,9 @@ # Third-party software notices The project depends on third-party npm packages whose own licenses apply. This -file records the packages added for the browser-local SDR feature; the complete -resolved dependency inventory remains in `package-lock.json`. +file records the packages added for the browser-local SDR and on-device voice +features; the complete resolved dependency inventory remains in +`package-lock.json`. ## Web RTL-SDR @@ -19,3 +20,23 @@ resolved dependency inventory remains in `package-lock.json`. - Source: - License: Apache License 2.0 - License text: + +## On-device voice + +Loaded only when voice runs on this device. + +| Package | Version | Source | License | +| --- | --- | --- | --- | +| `@litert-lm/core` (runtime and wasm served from the build) | 0.17.1 | | Apache-2.0 | +| `@huggingface/transformers` (speech recognition) | 4.3.0 | | Apache-2.0 | +| `onnxruntime-web` (with transformers.js 4.3.0) | 1.31.0-dev.20260914 | | MIT | +| `kokoro-js` (natural voice) | 1.2.1 | | Apache-2.0 | +| `@huggingface/transformers` (used by kokoro-js) | 3.8.1 | | Apache-2.0 | +| `onnxruntime-web` (used by kokoro-js) | 1.22.0-dev.20250409 | | MIT | + +Natural voice also uses `phonemizer` 1.2.1 (), +which embeds eSpeak NG (, GPL-3.0-or-later). +It is not part of this repository or its build: when natural voice first +starts, the user's browser downloads the pinned file from +`cdn.jsdelivr.net/npm/phonemizer@1.2.1` and checks its SHA-256 before use. +`GEV_NATURAL_VOICE=off` turns natural voice off. diff --git a/build/onDeviceRuntime.js b/build/onDeviceRuntime.js new file mode 100644 index 000000000..4b192f9ea --- /dev/null +++ b/build/onDeviceRuntime.js @@ -0,0 +1,170 @@ +import { createRequire } from 'node:module'; +import path from 'node:path'; + +// Two of the LiteRT-LM package's four builds: relaxed SIMD with JavaScript +// promise integration (current Chrome and Edge) and the most compatible one. +const LITERT_VARIANTS = { + internal: 'litertlm_wasm_internal', + compatAsyncify: 'litertlm_wasm_compat_asyncify_internal', +}; +const ASSETS_MODULE = path.join( + 'src', + 'voice', + 'local', + 'workers', + 'runtimeAssets.js', +); +// kokoro-js imports `phonemizer`, which embeds eSpeak NG. The build never +// includes it: the import resolves to this loader, which downloads the pinned +// file from the public npm CDN at runtime and checks its SHA-256. +const PHONEMIZER_MODULE = path.join( + 'src', + 'voice', + 'local', + 'workers', + 'phonemizerRuntime.js', +); + +/** Strings that only appear in the eSpeak NG phonemizer build. */ +export const ESPEAK_MARKERS = Object.freeze([ + 'eSpeakNGWorker', + 'espeak-ng-data', + 'espeakEVENT', +]); + +/** Absolute path of the runtime phonemizer loader for a checkout. */ +export function phonemizerLoaderPath(root = process.cwd()) { + return path.join(root, PHONEMIZER_MODULE); +} + +/** Names of emitted chunks or assets that contain eSpeak NG code. */ +export function findEspeakChunks(bundle) { + const found = []; + for (const [name, output] of Object.entries(bundle || {})) { + const text = + typeof output.code === 'string' + ? output.code + : typeof output.source === 'string' + ? output.source + : output.source instanceof Uint8Array + ? Buffer.from(output.source).toString('latin1') + : ''; + if (ESPEAK_MARKERS.some((marker) => text.includes(marker))) + found.push(name); + } + return found; +} + +const posix = (file) => file.split(path.sep).join('/'); + +/** Directory of the installed package that `from` resolves `name` to. */ +function packageDir(from, name) { + const entry = createRequire(from).resolve(name); + const marker = `${path.sep}node_modules${path.sep}${name.split('/').join(path.sep)}${path.sep}`; + const at = entry.lastIndexOf(marker); + if (at < 0) throw new Error(`Cannot locate ${name} from ${from}`); + return entry.slice(0, at + marker.length - 1); +} + +/** + * Serves the on-device voice runtimes from the application build instead of + * a CDN. It replaces src/voice/local/workers/runtimeAssets.js with imports + * of the installed LiteRT-LM Emscripten loaders (classic scripts, exported + * here as modules) and wasm files, and the ONNX Runtime Web files that the + * two transformers.js copies (speech recognition, Kokoro) use. + */ +export function onDeviceRuntimePlugin({ root = process.cwd() } = {}) { + let paths = null; + const resolvePaths = () => { + if (paths) return paths; + const rootFile = path.join(root, 'package.json'); + const require = createRequire(rootFile); + const litert = path.join(packageDir(rootFile, '@litert-lm/core'), 'wasm'); + const ort = path.join( + packageDir( + require.resolve('@huggingface/transformers'), + 'onnxruntime-web', + ), + 'dist', + ); + const kokoroTransformers = path.join( + packageDir(require.resolve('kokoro-js'), '@huggingface/transformers'), + 'dist', + ); + paths = { + assets: path.join(root, ASSETS_MODULE), + loaders: new Set( + Object.values(LITERT_VARIANTS).map((name) => + path.join(litert, `${name}.js`), + ), + ), + source() { + const lines = []; + const runtimes = []; + for (const [key, name] of Object.entries(LITERT_VARIANTS)) { + const file = posix(path.join(litert, name)); + lines.push( + `import ${key}Factory from ${JSON.stringify(`${file}.js`)};`, + ); + lines.push( + `import ${key}Wasm from ${JSON.stringify(`${file}.wasm?url`)};`, + ); + runtimes.push( + `${key}: { ModuleFactory: ${key}Factory, wasmUrl: ${key}Wasm }`, + ); + } + const onnx = []; + for (const [key, dir, name] of [ + ['transformers', ort, 'ort-wasm-simd-threaded.asyncify'], + ['kokoro', kokoroTransformers, 'ort-wasm-simd-threaded.jsep'], + ]) { + const file = posix(path.join(dir, name)); + lines.push( + `import ${key}Mjs from ${JSON.stringify(`${file}.mjs?url`)};`, + ); + lines.push( + `import ${key}OrtWasm from ${JSON.stringify(`${file}.wasm?url`)};`, + ); + onnx.push(`${key}: { mjs: ${key}Mjs, wasm: ${key}OrtWasm }`); + } + lines.push(`export const litertRuntimes = { ${runtimes.join(', ')} };`); + lines.push(`export const onnxWasmPaths = { ${onnx.join(', ')} };`); + lines.push('export function requireRuntime(value) { return value; }'); + return lines.join('\n'); + }, + }; + return paths; + }; + const phonemizer = phonemizerLoaderPath(root); + return { + name: 'gev-on-device-runtime', + enforce: 'pre', + resolveId(source) { + return source === 'phonemizer' ? phonemizer : null; + }, + generateBundle(_options, bundle) { + const found = findEspeakChunks(bundle); + if (found.length) + this.error( + `eSpeak NG phonemizer code must not be bundled (found in ${found.join(', ')})`, + ); + }, + load(id) { + const file = path.normalize(id.split('?')[0]); + return file === resolvePaths().assets ? resolvePaths().source() : null; + }, + transform(code, id) { + const file = path.normalize(id.split('?')[0]); + if (!file.endsWith('.js') || !resolvePaths().loaders.has(file)) + return null; + // Modules are strict: the loaders' debug hook declares a function + // inside a block, which only sloppy scripts hoist. + const prelude = + 'var custom_dbg = function () { console.warn.apply(console, arguments); };\n'; + return { + code: `${prelude}${code}\nexport default ModuleFactory;\n`, + map: null, + }; + }, + }; +} diff --git a/build/vite.js b/build/vite.js index cdff4a27c..5ac45a7ac 100644 --- a/build/vite.js +++ b/build/vite.js @@ -1,5 +1,9 @@ import { applicationHtmlPlugin } from './application-html.js'; import cesium from 'vite-plugin-cesium'; +import { + onDeviceRuntimePlugin, + phonemizerLoaderPath, +} from './onDeviceRuntime.js'; /** Build browser assets with explicit inputs; never load environment or providers. */ export function createBrowserViteConfig({ @@ -9,10 +13,22 @@ export function createBrowserViteConfig({ cesiumToken, host = 'localhost', port = 4173, + naturalVoice, command, } = {}) { return { - plugins: [cesium(), applicationHtmlPlugin(), ...plugins], + plugins: [ + cesium(), + applicationHtmlPlugin(), + onDeviceRuntimePlugin(), + ...plugins, + ], + // Workers are bundled separately and need the on-device runtime too. + worker: { plugins: () => [onDeviceRuntimePlugin()] }, + // Also applied while pre-bundling kokoro-js for the dev server. + resolve: { + alias: [{ find: /^phonemizer$/, replacement: phonemizerLoaderPath() }], + }, ...(publicDir === undefined ? {} : { publicDir }), // A production build must not clean the dependency cache a running dev // server is still serving optimized module URLs from. @@ -26,6 +42,9 @@ export function createBrowserViteConfig({ '@jtarrio/signals/demod/modes.js', '@jtarrio/webrtlsdr/rtlsdr.js', 'egm96-universal', + '@litert-lm/core', + '@huggingface/transformers', + 'kokoro-js', ], }, server: { @@ -47,6 +66,9 @@ export function createBrowserViteConfig({ define: { 'import.meta.env.GOOGLE_MAPS_API_KEY': JSON.stringify(googleApiKey), 'import.meta.env.CESIUM_ION_TOKEN': JSON.stringify(cesiumToken), + // GEV_NATURAL_VOICE=off leaves natural (Kokoro) voice out of the + // on-device tier; replies use the browser's on-device voices or text. + 'import.meta.env.GEV_NATURAL_VOICE': JSON.stringify(naturalVoice ?? ''), }, build: { chunkSizeWarningLimit: 1500 }, }; diff --git a/package-lock.json b/package-lock.json index 17bb0691e..ae9b591ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,13 +9,16 @@ "version": "0.1.1", "license": "MIT", "dependencies": { + "@huggingface/transformers": "4.3.0", "@jtarrio/signals": "^0.10.0", "@jtarrio/webrtlsdr": "^3.0.6", + "@litert-lm/core": "0.17.1", "@mapbox/vector-tile": "^3.0.0", "@meri-imperiumi/eccodes-wasm": "^2.48.2", "cesium": "^1.124.0", "egm96-universal": "^1.1.1", "hls.js": "^1.7.3", + "kokoro-js": "1.2.1", "mgrs": "^2.1.0", "pbf": "^5.1.2", "satellite.js": "^6.0.2" @@ -87,7 +90,6 @@ "version": "1.11.3", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -536,11 +538,38 @@ "node": ">=18" } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.10.tgz", + "integrity": "sha512-SgS1D1bglQ94ceD4ZCL6eayUDy9uV1xuyk61OjgSxU5GJh7upqZCKII0JoTYMc6wWsg3JUgaPRX0ElQzXPk3Cw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz", + "integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.3.0.tgz", + "integrity": "sha512-fL1A/WUZwouPrOlYxU5dzIwD2T5J781JiB2jDR8bFe5DwCj0Gfudq+NEXCMno49kQgajHA7xQkrRLJlqG1veEA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.10", + "@huggingface/tokenizers": "^0.2.0", + "onnxruntime-node": "1.30.0", + "onnxruntime-web": "1.31.0-dev.20260914-8d85527a0", + "sharp": "^0.35.4" + } + }, "node_modules/@img/colour": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=18" @@ -553,7 +582,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -576,7 +604,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -596,7 +623,6 @@ "version": "0.35.4", "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -619,7 +645,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -636,7 +661,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -653,7 +677,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -670,7 +693,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -687,7 +709,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -704,7 +725,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -721,7 +741,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -738,7 +757,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -755,7 +773,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -772,7 +789,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -789,7 +805,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -812,7 +827,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -835,7 +849,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -858,7 +871,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -881,7 +893,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -904,7 +915,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -927,7 +937,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -950,7 +959,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -970,7 +978,6 @@ "version": "0.35.4", "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -990,7 +997,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0", "optional": true, "dependencies": { @@ -1010,7 +1016,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -1030,7 +1035,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -1050,7 +1054,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -1063,6 +1066,18 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@jtarrio/signals": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/@jtarrio/signals/-/signals-0.10.1.tgz", @@ -1079,6 +1094,21 @@ "@types/w3c-web-usb": "^1.0.10" } }, + "node_modules/@litert-lm/core": { + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@litert-lm/core/-/core-0.17.1.tgz", + "integrity": "sha512-eUxw6ggqZrvNA8Uk/w0TwglRWryn/clrjqHYyfxQWRl0uz5Txmte+qss0u3Dt+X6W/io4G8ADk97LmjMxVRHUg==", + "license": "Apache-2.0", + "dependencies": { + "@litertjs/wasm-utils": "^2.0.0" + } + }, + "node_modules/@litertjs/wasm-utils": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/@litertjs/wasm-utils/-/wasm-utils-2.5.3.tgz", + "integrity": "sha512-4fJiw6tBQnIs+0jH/OQ16nYOiSl8/+zZnn5h2ul8aTKHi05sVqZW580Tmk78pC2kT39A9OYBVdFawteZ/sI1kQ==", + "license": "Apache-2.0" + }, "node_modules/@mapbox/point-geometry": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@mapbox/point-geometry/-/point-geometry-1.1.0.tgz", @@ -1114,6 +1144,63 @@ "node": ">=24.0.0" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "license": "BSD-3-Clause" + }, "node_modules/@puppeteer/browsers": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.2.tgz", @@ -1549,6 +1636,15 @@ "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", "license": "MIT" }, + "node_modules/@types/node": { + "version": "26.6.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.3.tgz", + "integrity": "sha512-dsqMQQoeTLqu9wynDD00q573mNzso3IdQOAfHRJqLCcmCFPoGo9A1bDpUcv/9tnKpErQWv9uKeGfl37EIS02Yg==", + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -1576,6 +1672,15 @@ "node": ">=18.0.0" } }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "engines": { + "node": ">=14.0" + } + }, "node_modules/ansi-regex": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", @@ -1630,6 +1735,13 @@ "integrity": "sha512-1G3U4n5JE6RAiALMxu0p1XmeZkTeCwGKykzsLTCqVzfSDaN6S7fKnkIkfejogz+iwqBWc0UYAIKnKHNN7pSfDg==", "license": "MIT" }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/cesium": { "version": "1.138.0", "resolved": "https://registry.npmjs.org/cesium/-/cesium-1.138.0.tgz", @@ -1648,6 +1760,15 @@ "node": ">=20.19.0" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/chromium-bidi": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-17.0.2.tgz", @@ -1721,6 +1842,40 @@ "dev": true, "license": "MIT" }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1746,12 +1901,17 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/devtools-protocol": { "version": "0.0.1666840", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1666840.tgz", @@ -1817,6 +1977,30 @@ "node": ">= 0.8" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.25.12", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", @@ -1876,6 +2060,18 @@ "dev": true, "license": "MIT" }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", @@ -1911,6 +2107,12 @@ } } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/fresh": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", @@ -1975,6 +2177,49 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/global-agent": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", + "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", + "license": "BSD-3-Clause", + "dependencies": { + "globalthis": "^1.0.2", + "matcher": "^4.0.0", + "semver": "^7.3.5", + "serialize-error": "^8.1.0" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -1988,6 +2233,24 @@ "integrity": "sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==", "license": "MIT" }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hls.js": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.7.3.tgz", @@ -2041,6 +2304,12 @@ "node": ">= 10.16.0" } }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/jsonfile": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", @@ -2060,6 +2329,698 @@ "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==", "license": "ISC" }, + "node_modules/kokoro-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/kokoro-js/-/kokoro-js-1.2.1.tgz", + "integrity": "sha512-oq0HZJWis3t8lERkMJh84WLU86dpYD0EuBPtqYnLlQzyFP1OkyBRDcweAqCfhNOpltyN9j/azp1H6uuC47gShw==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/transformers": "^3.5.1", + "phonemizer": "^1.2.1" + } + }, + "node_modules/kokoro-js/node_modules/@huggingface/transformers": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-3.8.1.tgz", + "integrity": "sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.3", + "onnxruntime-node": "1.21.0", + "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", + "sharp": "^0.34.1" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/kokoro-js/node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/kokoro-js/node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/kokoro-js/node_modules/onnxruntime-common": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz", + "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==", + "license": "MIT" + }, + "node_modules/kokoro-js/node_modules/onnxruntime-node": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz", + "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "global-agent": "^3.0.0", + "onnxruntime-common": "1.21.0", + "tar": "^7.0.1" + } + }, + "node_modules/kokoro-js/node_modules/onnxruntime-web": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/kokoro-js/node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.22.0-dev.20250409-89f8206ba4", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz", + "integrity": "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==", + "license": "MIT" + }, + "node_modules/kokoro-js/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/kokoro-js/node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/kokoro-js/node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, + "node_modules/kokoro-js/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ktx-parse": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/ktx-parse/-/ktx-parse-1.1.0.tgz", @@ -2101,6 +3062,21 @@ "sourcemap-codec": "^1.4.8" } }, + "node_modules/matcher": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", + "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mersenne-twister": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", @@ -2132,6 +3108,27 @@ "node": ">=4" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mitt": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", @@ -2181,6 +3178,15 @@ "integrity": "sha512-9d1HbpKLh3sdWlhXMhU6MMH+wQzKkrgfRkYV0EBdvt99YJfj0ilCJrWRDYG2130Tm4GXbEoTCx5b34JSaP+HhA==", "license": "MIT" }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -2194,6 +3200,72 @@ "node": ">= 0.8" } }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", + "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.6.0", + "global-agent": "^4.1.3", + "onnxruntime-common": "1.30.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.31.0-dev.20260914-8d85527a0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.31.0-dev.20260914-8d85527a0.tgz", + "integrity": "sha512-Iy7rtadoBgxS/LLvDr3QW38DB1PNXRnr0GJMcL0TAt7c9qjgVQl83UlGCVyeAnK2InpmW8Uc3PL8XIuqtDeF6g==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.31.0-dev.20260911-2a43ec07e", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.31.0-dev.20260911-2a43ec07e", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.31.0-dev.20260911-2a43ec07e.tgz", + "integrity": "sha512-gBuF6U32YErKIAt+yD7DeGBpjRxhJ1uJwako6ygjokSZULDU6Pd+KWComX8Tumk1hYVHRxjPj7mdMnD0ryAPOw==", + "license": "MIT" + }, + "node_modules/onnxruntime-web/node_modules/protobufjs": { + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", + "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/pako": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", @@ -2222,6 +3294,12 @@ "pbf": "bin/pbf" } }, + "node_modules/phonemizer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/phonemizer/-/phonemizer-1.2.1.tgz", + "integrity": "sha512-v0KJ4mi2T4Q7eJQ0W15Xd4G9k4kICSXE8bpDeJ8jisL4RyJhNWsweKTOi88QXFc4r4LZlz5jVL5lCHhkpdT71A==", + "license": "Apache-2.0" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -2242,6 +3320,12 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/postcss": { "version": "8.5.28", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", @@ -2385,6 +3469,23 @@ "integrity": "sha512-rRg/6Lb+IGfJqO05HZkN50UtY7K/JhxJag1kP23+zyMfrvoB0B7RWv06MbOzoc79RgCdNTiUaNsTT1AJZ7Z+cg==", "license": "MIT" }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/rollup": { "version": "4.62.0", "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.0.tgz", @@ -2440,7 +3541,6 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -2449,6 +3549,12 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, "node_modules/send": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", @@ -2474,6 +3580,21 @@ "node": ">= 0.8.0" } }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serve-static": { "version": "1.16.3", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", @@ -2501,7 +3622,6 @@ "version": "0.35.4", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", - "dev": true, "license": "Apache-2.0", "dependencies": { "@img/colour": "^1.1.0", @@ -2565,6 +3685,12 @@ "dev": true, "license": "MIT" }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -2608,6 +3734,22 @@ "url": "https://github.com/chalk/strip-ansi?sponsor=1" } }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", @@ -2655,6 +3797,18 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typed-query-selector": { "version": "2.12.2", "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", @@ -2662,6 +3816,12 @@ "dev": true, "license": "MIT" }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT" + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -2883,6 +4043,15 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/yargs": { "version": "18.1.0", "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", diff --git a/package.json b/package.json index e48881c6d..9d143b0f3 100644 --- a/package.json +++ b/package.json @@ -49,13 +49,16 @@ "qa:transit": "node scripts/qa-transit.mjs" }, "dependencies": { + "@huggingface/transformers": "4.3.0", "@jtarrio/signals": "^0.10.0", "@jtarrio/webrtlsdr": "^3.0.6", + "@litert-lm/core": "0.17.1", "@mapbox/vector-tile": "^3.0.0", "@meri-imperiumi/eccodes-wasm": "^2.48.2", "cesium": "^1.124.0", "egm96-universal": "^1.1.1", "hls.js": "^1.7.3", + "kokoro-js": "1.2.1", "mgrs": "^2.1.0", "pbf": "^5.1.2", "satellite.js": "^6.0.2" diff --git a/scripts/check-import-directions.mjs b/scripts/check-import-directions.mjs index a5e61f039..e508bdd20 100644 --- a/scripts/check-import-directions.mjs +++ b/scripts/check-import-directions.mjs @@ -24,6 +24,10 @@ const renderer = (file) => /^src\/layers\/[^/]+\/(?:index|rendering|snapshotRenderer|overlay|presentation|cards|controls)\.js$/.test( file, ); +// The runtime phonemizer loader imports a downloaded file only after its +// SHA-256 matches the pinned value; no other module may import a computed URL. +const verifiedRuntimeImport = (file) => + file === 'src/voice/local/workers/phonemizerRuntime.js'; // Two compatibility composition entries intentionally select standalone defaults. const compatibilityEdge = (from, to) => (from === 'server/providers/local.js' && @@ -84,7 +88,9 @@ export function checkImportDirections(root) { } let analysis; try { - analysis = analyzeModule(readFileSync(resolved, 'utf8')); + analysis = analyzeModule(readFileSync(resolved, 'utf8'), { + allowComputedImport: verifiedRuntimeImport(file), + }); } catch (error) { report(file, error.message); return null; diff --git a/scripts/format-scope.json b/scripts/format-scope.json index cd836a5cb..9e1846796 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -692,7 +692,7 @@ "src/sources/protocols.test.mjs", "src/voice/actionSchemas.js", "src/voice/actionSchemas.test.mjs", - "server/providers/openai/toolDescriptions.js", + "src/voice/actionDescriptions.js", "src/voice/session.js", "src/voice/sessionCommands.js", "src/voice/realtimeSession.js", @@ -864,5 +864,24 @@ "src/layers/traffic/motion.test.mjs", "scripts/qa-journey-recorder.mjs", "scripts/qa-alpr-journey.mjs", - "scripts/qa-context-arrivals.mjs" + "scripts/qa-context-arrivals.mjs", + "build/onDeviceRuntime.js", + "src/ui/styles/local-voice.css", + "src/ui/styles/voice-engine.css", + "src/voice/voiceEngine.js", + "src/voice/voiceEngineControls.js", + "src/voice/voiceEnginePresentation.js", + "src/voice/voiceEngine.test.mjs", + "src/voice/local/naturalVoice.test.mjs", + "src/voice/local/naturalVoiceSetting.js", + "src/voice/local/workers/phonemizerRuntime.js", + "src/voice/local/localVoice.test.mjs", + "src/voice/local/localVoiceUi.test.mjs", + "src/voice/local/localRuntime.test.mjs", + "src/voice/local/localWebProvider.test.mjs", + "src/voice/local/localStreaming.test.mjs", + "src/voice/local/modelCache.test.mjs", + "src/voice/local/localRadio.test.mjs", + "src/voice/local/onDeviceRuntime.test.mjs", + "src/voice/local/localWorkflow.test.mjs" ] diff --git a/scripts/module-analysis.mjs b/scripts/module-analysis.mjs index 0976159f7..c51a5d394 100644 --- a/scripts/module-analysis.mjs +++ b/scripts/module-analysis.mjs @@ -15,8 +15,13 @@ const browserGlobals = new Set([ 'getComputedStyle', ]); -/** Inspect literal module dependencies and browser-platform references without executing code. */ -export function analyzeModule(source) { +/** + * Inspect literal module dependencies and browser-platform references without + * executing code. `allowComputedImport` admits dynamic imports of computed + * specifiers (left out of the graph) for the one loader that imports bytes it + * has already verified. + */ +export function analyzeModule(source, { allowComputedImport = false } = {}) { const imports = []; const browser = new Set(); function walk(node, parent, field) { @@ -30,11 +35,11 @@ export function analyzeModule(source) { ].includes(node.type) && node.source ) { - if (node.source.type !== 'StringLiteral') + if (node.source.type === 'StringLiteral') imports.push(node.source.value); + else if (!(allowComputedImport && node.type === 'ImportExpression')) throw new Error( 'Computed module imports are not allowed in runtime code', ); - imports.push(node.source.value); } if (node.type === 'CallExpression' && node.callee?.name === 'require') throw new Error('Runtime modules must use ES imports'); diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 5e27c4db7..0ceaf58f9 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -27,7 +27,11 @@ "vite-build": { "runtime": "node", "exports": ["./build/vite", "./build/html"], - "modules": ["build/vite.js", "build/application-html.js"], + "modules": [ + "build/vite.js", + "build/application-html.js", + "build/onDeviceRuntime.js" + ], "external": ["vite-plugin-cesium"] }, "live-providers": { @@ -283,7 +287,7 @@ "src/voice/voiceCost.js", "src/voice/actionSchemas.js", "src/voice/layerManifest.js", - "server/providers/openai/toolDescriptions.js", + "src/voice/actionDescriptions.js", "src/sources/rateLimit.js" ], "external": [] @@ -1299,7 +1303,12 @@ "src/voice/narration.js", "src/voice/earcon.js", "src/voice/pointerContext.js", - "src/voice/pointerReticle.js" + "src/voice/pointerReticle.js", + "src/voice/voiceEngineControls.js", + "src/voice/voiceEnginePresentation.js", + "src/voice/local/modelCatalog.js", + "src/ui/chipGroup.js", + "src/ui/uiLifetime.js" ], "external": [] }, @@ -2056,7 +2065,37 @@ "src/sources/requestDeadline.js", "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", - "src/data/infrastructureOverlayEntry.js" + "src/data/infrastructureOverlayEntry.js", + "src/voice/voiceEngine.js", + "src/voice/voiceEngineControls.js", + "src/voice/voiceEnginePresentation.js", + "src/voice/local/gemmaToolCalls.js", + "src/voice/local/localAudio.js", + "src/voice/local/localEngine.js", + "src/voice/local/localPrompt.js", + "src/voice/local/localVoiceControls.js", + "src/voice/local/localVoicePreferences.js", + "src/voice/local/localVoicePresentation.js", + "src/voice/local/localVoiceStore.js", + "src/voice/local/localWebSession.js", + "src/voice/local/modelCatalog.js", + "src/voice/local/replies.js", + "src/voice/local/speechText.js", + "src/voice/local/toolset.js", + "src/voice/local/workers/modelCache.js", + "src/voice/local/workers/runtimeAssets.js", + "src/voice/local/workers/phonemizerRuntime.js", + "src/voice/local/workerClient.js", + "src/voice/local/localWebProvider.js", + "src/voice/local/localWebFlag.js", + "src/voice/local/localRadio.js", + "src/voice/local/speechOutput.js", + "src/voice/local/streamTee.js", + "src/voice/local/sha256.js", + "src/voice/local/naturalVoiceSetting.js", + "src/ui/styles/local-voice.css", + "src/voice/actionDescriptions.js", + "src/voice/actionSchemas.js" ], "external": [ "hls.js", @@ -2789,7 +2828,12 @@ "src/voice/referents.js", "src/voice/layerManifest.js", "src/voice/speech.js", - "src/voice/pointerReticle.js" + "src/voice/pointerReticle.js", + "src/voice/voiceEngineControls.js", + "src/voice/voiceEnginePresentation.js", + "src/voice/local/modelCatalog.js", + "src/ui/chipGroup.js", + "src/ui/uiLifetime.js" ], "external": [] }, diff --git a/server/providers/openai/tools.js b/server/providers/openai/tools.js index a1bbc4745..97de0c148 100644 --- a/server/providers/openai/tools.js +++ b/server/providers/openai/tools.js @@ -1,4 +1,4 @@ import { createActionTools } from '../../../src/voice/actionSchemas.js'; -import { ACTION_DESCRIPTIONS } from './toolDescriptions.js'; +import { ACTION_DESCRIPTIONS } from '../../../src/voice/actionDescriptions.js'; export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS); diff --git a/server/standalone/vite.config.js b/server/standalone/vite.config.js index ffa5a1e27..27b3aaeb1 100644 --- a/server/standalone/vite.config.js +++ b/server/standalone/vite.config.js @@ -18,6 +18,7 @@ export default defineConfig(({ command, mode }) => { cesiumToken: process.env.CESIUM_ION_TOKEN, host: process.env.HOST, port: process.env.PORT, + naturalVoice: process.env.GEV_NATURAL_VOICE, command, }); }); diff --git a/src/app/tools.js b/src/app/tools.js index 613add71c..bef3eeb38 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -157,8 +157,13 @@ export function createApplicationTools({ defer(() => { if (window.__godsEyeView === debug) delete window.__godsEyeView; }); - const voiceCommands = initGevVoiceCommands({ + let voiceCommands = initGevVoiceCommands({ ...voice, + // Choosing Cloud or On-device rebuilds voice; track the live controls. + onReplace: (next) => { + voiceCommands = next; + debug.voiceCommands = next; + }, floorServices: operations.surface.groundFloor, annotationResolver: operations.annotationResolver, searchNavigation: operations.searchAndFlyTo, diff --git a/src/radioMarkup.test.mjs b/src/radioMarkup.test.mjs index 23ad778d5..3ac88e4d7 100644 --- a/src/radioMarkup.test.mjs +++ b/src/radioMarkup.test.mjs @@ -20,7 +20,7 @@ const radio = ['playback', 'interaction'].map(name => ).join('\n').replace(/layerState\.|parts\.\w+\./g, ''); const rocketLaunches = readLayerSource(new URL('./data/rocketLaunches.js', import.meta.url), 'utf8'); const realtime = readRealtimeSource(); -const voice = ['./voice/actionSchemas.js', '../server/providers/openai/toolDescriptions.js', '../server/providers/openai/instructions.js'].map(file => readFileSync(new URL(file, import.meta.url), 'utf8')).join('\n'); +const voice = ['./voice/actionSchemas.js', './voice/actionDescriptions.js', '../server/providers/openai/instructions.js'].map(file => readFileSync(new URL(file, import.meta.url), 'utf8')).join('\n'); const css = readStylesheet(new URL('../style.css', import.meta.url)); function realtimeTools() { return GEV_REALTIME_TOOLS; } diff --git a/src/tooling/importDirections.test.mjs b/src/tooling/importDirections.test.mjs index b34fee3b9..913b2f608 100644 --- a/src/tooling/importDirections.test.mjs +++ b/src/tooling/importDirections.test.mjs @@ -43,6 +43,12 @@ test('module analysis parses dependencies and platform references without readin ['document', 'navigator'], ); assert.throws(() => analyzeModule('import(name)'), /Computed/); + // Only the verified runtime loader may import a computed URL, and only + // dynamically; it adds nothing to the module graph. + assert.deepEqual( + analyzeModule('import(url)', { allowComputedImport: true }).imports, + [], + ); assert.throws(() => analyzeModule("require('node:fs')"), /ES imports/); }); diff --git a/src/tooling/viteBuild.test.mjs b/src/tooling/viteBuild.test.mjs index 497035a40..7123fb84a 100644 --- a/src/tooling/viteBuild.test.mjs +++ b/src/tooling/viteBuild.test.mjs @@ -12,7 +12,8 @@ test('explicit build inputs preserve browser-only defines, plugin order and loop googleApiKey: 'browser-fixture', cesiumToken: 'ion-fixture', }); - assert.equal(config.plugins[2], plugin); + assert.equal(config.plugins[2].name, 'gev-on-device-runtime'); + assert.equal(config.plugins[3], plugin); assert.equal(config.server.host, 'localhost'); assert.equal(config.server.port, 4173); assert.deepEqual(config.server.allowedHosts, [ @@ -30,7 +31,14 @@ test('explicit build inputs preserve browser-only defines, plugin order and loop assert.deepEqual(config.define, { 'import.meta.env.GOOGLE_MAPS_API_KEY': '"browser-fixture"', 'import.meta.env.CESIUM_ION_TOKEN': '"ion-fixture"', + 'import.meta.env.GEV_NATURAL_VOICE': '""', }); + assert.equal( + createBrowserViteConfig({ naturalVoice: 'off' }).define[ + 'import.meta.env.GEV_NATURAL_VOICE' + ], + '"off"', + ); assert.equal( createBrowserViteConfig({ host: '0.0.0.0', port: '4800' }).server .allowedHosts, @@ -51,7 +59,7 @@ test('build helper does not discover environment values or construct local provi config.define['import.meta.env.GOOGLE_MAPS_API_KEY'], undefined, ); - assert.equal(config.plugins.length, 2); + assert.equal(config.plugins.length, 3); } finally { if (before === undefined) delete process.env.GOOGLE_MAPS_API_KEY; else process.env.GOOGLE_MAPS_API_KEY = before; @@ -63,7 +71,7 @@ test('root config retains existing named exports and standalone provider order', assert.equal(compatibility[name], value, name); const config = standaloneConfig({ mode: 'test' }); assert.deepEqual( - config.plugins.slice(2, -1).map((plugin) => plugin.name), + config.plugins.slice(3, -1).map((plugin) => plugin.name), providers.localProviderPlugins().map((plugin) => plugin.name), ); assert.equal(config.plugins.at(-2).name, 'gev-key-setup'); diff --git a/src/ui/styles/local-voice.css b/src/ui/styles/local-voice.css new file mode 100644 index 000000000..f354e893b --- /dev/null +++ b/src/ui/styles/local-voice.css @@ -0,0 +1,100 @@ +/* ------------------------------------------------------------------ * + * On-device model settings. Loaded with the on-device provider and + * rendered inside the voice engine tray (voice-engine.css), sharing its + * headings and the shared .data-toggle-chip chips. + * ------------------------------------------------------------------ */ +#gev-voice-control .gev-voice-local-settings { + display: grid; + gap: 0.34rem; + padding-top: 0.34rem; + border-top: 1px solid rgba(54, 220, 255, 0.14); +} + +#gev-voice-control .gev-voice-local-state { + overflow: hidden; + color: var(--text-secondary); + font-family: var(--font-mono); + font-size: 0.46rem; + letter-spacing: 0.06rem; + text-align: right; + text-overflow: ellipsis; +} + +#gev-voice-control .gev-voice-local-state.error { + color: #ffb4bd; + white-space: normal; +} + +#gev-voice-control .gev-voice-local-row { + display: grid; + grid-template-columns: 3.4rem 1fr; + align-items: center; + gap: 0.4rem; +} + +#gev-voice-control .gev-voice-local-notice { + margin: 0; + color: var(--text-secondary); + font-family: var(--font-sans); + font-size: 0.58rem; + line-height: 1.35; +} + +#gev-voice-control .gev-voice-local-progress { + display: grid; + gap: 0.22rem; + color: var(--text-secondary); + font-family: var(--font-mono); + font-size: 0.46rem; + letter-spacing: 0.05rem; +} + +#gev-voice-control .gev-voice-local-progress[hidden], +#gev-voice-control .gev-voice-local-note[hidden], +#gev-voice-control .gev-voice-local-notice[hidden] { + display: none; +} + +#gev-voice-control .gev-voice-local-bar { + height: 3px; + overflow: hidden; + background: rgba(255, 255, 255, 0.1); + border-radius: 2px; +} + +#gev-voice-control .gev-voice-local-bar > span { + display: block; + width: 0; + height: 100%; + background: var(--accent); + transition: width 200ms ease; +} + +#gev-voice-control .gev-voice-local-storage { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.4rem; +} + +#gev-voice-control .gev-voice-local-storage-note { + color: var(--text-secondary); + font-family: var(--font-sans); + font-size: 0.6rem; +} + +#gev-voice-control .gev-voice-local-storage-note[hidden] { + display: none; +} + +#gev-voice-control .gev-voice-local-note { + color: #ffc46b; + font-family: var(--font-sans); + font-size: 0.6rem; +} + +@media (prefers-reduced-motion: reduce) { + #gev-voice-control .gev-voice-local-bar > span { + transition: none; + } +} diff --git a/src/ui/styles/voice-engine.css b/src/ui/styles/voice-engine.css new file mode 100644 index 000000000..84dc3b407 --- /dev/null +++ b/src/ui/styles/voice-engine.css @@ -0,0 +1,142 @@ +/* ------------------------------------------------------------------ * + * Voice engine tray (Cloud / On-device) nested in #gev-voice-control. + * Shares the help tray's surface and the shared .data-toggle-chip chips. + * The on-device provider's model rows (local-voice.css) load with it. + * ------------------------------------------------------------------ */ +#gev-voice-control .gev-voice-engine-tray { + position: absolute; + bottom: calc(100% + 0.7rem); + left: 50%; + z-index: 8; + display: grid; + gap: 0.34rem; + box-sizing: border-box; + width: clamp(18rem, 27vw, 22rem); + max-width: calc(100vw - 2rem); + max-height: min(34rem, calc(100vh - 8rem)); + overflow-y: auto; + padding: 0.62rem 0.72rem; + visibility: hidden; + opacity: 0; + color: rgba(223, 245, 250, 0.92); + background: rgba(7, 22, 31, 0.96); + border: 1px solid rgba(54, 220, 255, 0.34); + border-radius: 0.38rem; + box-shadow: + 0 0.85rem 2rem rgba(0, 0, 0, 0.58), + 0 0 1.2rem rgba(0, 212, 255, 0.08); + backdrop-filter: blur(24px) saturate(1.3); + -webkit-backdrop-filter: blur(24px) saturate(1.3); + pointer-events: none; + transform: translate(-50%, 0.65rem) scale(0.985); + transform-origin: bottom center; + transition: + opacity 160ms ease, + visibility 160ms ease, + transform 210ms cubic-bezier(0.2, 0.8, 0.2, 1); +} + +#gev-voice-control[data-voice-tray='open'] .gev-voice-engine-tray, +#gev-voice-control[data-local-tray='open'] .gev-voice-engine-tray { + visibility: visible; + opacity: 1; + pointer-events: auto; + transform: translate(-50%, 0) scale(1); +} + +/* The tray and the voice card share one slot: an open tray takes it. */ +#gev-voice-control[data-voice-tray='open'] .gev-voice-help-tray, +#gev-voice-control[data-voice-tray='open'] .gev-voice-card, +#gev-voice-control[data-local-tray='open'] .gev-voice-help-tray, +#gev-voice-control[data-local-tray='open'] .gev-voice-card { + display: none; +} + +#command-dock.dock-has-pinned-tray .gev-voice-engine-tray { + bottom: calc(100% + 1.7rem + var(--dock-pinned-stack-height)); +} + +#gev-voice-control .gev-voice-engine-toggle.active { + color: var(--text-primary); + border-color: rgba(0, 212, 255, 0.6); + background: rgba(0, 212, 255, 0.12); +} + +#gev-voice-control .gev-voice-local-heading { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 0.6rem; +} + +#gev-voice-control .gev-voice-local-kicker, +#gev-voice-control .gev-voice-local-label { + color: rgba(54, 220, 255, 0.72); + font-family: var(--font-mono); + font-size: 0.46rem; + font-weight: 600; + letter-spacing: 0.1rem; +} + +#gev-voice-control .gev-voice-local-hint { + margin: 0; + color: var(--text-dim); + font-family: var(--font-sans); + font-size: 0.6rem; + line-height: 1.35; +} + +#gev-voice-control .gev-voice-local-chips { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; +} + +#gev-voice-control .gev-voice-engine-device { + display: grid; + gap: 0.18rem; + padding-top: 0.34rem; + border-top: 1px solid rgba(54, 220, 255, 0.14); +} + +#gev-voice-control .gev-voice-engine-fact { + color: var(--text-secondary); + font-family: var(--font-mono); + font-size: 0.5rem; + letter-spacing: 0.04rem; +} + +#gev-voice-control .gev-voice-engine-guidance { + display: grid; + gap: 0.1rem; + margin: 0.12rem 0 0; + padding-left: 0.9rem; + color: var(--text-dim); + font-family: var(--font-sans); + font-size: 0.58rem; + line-height: 1.3; +} + +#gev-voice-control .gev-voice-engine-verdict { + margin: 0.1rem 0 0; + color: rgba(223, 245, 250, 0.92); + font-family: var(--font-sans); + font-size: 0.6rem; + line-height: 1.35; +} + +#gev-voice-control .gev-voice-engine-verdict.error { + color: #ffb4bd; +} + +#gev-voice-control .gev-voice-engine-fact[hidden], +#gev-voice-control .gev-voice-engine-verdict[hidden], +#gev-voice-control .gev-voice-local-slot:empty { + display: none; +} + +@media (prefers-reduced-motion: reduce) { + #gev-voice-control .gev-voice-engine-tray { + transition: none; + } +} diff --git a/server/providers/openai/toolDescriptions.js b/src/voice/actionDescriptions.js similarity index 99% rename from server/providers/openai/toolDescriptions.js rename to src/voice/actionDescriptions.js index cd49cf4ec..6863c95c9 100644 --- a/server/providers/openai/toolDescriptions.js +++ b/src/voice/actionDescriptions.js @@ -1,7 +1,4 @@ -import { - voiceLayerAliasHint, - voiceQueryFieldHint, -} from '../../../src/voice/layerManifest.js'; +import { voiceLayerAliasHint, voiceQueryFieldHint } from './layerManifest.js'; export const ACTION_DESCRIPTIONS = { fly_to_location: { diff --git a/src/voice/gevRealtime.js b/src/voice/gevRealtime.js index 3ea12139d..d8571cd74 100644 --- a/src/voice/gevRealtime.js +++ b/src/voice/gevRealtime.js @@ -1,12 +1,26 @@ import { createGevActionRunner } from './gevActions.js'; import { createVoiceCommands } from './commands.js'; +import { createLocalWebSessionLoader } from './local/localWebProvider.js'; import { cameraPoseKey, createPointerTracker } from './pointerContext.js'; import { createPointerPicker } from './pointerPick.js'; import { createReferentRegistry } from './referents.js'; +import { readVoiceEngine, writeVoiceEngine } from './voiceEngine.js'; export * from './realtimeController.js'; -/** Compose the standalone action runner with the voice controls. */ -export function initGevVoiceCommands(options) { +/** + * Compose the standalone action runner with the voice controls on the + * chosen engine (Cloud or On-device). Choosing the other engine in the + * voice control persists it and rebuilds voice on that engine; `onReplace` + * receives the new controls. + */ +export function initGevVoiceCommands( + options, + { + engine = readVoiceEngine(), + onReplace = options.onReplace, + openTray = false, + } = {}, +) { const canvas = options.viewer?.scene?.canvas || null; const cameraKey = () => cameraPoseKey(options.viewer); // Point-and-ask: the tracker and referents are shared by the runner (which @@ -25,11 +39,37 @@ export function initGevVoiceCommands(options) { const referents = createReferentRegistry(); // The controller exists only after the runner, so its hooks bind late. const deixis = { pointer, referents, cameraKey }; + // The on-device provider loads only when that engine is chosen. + const local = + engine === 'on-device' + ? { + createSession: (hooks) => + createLocalWebSessionLoader({ ...hooks, viewer: options.viewer }), + } + : {}; + let replaced = false; const commands = createVoiceCommands({ ...options, + ...local, pointer, referents, runner: createGevActionRunner({ ...options, deixis }), + engineChoice: { + engine, + openTray, + select(next) { + if (replaced || next === engine || !writeVoiceEngine(next)) return; + replaced = true; + // The new control opens its tray so the choice's settings show. + onReplace?.( + initGevVoiceCommands(options, { + engine: next, + onReplace, + openTray: true, + }), + ); + }, + }, }); deixis.imageFrame = () => commands.retainedImageFrame?.() || null; deixis.onPointerUsed = () => commands.announcePointer?.(); diff --git a/src/voice/local/gemmaToolCalls.js b/src/voice/local/gemmaToolCalls.js new file mode 100644 index 000000000..c60f00efd --- /dev/null +++ b/src/voice/local/gemmaToolCalls.js @@ -0,0 +1,230 @@ +// Gemma 4 tool-call text format: +// <|tool_call>call:NAME{key:<|"|>text<|"|>,flag:true,list:[1,2]} +// Keys are bare, strings are wrapped in the <|"|> token and other scalars +// are bare JSON-like words. Engines that template for us return structured +// calls; this parser covers raw text output and leaked markup. + +const QUOTE = '<|"|>'; +const CALL_PATTERN = /<\|tool_call>([\s\S]*?)(?:|$)/g; +const MARKUP_PATTERN = + /<\|tool_call>[\s\S]*?(?:|$)|<\|tool_response>[\s\S]*?(?:|$)|<\|channel>[\s\S]*?(?:|$)|<\|?turn\|?>||<\|"\|>/g; + +/** + * Extracts every complete or trailing tool call from model text. + * @param {string} text + * @returns {{calls: Array<{name: string, arguments: object}>, text: string}} + */ +export function parseGemmaToolCalls(text) { + const source = String(text || ''); + const calls = []; + for (const match of source.matchAll(CALL_PATTERN)) { + const call = parseCallBody(match[1]); + if (call) calls.push(call); + } + return { calls, text: stripGemmaMarkup(source) }; +} + +/** Removes tool, thought and turn markup so it never reaches speech. */ +export function stripGemmaMarkup(text) { + return String(text || '') + .replace(MARKUP_PATTERN, '') + .trim(); +} + +function parseCallBody(body) { + const match = /^\s*call:([A-Za-z_][\w-]*)\s*/.exec(body); + if (!match) return null; + const rest = body.slice(match[0].length); + if (!rest.trim()) return { name: match[1], arguments: {} }; + try { + const parser = new ValueParser(rest); + const value = parser.parseValue(); + return { + name: match[1], + arguments: value && typeof value === 'object' ? value : {}, + }; + } catch { + return null; + } +} + +class ValueParser { + constructor(text) { + this.text = text; + this.index = 0; + } + + skip() { + while (/\s/.test(this.text[this.index] || '')) this.index++; + } + + startsWith(token) { + return this.text.startsWith(token, this.index); + } + + parseValue() { + this.skip(); + if (this.startsWith(QUOTE)) return this.parseString(); + const char = this.text[this.index]; + if (char === '{') return this.parseObject(); + if (char === '[') return this.parseArray(); + if (char === '"') return this.parseJsonString(); + return this.parseWord(); + } + + parseString() { + this.index += QUOTE.length; + const end = this.text.indexOf(QUOTE, this.index); + if (end < 0) throw new SyntaxError('Unterminated string'); + const value = this.text.slice(this.index, end); + this.index = end + QUOTE.length; + return value; + } + + parseJsonString() { + let end = this.index + 1; + while (end < this.text.length && this.text[end] !== '"') { + if (this.text[end] === '\\') end++; + end++; + } + const value = JSON.parse(this.text.slice(this.index, end + 1)); + this.index = end + 1; + return value; + } + + parseObject() { + this.index++; + const out = {}; + for (;;) { + this.skip(); + if (this.text[this.index] === '}') { + this.index++; + return out; + } + const key = this.parseKey(); + this.skip(); + if (this.text[this.index] !== ':') throw new SyntaxError('Expected :'); + this.index++; + out[key] = this.parseValue(); + this.skip(); + if (this.text[this.index] === ',') this.index++; + else if (this.text[this.index] !== '}') + throw new SyntaxError('Expected , or }'); + } + } + + parseKey() { + if (this.startsWith(QUOTE)) return this.parseString(); + if (this.text[this.index] === '"') return this.parseJsonString(); + const match = /^[A-Za-z_$][\w$-]*/.exec(this.text.slice(this.index)); + if (!match) throw new SyntaxError('Expected key'); + this.index += match[0].length; + return match[0]; + } + + parseArray() { + this.index++; + const out = []; + for (;;) { + this.skip(); + if (this.text[this.index] === ']') { + this.index++; + return out; + } + out.push(this.parseValue()); + this.skip(); + if (this.text[this.index] === ',') this.index++; + else if (this.text[this.index] !== ']') + throw new SyntaxError('Expected , or ]'); + } + } + + parseWord() { + const match = /^[^,}\]\s]+/.exec(this.text.slice(this.index)); + if (!match) throw new SyntaxError('Expected value'); + this.index += match[0].length; + const word = match[0]; + if (word === 'true') return true; + if (word === 'false') return false; + if (word === 'null') return null; + const number = Number(word); + return Number.isFinite(number) ? number : word; + } +} + +/** Serializes arguments in the Gemma call format (used by tests and evals). */ +export function formatGemmaValue(value) { + if (typeof value === 'string') return QUOTE + value + QUOTE; + if (Array.isArray(value)) + return '[' + value.map(formatGemmaValue).join(',') + ']'; + if (value && typeof value === 'object') + return ( + '{' + + Object.entries(value) + .map(([key, item]) => `${key}:${formatGemmaValue(item)}`) + .join(',') + + '}' + ); + return String(value); +} + +/** Formats one call exactly as a Gemma 4 model emits it. */ +export function formatGemmaToolCall(name, args = {}) { + return `<|tool_call>call:${name}${formatGemmaValue(args)}`; +} + +const REGIONS = [ + ['<|tool_call>', ''], + ['<|tool_response>', ''], + ['<|channel>', ''], +]; +const TOKENS = ['<|turn>', '', '<|"|>']; +const MARKERS = [...REGIONS.map(([open]) => open), ...TOKENS]; + +/** + * Removes Gemma markup from streamed text without touching whitespace. + * Markup split across chunks is held until it can be recognised. + * push() returns speakable text; flush() returns what is left. + */ +export function createMarkupFilter() { + let buffer = ''; + const drain = (final) => { + let out = ''; + for (;;) { + const at = buffer.indexOf('<'); + if (at < 0) { + out += buffer; + buffer = ''; + return out; + } + out += buffer.slice(0, at); + buffer = buffer.slice(at); + const region = REGIONS.find(([open]) => buffer.startsWith(open)); + if (region) { + const end = buffer.indexOf(region[1], region[0].length); + if (end < 0) { + if (final) buffer = ''; + return out; + } + buffer = buffer.slice(end + region[1].length); + continue; + } + const token = TOKENS.find((item) => buffer.startsWith(item)); + if (token) { + buffer = buffer.slice(token.length); + continue; + } + const partial = MARKERS.some((marker) => marker.startsWith(buffer)); + if (partial && !final) return out; + out += '<'; + buffer = buffer.slice(1); + } + }; + return { + push: (text) => { + buffer += String(text || ''); + return drain(false); + }, + flush: () => drain(true), + }; +} diff --git a/src/voice/local/localAudio.js b/src/voice/local/localAudio.js new file mode 100644 index 000000000..d5c3d0afb --- /dev/null +++ b/src/voice/local/localAudio.js @@ -0,0 +1,228 @@ +const CAPTURE_WORKLET = ` +class GevCapture extends AudioWorkletProcessor { + process(inputs) { + const channel = inputs[0] && inputs[0][0]; + if (channel) this.port.postMessage(channel.slice(0)); + return true; + } +} +registerProcessor('gev-capture', GevCapture); +`; + +// Audio kept from just before a press so the first word is not clipped. +const PRE_ROLL_SECONDS = 0.5; + +export const CAPTURE_SAMPLE_RATE = 16000; + +/** + * Microphone capture resampled to 16 kHz. start()/stop() bracket one + * utterance; stop() resolves with the samples captured in between, plus a + * short pre-roll from before start(). + */ +export async function createMicrophoneCapture({ + getUserMedia = (constraints) => + navigator.mediaDevices.getUserMedia(constraints), +} = {}) { + const stream = await getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + // Capture at the device rate and resample: a 16 kHz context can deliver + // silence from some inputs, and not every browser supports one. + const context = new AudioContext(); + let source; + let node; + let sink; + try { + const moduleUrl = URL.createObjectURL( + new Blob([CAPTURE_WORKLET], { type: 'application/javascript' }), + ); + try { + await context.audioWorklet.addModule(moduleUrl); + } finally { + URL.revokeObjectURL(moduleUrl); + } + source = context.createMediaStreamSource(stream); + node = new AudioWorkletNode(context, 'gev-capture'); + sink = context.createGain(); + } catch (error) { + stream.getTracks().forEach((track) => track.stop()); + context.close().catch(() => {}); + throw error; + } + sink.gain.value = 0; + source.connect(node); + node.connect(sink); + sink.connect(context.destination); + let chunks = []; + let recording = false; + let generation = 0; + const preRoll = []; + let preRollSamples = 0; + const preRollLimit = PRE_ROLL_SECONDS * context.sampleRate; + node.port.onmessage = ({ data }) => { + if (recording) { + chunks.push(data); + return; + } + preRoll.push(data); + preRollSamples += data.length; + while (preRollSamples - preRoll[0].length > preRollLimit) + preRollSamples -= preRoll.shift().length; + }; + return { + stream, + get recording() { + return recording; + }, + start() { + generation++; + chunks = preRoll.splice(0); + preRollSamples = 0; + recording = true; + context.resume().catch(() => {}); + }, + async stop() { + if (!recording) return new Float32Array(0); + const utterance = generation; + const captured = chunks; + // Let the last render quantum arrive before closing the utterance. + await new Promise((resolve) => setTimeout(resolve, 60)); + // A new utterance started meanwhile: it owns the recording now. + if (utterance === generation) { + recording = false; + chunks = []; + } + const length = captured.reduce((sum, chunk) => sum + chunk.length, 0); + const audio = new Float32Array(length); + let offset = 0; + for (const chunk of captured) { + audio.set(chunk, offset); + offset += chunk.length; + } + return resampleTo16k(audio, context.sampleRate); + }, + close() { + generation++; + recording = false; + try { + source.disconnect(); + node.disconnect(); + sink.disconnect(); + } catch { + /* already disconnected */ + } + stream.getTracks().forEach((track) => track.stop()); + context.close().catch(() => {}); + }, + }; +} + +/** + * Box-filter resampler to 16 kHz; adequate for speech recognition input. + * @param {Float32Array} samples + * @param {number} rate + */ +export function resampleTo16k(samples, rate) { + if (!rate || rate === CAPTURE_SAMPLE_RATE) return samples; + const ratio = rate / CAPTURE_SAMPLE_RATE; + const out = new Float32Array(Math.floor(samples.length / ratio)); + for (let index = 0; index < out.length; index++) { + const start = Math.floor(index * ratio); + const end = Math.min(samples.length, Math.floor((index + 1) * ratio)); + let sum = 0; + for (let cursor = start; cursor < end; cursor++) sum += samples[cursor]; + out[index] = sum / Math.max(1, end - start); + } + return out; +} + +/** Gapless queue for synthesized speech with an interruptible output. */ +export function createSpeechPlayer({ onSpeaking, onIdle } = {}) { + let context = null; + let gain = null; + let destination = null; + let nextStart = 0; + let playing = 0; + let generation = 0; + let idleWaiters = []; + + function settleIdle() { + onIdle?.(); + const waiters = idleWaiters; + idleWaiters = []; + for (const resolve of waiters) resolve(); + } + + function ensure() { + if (context) return; + context = new AudioContext(); + destination = context.createMediaStreamDestination(); + connectGain(); + } + + function connectGain() { + gain = context.createGain(); + gain.connect(context.destination); + gain.connect(destination); + } + + return { + get stream() { + ensure(); + return destination.stream; + }, + get speaking() { + return playing > 0; + }, + /** Schedules samples; resolves with the delay until they are audible. */ + enqueue(samples, sampleRate) { + ensure(); + context.resume().catch(() => {}); + const buffer = context.createBuffer(1, samples.length, sampleRate); + buffer.copyToChannel(samples, 0); + const source = context.createBufferSource(); + source.buffer = buffer; + source.connect(gain); + const at = Math.max(context.currentTime + 0.02, nextStart); + source.start(at); + nextStart = at + buffer.duration; + const epoch = generation; + if (playing++ === 0) onSpeaking?.(); + source.onended = () => { + if (epoch !== generation) return; + if (--playing === 0) settleIdle(); + }; + return (at - context.currentTime) * 1000; + }, + stop() { + if (!context) return; + generation++; + const wasPlaying = playing > 0; + playing = 0; + nextStart = 0; + try { + gain.disconnect(); + } catch { + /* already disconnected */ + } + connectGain(); + if (wasPlaying) settleIdle(); + }, + /** Settles once everything scheduled has finished playing. */ + idle() { + return playing + ? new Promise((resolve) => idleWaiters.push(resolve)) + : Promise.resolve(); + }, + close() { + this.stop(); + context?.close().catch(() => {}); + context = null; + }, + }; +} diff --git a/src/voice/local/localEngine.js b/src/voice/local/localEngine.js new file mode 100644 index 000000000..5d4441884 --- /dev/null +++ b/src/voice/local/localEngine.js @@ -0,0 +1,368 @@ +import { + TTS_MODEL, + checkLocalVoiceSupport, + storageAdvice, +} from './modelCatalog.js'; +import { MODEL_CACHE_NAME, isModelCached } from './workers/modelCache.js'; +import { createWorkerClient } from './workerClient.js'; + +// Caches written by transformers.js (speech recognition, Kokoro weights) and +// by kokoro-js (voice vectors). +const LIBRARY_CACHES = ['transformers-cache', 'kokoro-voices']; + +/** Language model client backed by the LiteRT-LM worker. */ +export function createLlmClient( + worker = new Worker(new URL('./workers/llm.worker.js', import.meta.url), { + type: 'module', + name: 'gev-local-llm', + }), +) { + const client = createWorkerClient(worker); + let turnSequence = 0; + let loadedUrl = null; + + function runStream(turnId, send, { onToolCalls, onDelta } = {}) { + return new Promise((resolve, reject) => { + const settle = (callback, value) => { + unsubscribe(); + unsubscribeFailure(); + callback(value); + }; + const unsubscribe = client.subscribe((data) => { + if (data.turnId !== turnId) return; + if (data.type === 'tool_calls') onToolCalls?.(data.calls, data.ms); + else if (data.type === 'delta') onDelta?.(data.text); + else if (data.type === 'done') settle(resolve, data); + else if (data.type === 'error') settle(reject, new Error(data.message)); + }); + const unsubscribeFailure = client.onFailure((error) => + settle(reject, error), + ); + send(); + }); + } + + return { + get loadedUrl() { + return loadedUrl; + }, + async load(model, { onProgress, maxNumTokens = 8192 } = {}) { + const result = await client.request( + { + type: 'load', + modelUrl: model.url, + sha256: model.sha256 || null, + bytes: model.bytes || 0, + maxNumTokens, + }, + { onProgress, done: (data) => data.type === 'loaded' }, + ); + loadedUrl = model.url; + return result; + }, + prepare({ system, tools, constrained = false, maxOutputTokens = 256 }) { + return client.request( + { type: 'prepare', system, tools, constrained, maxOutputTokens }, + { done: (data) => data.type === 'prepared' }, + ); + }, + /** Starts one turn on a clone of the prepared prefix. */ + turn(text, handlers = {}) { + const turnId = `turn-${++turnSequence}`; + const done = runStream( + turnId, + () => client.post({ type: 'turn', turnId, text }), + handlers, + ); + return { turnId, done }; + }, + continueTurn(turnId, results, handlers = {}) { + return runStream( + turnId, + () => client.post({ type: 'tool_result', turnId, results }), + handlers, + ); + }, + cancel(turnId) { + client.post({ type: 'cancel', turnId }); + }, + release(turnId) { + return client.request({ type: 'release', turnId }).catch(() => {}); + }, + terminate() { + loadedUrl = null; + client.terminate(); + }, + }; +} + +/** Speech recognition client backed by a transformers.js worker. */ +export function createSttClient() { + const client = createWorkerClient( + new Worker(new URL('./workers/stt.worker.js', import.meta.url), { + type: 'module', + name: 'gev-local-stt', + }), + ); + return { + load(model, { onProgress } = {}) { + return client.request( + { + type: 'load', + repo: model.repo, + revision: model.revision, + dtype: model.dtype, + language: model.language || null, + }, + { onProgress, done: (data) => data.type === 'loaded' }, + ); + }, + transcribe(audio) { + return client.request( + { type: 'transcribe', audio }, + { + transfer: [audio.buffer], + done: (data) => data.type === 'transcript', + }, + ); + }, + terminate() { + client.terminate(); + }, + }; +} + +/** Kokoro speech synthesis client backed by a worker. */ +export function createKokoroClient({ + voice = TTS_MODEL.voice, + speed = 1, +} = {}) { + const client = createWorkerClient( + new Worker(new URL('./workers/tts.worker.js', import.meta.url), { + type: 'module', + name: 'gev-local-tts', + }), + ); + return { + load({ onProgress } = {}) { + return client.request( + { + type: 'load', + repo: TTS_MODEL.repo, + revision: TTS_MODEL.revision, + dtype: TTS_MODEL.dtype, + voice, + }, + { onProgress, done: (data) => data.type === 'loaded' }, + ); + }, + synthesize(text) { + return client.request( + { type: 'speak', text, voice, speed }, + { done: (data) => data.type === 'audio' }, + ); + }, + flush() { + return client.request({ type: 'flush' }).catch(() => {}); + }, + terminate() { + client.terminate(); + }, + }; +} + +/** Checks WebGPU, adapter limits and reported memory for a model. */ +export async function probeLocalVoiceSupport(model) { + const hasWebGpu = Boolean(globalThis.navigator?.gpu); + let adapter = null; + if (hasWebGpu) { + try { + adapter = await navigator.gpu.requestAdapter(); + } catch { + adapter = null; + } + } + return checkLocalVoiceSupport({ + hasWebGpu, + adapter: Boolean(adapter), + maxBufferSize: adapter?.limits?.maxBufferSize, + deviceMemoryGB: globalThis.navigator?.deviceMemory, + isSecureContext: globalThis.isSecureContext, + model, + }); +} + +async function libraryCacheHas(fragment, cachesApi = globalThis.caches) { + for (const name of LIBRARY_CACHES) { + try { + const cache = await cachesApi.open(name); + const keys = await cache.keys(); + if (keys.some((request) => request.url.includes(fragment))) return true; + } catch { + /* cache unavailable */ + } + } + return false; +} + +/** + * Asks the browser to keep downloaded models and reports when the models + * still to download do not fit, or when storage is not persistent. + * @param {{llm: object, stt: object, ttsEngine: string}} selection + */ +export async function checkModelStorage({ llm, stt, ttsEngine }) { + const storage = globalThis.navigator?.storage; + let persisted; + try { + persisted = (await storage?.persisted?.()) || (await storage?.persist?.()); + } catch { + persisted = undefined; + } + const pending = []; + if (!(await isModelCached(llm.url))) pending.push(llm); + if (!(await libraryCacheHas(`${stt.repo}/resolve/`))) + pending.push({ label: stt.label, bytes: stt.approxBytes }); + if ( + ttsEngine === 'kokoro' && + !(await libraryCacheHas(`${TTS_MODEL.repo}/resolve/`)) + ) + pending.push({ label: 'Kokoro voice', bytes: TTS_MODEL.approxBytes }); + let estimate = null; + try { + estimate = await storage?.estimate?.(); + } catch { + estimate = null; + } + return { + note: storageAdvice(estimate, pending, { + persisted: persisted === undefined ? undefined : Boolean(persisted), + }), + persisted, + usage: estimate?.usage ?? null, + }; +} + +/** Deletes every downloaded on-device voice model from browser storage. */ +export async function clearDownloadedModels(cachesApi = globalThis.caches) { + let removed = 0; + for (const name of [MODEL_CACHE_NAME, ...LIBRARY_CACHES]) { + try { + if (await cachesApi.delete(name)) removed++; + } catch { + /* cache unavailable */ + } + } + return removed; +} + +let sharedEngine = null; + +/** + * The page's on-device engine: workers for the language model, speech + * recognition and (when chosen) Kokoro speech. Loaded models outlive a + * voice session so reopening the microphone is instant. + */ +export function localVoiceEngine() { + if (sharedEngine) return sharedEngine; + const llm = createLlmClient(); + const stt = createSttClient(); + let kokoro = null; + const engine = { + llm, + stt, + get kokoro() { + return kokoro; + }, + llmId: null, + sttId: null, + kokoroReady: false, + prepared: false, + loads: [], + pending: null, + /** + * Loads whatever is missing, then prefills the tool prompt once. + * @param {{llmModel: object, sttModel: object, ttsEngine: string, + * system: string, tools: object[], onProgress?: Function, + * onPhase?: Function}} options + */ + async ensure(options) { + // A second start while models load waits for the first, then only + // loads what is still missing. + while (engine.pending) await engine.pending.catch(() => {}); + engine.pending = load(options); + try { + return await engine.pending; + } finally { + engine.pending = null; + } + }, + terminate() { + llm.terminate(); + stt.terminate(); + kokoro?.terminate(); + if (sharedEngine === engine) sharedEngine = null; + }, + }; + async function load({ + llmModel, + sttModel, + ttsEngine, + system, + tools, + onProgress, + onPhase, + }) { + const timings = {}; + const jobs = []; + const report = (key) => (event) => onProgress?.(key, event); + if (engine.llmId !== llmModel.id) { + engine.prepared = false; + engine.llmId = null; + jobs.push( + llm.load(llmModel, { onProgress: report('llm') }).then((result) => { + timings.llm = result; + engine.llmId = llmModel.id; + }), + ); + } + if (engine.sttId !== sttModel.id) { + engine.sttId = null; + jobs.push( + stt.load(sttModel, { onProgress: report('stt') }).then((result) => { + timings.stt = result; + engine.sttId = sttModel.id; + }), + ); + } + if (ttsEngine === 'kokoro' && !engine.kokoroReady) { + kokoro ||= createKokoroClient(); + // Natural voice is optional: when it cannot load (its phonemizer + // download or hash check failed, or the weights did not load) the + // session speaks with a system voice instead of failing to start. + const client = kokoro; + jobs.push( + client.load({ onProgress: report('tts') }).then( + (result) => { + timings.tts = result; + engine.kokoroReady = true; + }, + (error) => { + timings.ttsError = error?.message || String(error); + client.terminate(); + if (kokoro === client) kokoro = null; + }, + ), + ); + } + await Promise.all(jobs); + if (!engine.prepared) { + onPhase?.('preparing'); + timings.prepare = await llm.prepare({ system, tools }); + engine.prepared = true; + } + engine.loads.push({ at: Date.now(), llm: llmModel.id, timings }); + return engine; + } + sharedEngine = engine; + return engine; +} diff --git a/src/voice/local/localPrompt.js b/src/voice/local/localPrompt.js new file mode 100644 index 000000000..36a98c339 --- /dev/null +++ b/src/voice/local/localPrompt.js @@ -0,0 +1,22 @@ +/** + * Compact instructions for small on-device models. The full cloud prompt is + * about 4.4k tokens; this profile keeps only the routing rules that change + * which action is called. + */ +export const LOCAL_SYSTEM_PROMPT = [ + "You control God's Eye View, a 3D globe app, by calling tools.", + 'For any request to change the map, camera, layers, style or HUD, call the matching tool. Call several tools when the user asks for several changes. Never invent tool names.', + 'Only reply in words when no tool fits, such as small talk. Reply in one short sentence, no lists, no markdown.', + 'Rules:', + '- "take me to / go to / fly to / show me " = fly_to_location with the place as query.', + '- "zoom in/out a bit" = adjust_camera_zoom. "whole earth / globe view / zoom all the way out" = zoom_to_globe.', + '- "show / turn on " = set_layer_visibility enabled=true; "hide / turn off" = enabled=false. "satellites" always means the satellites layer.', + '- night vision = set_visual_style surveillance; thermal = thermal; "normal look" = normal.', + '- "what am I looking at / what is this / what city is this" = get_entity_context.', + '- how many / which / biggest / fastest / highest questions about flights, ships, fires, earthquakes = analyst_query.', + '- "track / follow " = track_entity. "stop tracking" = stop_tracking.', + '- "follow / track the nearest aircraft" = analyst_query with layers=[flights], sortBy=distance, limit=1, then track_entity with the returned callsign. select_nearest_aircraft only when the user names a place.', + '- "mark / annotate / outline " = annotate_map.', + '- Basemap changes (set_map_stack) only for named imagery such as "Bing aerial" or "OSM".', + '- "turn on the radio" = control_radio action=play.', +].join('\n'); diff --git a/src/voice/local/localRadio.js b/src/voice/local/localRadio.js new file mode 100644 index 000000000..fe7450e2a --- /dev/null +++ b/src/voice/local/localRadio.js @@ -0,0 +1,64 @@ +import { startPreparedRadioAfterPlaybackReady } from '../realtimeProtocol.js'; + +/** + * Radio ownership for the on-device voice. The shared runner prepares a + * station without playing it; once the spoken confirmation has finished, + * start() plays it muted, and only when playback is confirmed stops voice, + * which releases the mute. Any newer turn or stop cancels the handoff. + */ +export function createLocalRadioHandoff({ radioLayer = null } = {}) { + let epoch = 0; + let attempt = null; + const stopAttempt = (attemptId) => + radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId }); + return { + get inFlight() { + return attempt !== null; + }, + /** Silences Radio while voice speaks, unless voice is handing off to it. */ + silenceForVoice() { + if (attempt) return false; + radioLayer?.setVoiceDucked?.(true); + return radioLayer?.pause?.({ origin: 'voice-duck' }) || false; + }, + /** Releases the voice mute; called when voice stops. */ + release() { + radioLayer?.setVoiceDucked?.(false); + }, + cancel() { + epoch++; + if (!attempt) return; + const attemptId = attempt; + attempt = null; + stopAttempt(attemptId); + }, + /** + * @param {object} result A control_radio result with radioPlaybackRequested. + * @param {{isCurrent: () => boolean, stopVoice: () => void}} hooks + */ + async start(result, { isCurrent = () => true, stopVoice }) { + const handoff = ++epoch; + const attemptId = `voice-local-radio-${handoff}`; + attempt = attemptId; + radioLayer?.setVoiceDucked?.(true); + const outcome = await startPreparedRadioAfterPlaybackReady(result, { + prepareRadio: () => radioLayer?.playForVoice?.({ attemptId }), + stopVoice, + cancelRadio: () => stopAttempt(attemptId), + isCurrent: () => handoff === epoch && Boolean(isCurrent()), + }); + if (attempt === attemptId) attempt = null; + return outcome; + }, + }; +} + +/** The last prepared Radio playback among a turn's action results. */ +export function pendingRadioPlayback(executed = []) { + return ( + executed + .map((entry) => entry.result) + .filter((result) => result?.ok && result.radioPlaybackRequested) + .at(-1) || null + ); +} diff --git a/src/voice/local/localRadio.test.mjs b/src/voice/local/localRadio.test.mjs new file mode 100644 index 000000000..768c6a39f --- /dev/null +++ b/src/voice/local/localRadio.test.mjs @@ -0,0 +1,148 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { createLocalRadioHandoff, pendingRadioPlayback } from './localRadio.js'; +import { + createSilentOutput, + createSystemOutput, + pickOnDeviceVoice, +} from './speechOutput.js'; + +function fakeRadio({ plays = true, delay = 0 } = {}) { + const calls = []; + return { + calls, + setVoiceDucked: (ducked) => calls.push(['ducked', ducked]), + pause: (options) => { + calls.push(['pause', options.origin]); + return true; + }, + playForVoice: async ({ attemptId }) => { + calls.push(['play', attemptId]); + await new Promise((resolve) => setTimeout(resolve, delay)); + return plays; + }, + stopPlayback: ({ attemptId }) => calls.push(['stop', attemptId]), + }; +} + +const prepared = { + ok: true, + action: 'control_radio', + radioPlaybackRequested: true, +}; + +test('a prepared station plays muted, then voice stops and releases it', async () => { + const radio = fakeRadio(); + const handoff = createLocalRadioHandoff({ radioLayer: radio }); + let stopped = 0; + const outcome = await handoff.start(prepared, { + stopVoice: () => { + stopped++; + handoff.release(); + }, + }); + assert.equal(outcome.result.ok, true); + assert.equal(outcome.result.audioState, 'playing'); + assert.equal(stopped, 1); + assert.deepEqual(radio.calls, [ + ['ducked', true], + ['play', 'voice-local-radio-1'], + ['ducked', false], + ]); +}); + +test('a newer turn cancels the handoff and stops only its attempt', async () => { + const radio = fakeRadio({ delay: 20 }); + const handoff = createLocalRadioHandoff({ radioLayer: radio }); + let stopped = 0; + const pending = handoff.start(prepared, { stopVoice: () => stopped++ }); + assert.equal(handoff.inFlight, true); + assert.equal(handoff.silenceForVoice(), false, 'voice does not pause it'); + handoff.cancel(); + const outcome = await pending; + assert.equal(outcome.cancelled, true); + assert.equal(stopped, 0); + assert.deepEqual( + radio.calls.filter(([name]) => name === 'stop'), + [ + ['stop', 'voice-local-radio-1'], + ['stop', 'voice-local-radio-1'], + ], + ); +}); + +test('a station that fails to start keeps voice on and reports the error', async () => { + const radio = fakeRadio({ plays: false }); + const handoff = createLocalRadioHandoff({ radioLayer: radio }); + let stopped = 0; + const outcome = await handoff.start(prepared, { + isCurrent: () => true, + stopVoice: () => stopped++, + }); + assert.equal(stopped, 0); + assert.equal(outcome.cancelled, false); + assert.equal(outcome.result.ok, false); + assert.equal(outcome.result.audioState, 'error'); +}); + +test('voice speech silences Radio when no handoff is running', () => { + const radio = fakeRadio(); + const handoff = createLocalRadioHandoff({ radioLayer: radio }); + assert.equal(handoff.silenceForVoice(), true); + assert.deepEqual(radio.calls, [ + ['ducked', true], + ['pause', 'voice-duck'], + ]); + assert.equal(pendingRadioPlayback([{ result: { ok: true } }]), null); + assert.equal( + pendingRadioPlayback([ + { result: { ok: false, radioPlaybackRequested: true } }, + { result: prepared }, + ]), + prepared, + ); +}); + +test('the system voice uses only on-device voices', async () => { + assert.equal( + pickOnDeviceVoice([{ lang: 'en-US', localService: false }]), + null, + ); + const local = { lang: 'en-GB', localService: true }; + assert.equal(pickOnDeviceVoice([local]), local); + const spoken = []; + let cancelled = 0; + const synthesis = { + speak: (utterance) => spoken.push(utterance), + cancel: () => cancelled++, + }; + class Utterance { + constructor(text) { + this.text = text; + } + } + const output = createSystemOutput({ synthesis, Utterance, voice: local }); + await output.speak('One.'); + await output.speak('Two.'); + assert.deepEqual( + spoken.map((utterance) => [utterance.text, utterance.voice]), + [ + ['One.', local], + ['Two.', local], + ], + ); + let idle = false; + const waiting = output.idle().then(() => { + idle = true; + }); + spoken[0].onend(); + await Promise.resolve(); + assert.equal(idle, false); + spoken[1].onend(); + await waiting; + output.stop(); + assert.equal(cancelled, 1); + const silent = createSilentOutput(); + assert.equal(await silent.speak('x'), null); +}); diff --git a/src/voice/local/localRuntime.test.mjs b/src/voice/local/localRuntime.test.mjs new file mode 100644 index 000000000..b70e69c42 --- /dev/null +++ b/src/voice/local/localRuntime.test.mjs @@ -0,0 +1,106 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +import { resampleTo16k } from './localAudio.js'; +import { LITERT_VERSION, createWorkerClient } from './workerClient.js'; + +class FakeWorker extends EventTarget { + constructor() { + super(); + this.sent = []; + this.terminated = false; + } + postMessage(message) { + this.sent.push(message); + } + reply(data) { + this.dispatchEvent(Object.assign(new Event('message'), { data })); + } + crash(message) { + this.dispatchEvent(Object.assign(new Event('error'), { message })); + } + terminate() { + this.terminated = true; + } +} + +test('the LiteRT runtime version matches the installed package', () => { + const installed = JSON.parse( + readFileSync( + new URL( + '../../../node_modules/@litert-lm/core/package.json', + import.meta.url, + ), + 'utf8', + ), + ); + assert.equal(LITERT_VERSION, installed.version); +}); + +test('worker requests resolve on their own reply and report progress', async () => { + const worker = new FakeWorker(); + const client = createWorkerClient(worker); + const progress = []; + const first = client.request( + { type: 'load' }, + { + onProgress: (event) => progress.push(event.loaded), + done: (data) => data.type === 'loaded', + }, + ); + const second = client.request({ type: 'other' }); + const [a, b] = worker.sent; + worker.reply({ type: 'progress', id: a.id, loaded: 5 }); + worker.reply({ type: 'ok', id: b.id }); + worker.reply({ type: 'loaded', id: a.id, ms: 1 }); + assert.deepEqual(await second, { type: 'ok', id: b.id }); + assert.deepEqual(await first, { type: 'loaded', id: a.id, ms: 1 }); + assert.deepEqual(progress, [5]); + const failing = client.request({ type: 'x' }); + worker.reply({ type: 'error', id: worker.sent[2].id, message: 'nope' }); + await assert.rejects(failing, /nope/); +}); + +test('a worker failure rejects pending requests and open streams', async () => { + const worker = new FakeWorker(); + const client = createWorkerClient(worker); + const pending = client.request({ type: 'load' }); + let streamError = null; + client.onFailure((error) => { + streamError = error; + }); + worker.crash('boom'); + await assert.rejects(pending, /boom/); + assert.match(streamError.message, /boom/); + await assert.rejects(client.request({ type: 'again' }), /boom/); + let late = null; + client.onFailure((error) => { + late = error; + }); + assert.match(late.message, /boom/); +}); + +test('terminating a worker settles its streams as aborted', async () => { + const worker = new FakeWorker(); + const client = createWorkerClient(worker); + const pending = client.request({ type: 'turn' }); + let reason = null; + client.onFailure((error) => { + reason = error; + }); + client.terminate(); + assert.equal(worker.terminated, true); + await assert.rejects(pending, (error) => error.name === 'AbortError'); + assert.equal(reason.name, 'AbortError'); + client.post({ type: 'ignored' }); + assert.equal(worker.sent.length, 1); +}); + +test('microphone audio is resampled to 16 kHz by averaging', () => { + const samples = Float32Array.from({ length: 48 }, (_, index) => index % 3); + const out = resampleTo16k(samples, 48000); + assert.equal(out.length, 16); + assert.ok(out.every((value) => Math.abs(value - 1) < 1e-6)); + assert.equal(resampleTo16k(samples, 16000), samples); +}); diff --git a/src/voice/local/localStreaming.test.mjs b/src/voice/local/localStreaming.test.mjs new file mode 100644 index 000000000..195d52c60 --- /dev/null +++ b/src/voice/local/localStreaming.test.mjs @@ -0,0 +1,43 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { createMarkupFilter } from './gemmaToolCalls.js'; +import { createSentenceSplitter } from './speechText.js'; + +function speak(chunks) { + const filter = createMarkupFilter(); + const splitter = createSentenceSplitter({ minChars: 4 }); + const spoken = []; + for (const chunk of chunks) spoken.push(...splitter.push(filter.push(chunk))); + spoken.push(...splitter.push(filter.flush()), ...splitter.flush()); + return spoken; +} + +test('streamed answers keep the spaces between chunks', () => { + assert.deepEqual(speak(['There', ' are', ' twelve', ' aircraft', '.']), [ + 'There are twelve aircraft.', + ]); +}); + +test('markup split across chunks never reaches speech', () => { + assert.deepEqual( + speak([ + 'Twelve', + ' found. <|tool_', + 'call>call:x{a:<|"|>b<|"|>} Done', + '', + ]), + ['Twelve found.', 'Done'], + ); +}); + +test('an unclosed region is dropped and stray angle brackets survive', () => { + const filter = createMarkupFilter(); + assert.equal(filter.push('Hi <|channel>thinking'), 'Hi '); + assert.equal(filter.flush(), ''); + const text = createMarkupFilter(); + assert.equal(text.push('a < b and c <'), 'a < b and c '); + assert.equal(text.flush(), '<'); +}); diff --git a/src/voice/local/localVoice.test.mjs b/src/voice/local/localVoice.test.mjs new file mode 100644 index 000000000..80593bb05 --- /dev/null +++ b/src/voice/local/localVoice.test.mjs @@ -0,0 +1,424 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { GEV_ACTION_SCHEMAS } from '../actionSchemas.js'; +import { + CORE_ACTIONS, + LOCAL_ACTIONS, + LOCAL_EXCLUDED_ACTIONS, + MORE_ACTIONS_TOOL, + buildLocalTools, + coerceArguments, + describeAction, + describeMoreActions, + normalizeSchema, +} from './toolset.js'; +import { + formatGemmaToolCall, + parseGemmaToolCalls, + stripGemmaMarkup, +} from './gemmaToolCalls.js'; +import { + actionReply, + combineReplies, + compactResultForModel, + fallbackAnswer, + needsSpokenAnswer, + progressReply, +} from './replies.js'; +import { + createSentenceSplitter, + normalizeForSpeech, + spokenExcerpt, +} from './speechText.js'; +import { + LLM_MODELS, + checkLocalVoiceSupport, + findLlmModel, + recommendLlmModel, + storageAdvice, +} from './modelCatalog.js'; +import { LOCAL_SYSTEM_PROMPT } from './localPrompt.js'; + +const ENGINE_KEYS = new Set([ + 'type', + 'description', + 'properties', + 'required', + 'items', + 'enum', +]); + +function assertEngineSchema(schema, path = 'schema') { + assert.ok(schema.type, `${path} has a type`); + for (const key of Object.keys(schema)) + assert.ok(ENGINE_KEYS.has(key), `${path} drops ${key}`); + for (const [key, child] of Object.entries(schema.properties || {})) + assertEngineSchema(child, `${path}.${key}`); + if (schema.items) assertEngineSchema(schema.items, `${path}[]`); +} + +test('every action normalizes to the engine schema subset in both profiles', () => { + for (const { name } of GEV_ACTION_SCHEMAS) { + assertEngineSchema( + describeAction(name, { profile: 'full' }).parameters, + name, + ); + assertEngineSchema(describeAction(name).parameters, name); + } +}); + +test('untyped filter values become strings instead of breaking templates', () => { + const value = normalizeSchema({ type: 'object', properties: { value: {} } }); + assert.equal(value.properties.value.type, 'string'); + const tool = describeAction('analyst_query', { profile: 'full' }); + assert.equal( + tool.parameters.properties.filters.items.properties.value.type, + 'string', + ); +}); + +test('the local tool set declares every action it supports once', () => { + const tools = buildLocalTools(); + assert.ok(tools.every((tool) => tool.type === 'function')); + const names = tools.map((tool) => tool.function.name); + assert.deepEqual(names, [...LOCAL_ACTIONS]); + assert.deepEqual( + GEV_ACTION_SCHEMAS.map((schema) => schema.name) + .filter((name) => !names.includes(name)) + .sort(), + [...LOCAL_EXCLUDED_ACTIONS].sort(), + ); +}); + +test('a hierarchical tool set adds one index for the remaining actions', () => { + const tools = buildLocalTools({ core: CORE_ACTIONS }); + assert.equal(tools.length, CORE_ACTIONS.length + 1); + const names = tools.map((tool) => tool.function.name); + assert.deepEqual(names.slice(0, -1), [...CORE_ACTIONS]); + assert.equal(names.at(-1), MORE_ACTIONS_TOOL); + const indexed = describeMoreActions().parameters.properties.action.enum; + assert.equal(indexed.length, LOCAL_ACTIONS.length - CORE_ACTIONS.length); + assert.ok(indexed.includes('control_radio')); + assert.ok(!indexed.includes('fly_to_location')); +}); + +test('compact projections keep a valid subset of the canonical arguments', () => { + const annotate = describeAction('annotate_map').parameters; + assert.deepEqual(Object.keys(annotate.properties), ['annotations', 'flyTo']); + assert.deepEqual( + Object.keys(annotate.properties.annotations.items.properties), + ['type', 'target', 'label'], + ); + assert.deepEqual(annotate.properties.annotations.items.required, ['type']); + const fly = describeAction('fly_to_location').parameters; + assert.deepEqual(Object.keys(fly.properties), ['query', 'rangeM']); + assert.deepEqual(fly.required, ['query']); + assert.match( + describeAction('set_layer_visibility').parameters.properties.layerId + .description, + /local-firms/, + ); +}); + +test('the compact prompt stays small', () => { + const chars = + LOCAL_SYSTEM_PROMPT.length + JSON.stringify(buildLocalTools()).length; + // The whole prefix is prefilled once per session and cloned per turn; + // about four characters per token keeps it near 5k tokens. + assert.ok(chars < 20000, `prompt is ${chars} chars`); +}); + +test('arguments are coerced toward the canonical schema', () => { + assert.deepEqual( + coerceArguments('set_layer_visibility', { + layerId: 'Local_Firms', + enabled: 'true', + bogus: 1, + }), + { layerId: 'local-firms', enabled: true }, + ); + assert.deepEqual( + coerceArguments('fly_to_location', { query: 'Tokyo', rangeM: '2,000' }), + { + query: 'Tokyo', + rangeM: 2000, + }, + ); + const query = coerceArguments('analyst_query', { + layers: 'flights', + filters: [{ field: 'altitudeM', op: 'gt', value: '12192' }], + }); + assert.deepEqual(query.layers, ['flights']); + assert.equal(query.filters[0].value, 12192); + const identifiers = coerceArguments('analyst_query', { + filters: [ + { field: 'icao24', op: 'eq', value: '001234' }, + { field: 'onGround', op: 'eq', value: 'false' }, + { field: 'mystery', op: 'eq', value: '42' }, + { field: 'callsign', op: 'eq', value: 123 }, + ], + }); + assert.deepEqual( + identifiers.filters.map((filter) => filter.value), + ['001234', false, '42', '123'], + ); + assert.deepEqual( + normalizeSchema({ type: 'number', enum: [1, 2] }).enum, + [1, 2], + ); + assert.deepEqual(coerceArguments('zoom_to_globe', null), {}); +}); + +test('Gemma tool calls parse from raw model text', () => { + const args = { + layerId: 'satellites', + enabled: false, + nested: { list: [1, 'two', true], text: 'a, b: c' }, + }; + const raw = `Sure.${formatGemmaToolCall('set_layer_visibility', args)}<|tool_response>`; + const parsed = parseGemmaToolCalls(raw); + assert.deepEqual(parsed.calls, [ + { name: 'set_layer_visibility', arguments: args }, + ]); + assert.equal(parsed.text, 'Sure.'); + const two = parseGemmaToolCalls( + '<|tool_call>call:zoom_to_globe{}<|tool_call>call:set_hud{visible:<|"|>off<|"|>}', + ); + assert.deepEqual( + two.calls.map((call) => call.name), + ['zoom_to_globe', 'set_hud'], + ); + assert.deepEqual(two.calls[1].arguments, { visible: 'off' }); + assert.deepEqual( + parseGemmaToolCalls('<|tool_call>call:stop_tracking').calls, + [{ name: 'stop_tracking', arguments: {} }], + ); + assert.deepEqual( + parseGemmaToolCalls('<|tool_call>call:x{a:<|"|>unterminated').calls, + [], + ); + assert.equal( + stripGemmaMarkup('<|channel>thought\nhmmFlights on.'), + 'Flights on.', + ); +}); + +test('action replies confirm the resulting state tersely', () => { + assert.equal( + actionReply( + 'set_layer_visibility', + { layerId: 'local-firms', enabled: true }, + { ok: true, layerId: 'local-firms', enabled: true }, + ), + 'Fires on.', + ); + assert.equal( + actionReply( + 'fly_to_location', + { query: 'tokyo' }, + { ok: true, label: 'Tokyo, Japan' }, + ), + 'Flying to Tokyo.', + ); + assert.equal( + actionReply( + 'set_visual_style', + { style: 'surveillance' }, + { ok: true, style: 'surveillance' }, + ), + 'Night vision.', + ); + assert.equal( + actionReply( + 'set_hud', + { visible: 'off' }, + { ok: true, hud: { visible: false } }, + ), + 'HUD off.', + ); + assert.equal( + actionReply( + 'track_entity', + { query: 'x' }, + { ok: false, error: 'Nothing matched "UAL999"' }, + ), + 'Nothing matched "UAL999".', + ); + assert.equal( + actionReply('zoom_to_globe', {}, { ok: false, cancelled: true }), + null, + ); + assert.equal( + combineReplies(['HUD off.', null, 'Flying to Paris.']), + 'HUD off. Flying to Paris.', + ); + assert.equal( + actionReply( + 'set_map_stack', + { stack: 'bing-aerial' }, + { ok: true, requested: 'bing-aerial' }, + ), + 'Bing aerial basemap.', + ); + assert.equal( + progressReply('set_layer_visibility', { + layerId: 'local-firms', + enabled: true, + }), + 'Turning on fires.', + ); + assert.equal(progressReply('set_hud', { visible: 'off' }), null); + assert.equal( + actionReply( + 'select_nearest_aircraft', + {}, + { ok: true, label: 'SWA123', location: 'Austin, Texas' }, + ), + 'Selected SWA123 near Austin.', + ); + assert.ok(needsSpokenAnswer('analyst_query')); + assert.ok(!needsSpokenAnswer('set_hud')); +}); + +test('query fallbacks and model payloads stay short', () => { + const result = { + ok: true, + action: 'analyst_query', + count: 46, + scopeLabel: 'flights over Texas', + feedState: 'stale', + items: Array.from({ length: 10 }, (_, index) => ({ + callsign: `AAL${index}`, + altitudeM: 10000.123456, + })), + coverage: { big: true }, + }; + assert.equal( + fallbackAnswer('analyst_query', result), + '46 flights over Texas, stale. Top: AAL0.', + ); + const compact = compactResultForModel(result); + assert.equal(compact.items.length, 3); + assert.equal(compact.items[0].altitudeM, 10000.12); + assert.equal(compact.coverage, undefined); + assert.equal(compact.action, undefined); + const large = { + ok: true, + camera: { latitude: 30.123456, longitude: -97.5 }, + layers: Array.from({ length: 40 }, (_, index) => ({ + id: `layer-${index}`, + detail: { nested: { deeper: 'x'.repeat(300) } }, + })), + }; + const fitted = compactResultForModel(large, { budget: 400 }); + assert.ok(JSON.stringify(fitted).length <= 400); + assert.equal(fitted.ok, true); +}); + +test('sentences stream out of partial model text', () => { + const splitter = createSentenceSplitter({ minChars: 5 }); + assert.deepEqual(splitter.push('There are 46 flights'), []); + assert.deepEqual(splitter.push(' in view. Dr. Smith is'), [ + 'There are 46 flights in view.', + ]); + assert.deepEqual(splitter.push(' here! Done'), ['Dr. Smith is here!']); + assert.deepEqual(splitter.flush(), ['Done']); + assert.equal(spokenExcerpt('One. Two. Three.'), 'One. Two.'); +}); + +test('speech text spells callsigns and drops markup', () => { + assert.equal( + normalizeForSpeech('**Tracking** UAL428 at 12 km'), + 'Tracking U A L four two eight at 12 kilometers', + ); + assert.equal(normalizeForSpeech('HUD off.'), 'HUD off.'); +}); + +test('model support checks explain what is missing', () => { + assert.equal(checkLocalVoiceSupport({ hasWebGpu: false }).ok, false); + assert.match(checkLocalVoiceSupport({ hasWebGpu: false }).reason, /WebGPU/); + assert.equal( + checkLocalVoiceSupport({ hasWebGpu: true, adapter: false }).ok, + false, + ); + const big = findLlmModel('gemma-4-12b'); + assert.equal( + checkLocalVoiceSupport({ + hasWebGpu: true, + adapter: true, + deviceMemoryGB: 8, + model: big, + }).ok, + false, + ); + assert.equal( + checkLocalVoiceSupport({ + hasWebGpu: true, + adapter: true, + deviceMemoryGB: 32, + model: big, + }).ok, + true, + ); + assert.equal(recommendLlmModel({ deviceMemoryGB: 32 }).id, 'gemma-4-12b'); + assert.equal(recommendLlmModel({ deviceMemoryGB: 8 }).id, 'gemma-4-e4b'); + assert.ok(LLM_MODELS.every((model) => model.url.endsWith('-web.litertlm'))); +}); + +test('storage advice warns only when a model cannot be kept', () => { + const model = findLlmModel('gemma-4-12b'); + const speech = { label: 'Kokoro voice', bytes: 1e9 }; + assert.equal(storageAdvice({ quota: 20e9, usage: 3e9 }, [model]), null); + assert.equal(storageAdvice({ quota: 20e9, usage: 17e9 }, []), null); + assert.equal(storageAdvice(null, [model]), null); + assert.equal( + storageAdvice({ quota: 20e9, usage: 13.5e9 }, [model]), + null, + 'the model alone fits', + ); + assert.match( + storageAdvice({ quota: 20e9, usage: 13.5e9 }, [model, speech]), + /6\.5 GB free but Gemma 4 12B, Kokoro voice need 7\.0 GB/, + ); + assert.match( + storageAdvice({ quota: 20e9, usage: 0 }, [model], { persisted: false }), + /may clear downloaded models/, + ); + assert.equal( + storageAdvice({ quota: 20e9, usage: 0 }, [], { persisted: false }), + null, + ); +}); + +test('model downloads are pinned to a revision and a digest', () => { + for (const model of LLM_MODELS) { + assert.doesNotMatch(model.url, /\/resolve\/main\//); + assert.match(model.url, /\/resolve\/[0-9a-f]{40}\//); + assert.match(model.sha256, /^[0-9a-f]{64}$/); + } +}); + +test('edge cases: JSON-quoted arguments, units, long sentences, device limits', () => { + assert.deepEqual( + parseGemmaToolCalls( + '<|tool_call>call:track_entity{"query":"UAL \\"428\\"",<|"|>layerId<|"|>:<|"|>flights<|"|>}', + ).calls[0].arguments, + { query: 'UAL "428"', layerId: 'flights' }, + ); + assert.equal( + normalizeForSpeech('Wind 12 m/s at 30000 ft, 400 kts'), + 'Wind 12 meters per second at 30000 feet, 400 knots', + ); + const splitter = createSentenceSplitter({ minChars: 4, maxChars: 20 }); + const parts = splitter.push('one two three four five six seven eight'); + assert.ok(parts.length >= 1); + assert.ok(parts.every((part) => part.length <= 20)); + assert.equal( + checkLocalVoiceSupport({ hasWebGpu: true, isSecureContext: false }).ok, + false, + ); + assert.equal( + checkLocalVoiceSupport({ hasWebGpu: true, maxBufferSize: 256e6 }).ok, + false, + ); +}); diff --git a/src/voice/local/localVoiceControls.js b/src/voice/local/localVoiceControls.js new file mode 100644 index 000000000..cd8b567af --- /dev/null +++ b/src/voice/local/localVoiceControls.js @@ -0,0 +1,166 @@ +import { syncChipGroup } from '../../ui/chipGroup.js'; +import { UiLifetime } from '../../ui/uiLifetime.js'; +import { localVoiceTrayView } from './localVoicePresentation.js'; + +const setText = (element, text) => { + if (element && element.textContent !== text) element.textContent = text; +}; + +/** + * On-device model settings, rendered into the voice engine tray's slot: + * model, hearing and voice chips, download progress, device and storage + * warnings. Captions and results stay in the shared voice card. Renders the + * provider store; selection goes through actions. While models load or the + * device is refused, the tray opens itself (data-local-tray). + */ +export class LocalVoiceControls { + /** + * @param {object} options + * @param {HTMLElement} options.root The voice control root. + * @param {object} options.store Local voice store (getState, subscribe). + * @param {object} [options.actions] selectModel(id), selectStt(id), + * selectTts(id), clearModels(). + * @param {object} [options.engineControls] The engine tray; supplies the + * slot and shows the model on its toggle. + */ + constructor({ root, store, actions = {}, engineControls = null }) { + this.root = root; + this.store = store; + this.actions = actions; + this.engineControls = engineControls; + this.lifetime = new UiLifetime(); + this.frame = null; + const document = root?.ownerDocument; + if (!document?.createElement) return; + const make = (tag, className, parent, text) => { + const node = document.createElement(tag); + if (className) node.className = className; + if (text) node.textContent = text; + parent?.appendChild(node); + return node; + }; + + const slot = + engineControls?.slot || + root.querySelector?.('.gev-voice-local-slot') || + make('div', 'gev-voice-local-slot', root); + const section = make('div', 'gev-voice-local-settings', slot); + section.setAttribute('role', 'group'); + section.setAttribute('aria-label', 'On-device voice'); + const heading = make('div', 'gev-voice-local-heading', section); + make('span', 'gev-voice-local-kicker', heading, 'ON-DEVICE MODELS'); + const state = make('span', 'gev-voice-local-state', heading); + state.setAttribute('role', 'status'); + const chipRow = (label, aria) => { + const row = make('div', 'gev-voice-local-row', section); + make('span', 'gev-voice-local-label', row, label); + const chips = make('div', 'gev-voice-local-chips', row); + chips.setAttribute('role', 'group'); + chips.setAttribute('aria-label', aria); + return chips; + }; + const models = chipRow('MODEL', 'Language model'); + const stt = chipRow('HEARING', 'Speech recognition model'); + const tts = chipRow('VOICE', 'Speech output'); + const notice = make('p', 'gev-voice-local-notice', section); + const progress = make('div', 'gev-voice-local-progress', section); + const progressLabel = make( + 'span', + 'gev-voice-local-progress-label', + progress, + ); + const bar = make('div', 'gev-voice-local-bar', progress); + const progressBar = make('span', '', bar); + const note = make('div', 'gev-voice-local-note', section); + const storage = make('div', 'gev-voice-local-storage', section); + const clear = make( + 'button', + 'data-toggle-chip gev-voice-local-clear', + storage, + ); + clear.type = 'button'; + const storageMessage = make( + 'span', + 'gev-voice-local-storage-note', + storage, + ); + this.section = section; + root.dataset.voiceProvider = 'local-web'; + this.elements = { + state, + models, + stt, + tts, + notice, + clear, + storageMessage, + progress, + progressLabel, + progressBar, + note, + }; + + const chipClick = (action) => (event) => { + const id = event.target?.closest?.('[data-chip-id]')?.dataset.chipId; + if (id) this.actions[action]?.(id); + }; + this.lifetime.listen(models, 'click', chipClick('selectModel')); + this.lifetime.listen(stt, 'click', chipClick('selectStt')); + this.lifetime.listen(tts, 'click', chipClick('selectTts')); + this.lifetime.listen(clear, 'click', () => { + if (!clear.disabled) this.actions.clearModels?.(); + }); + this.unsubscribe = store.subscribe(() => this.schedule()); + this.render(); + } + + schedule() { + if (this.lifetime.destroyed || this.frame !== null) return; + if (typeof requestAnimationFrame !== 'function') { + this.render(); + return; + } + this.frame = this.lifetime.frame(() => { + this.frame = null; + this.render(); + }); + } + + render() { + if (this.lifetime.destroyed || !this.section) return; + const view = localVoiceTrayView(this.store.getState()); + const el = this.elements; + this.engineControls?.setBadge(view.badge); + if (view.open) this.root.dataset.localTray = 'open'; + else delete this.root.dataset.localTray; + setText(el.state, view.status); + el.state.classList.toggle('error', view.statusError); + syncChipGroup(el.models, view.modelChips); + syncChipGroup(el.stt, view.sttChips); + syncChipGroup(el.tts, view.ttsChips); + el.notice.hidden = !view.notice; + setText(el.notice, view.notice || ''); + setText(el.clear, view.clearLabel); + el.clear.disabled = view.clearDisabled; + el.storageMessage.hidden = !view.storageMessage; + setText(el.storageMessage, view.storageMessage || ''); + el.progress.hidden = view.progressHidden; + setText(el.progressLabel, view.progressLabel); + el.progressBar.style.width = `${view.progressPercent}%`; + el.note.hidden = !view.note; + setText(el.note, view.note || ''); + } + + /** Removes the settings and listeners; the provider keeps its models. */ + destroy() { + this.lifetime.destroy(); + this.unsubscribe?.(); + this.unsubscribe = null; + this.section?.remove(); + this.engineControls?.setBadge(null); + if (this.root?.dataset) { + delete this.root.dataset.localTray; + delete this.root.dataset.voiceProvider; + } + } +} diff --git a/src/voice/local/localVoicePreferences.js b/src/voice/local/localVoicePreferences.js new file mode 100644 index 000000000..498f4cad8 --- /dev/null +++ b/src/voice/local/localVoicePreferences.js @@ -0,0 +1,117 @@ +import { + DEFAULT_STT_ID, + DEFAULT_TTS_ID, + findLlmModel, + findSttModel, + findTtsEngine, + recommendLlmModel, +} from './modelCatalog.js'; +import { naturalVoiceAvailable } from './naturalVoiceSetting.js'; + +export { LOCAL_WEB_VOICE, localWebVoiceRequested } from './localWebFlag.js'; + +export const LOCAL_VOICE_MODEL_STORAGE_KEY = 'godsEyeView.voiceLocal.model'; +export const LOCAL_VOICE_STT_STORAGE_KEY = 'godsEyeView.voiceLocal.stt'; +export const LOCAL_VOICE_TTS_STORAGE_KEY = 'godsEyeView.voiceLocal.tts'; +export const LOCAL_VOICE_NATURAL_NOTICE_KEY = + 'godsEyeView.voiceLocal.naturalNotice'; + +function storageOf(storage) { + if (storage !== undefined) return storage; + try { + return typeof localStorage === 'undefined' ? null : localStorage; + } catch { + return null; + } +} + +function read(storage, key) { + try { + return storageOf(storage)?.getItem(key) ?? null; + } catch { + return null; + } +} + +function write(storage, key, value) { + try { + storageOf(storage)?.setItem(key, value); + } catch { + /* best effort */ + } +} + +/** A speech output choice this build offers, or null. */ +function offeredTts(id, naturalVoice) { + const engine = findTtsEngine(id); + if (!engine || (engine.id === 'kokoro' && !naturalVoice)) return null; + return engine.id; +} + +/** + * Resolves the local voice settings: URL overrides, then stored choices, + * then a model recommended for the reported device memory. Unknown ids fall + * through, so a stale stored value can never select a missing model. When + * the build leaves natural voice out, speech defaults to system voices. + */ +export function readLocalVoiceSettings({ + search = globalThis.location?.search, + storage, + deviceMemoryGB = globalThis.navigator?.deviceMemory, + naturalVoice = naturalVoiceAvailable(), +} = {}) { + let params; + try { + params = new URLSearchParams(search || ''); + } catch { + params = new URLSearchParams(); + } + const llm = + findLlmModel(params.get('voiceModel'))?.id || + findLlmModel(read(storage, LOCAL_VOICE_MODEL_STORAGE_KEY))?.id || + recommendLlmModel({ deviceMemoryGB }).id; + const stt = + findSttModel(params.get('voiceStt'))?.id || + findSttModel(read(storage, LOCAL_VOICE_STT_STORAGE_KEY))?.id || + DEFAULT_STT_ID; + const tts = + offeredTts(params.get('voiceTts'), naturalVoice) || + offeredTts(read(storage, LOCAL_VOICE_TTS_STORAGE_KEY), naturalVoice) || + (naturalVoice ? DEFAULT_TTS_ID : 'system'); + return { llm, stt, tts, naturalVoice }; +} + +/** Persists a model choice. Unknown ids are ignored; returns the stored id. */ +export function writeLocalVoiceModel(id, storage) { + const model = findLlmModel(id); + if (model) write(storage, LOCAL_VOICE_MODEL_STORAGE_KEY, model.id); + return model?.id || null; +} + +/** Persists a speech recognition choice. Returns the stored id. */ +export function writeLocalVoiceStt(id, storage) { + const model = findSttModel(id); + if (model) write(storage, LOCAL_VOICE_STT_STORAGE_KEY, model.id); + return model?.id || null; +} + +/** Persists a speech output choice this build offers. Returns the id. */ +export function writeLocalVoiceTts( + id, + storage, + naturalVoice = naturalVoiceAvailable(), +) { + const chosen = offeredTts(id, naturalVoice); + if (chosen) write(storage, LOCAL_VOICE_TTS_STORAGE_KEY, chosen); + return chosen; +} + +/** Whether the natural voice download notice has been shown before. */ +export function naturalVoiceNoticeSeen(storage) { + return read(storage, LOCAL_VOICE_NATURAL_NOTICE_KEY) === 'seen'; +} + +/** Records that the natural voice download notice was shown. */ +export function markNaturalVoiceNoticeSeen(storage) { + write(storage, LOCAL_VOICE_NATURAL_NOTICE_KEY, 'seen'); +} diff --git a/src/voice/local/localVoicePresentation.js b/src/voice/local/localVoicePresentation.js new file mode 100644 index 000000000..79e5f8f00 --- /dev/null +++ b/src/voice/local/localVoicePresentation.js @@ -0,0 +1,139 @@ +import { + LLM_MODELS, + STT_MODELS, + TTS_ENGINES, + formatBytes, +} from './modelCatalog.js'; + +const BUSY_PHASES = new Set(['checking', 'loading', 'preparing']); + +/** Shown in the tray until natural voice first starts. */ +export const NATURAL_VOICE_NOTICE = + 'Natural voice downloads an open-source speech component (GPL) to this browser.'; + +/** Short chip label for a model, e.g. "12B" for "Gemma 4 12B". */ +export function shortModelLabel(model) { + if (!model) return '--'; + if (model.short) return model.short; + const match = /Gemma 4 (.+)$/.exec(model.label || ''); + return (match ? match[1] : model.label || model.id).toUpperCase(); +} + +function progressLine(progress, models) { + const entries = Object.entries(progress || {}).filter(([, event]) => event); + if (!entries.length) return null; + const active = + entries.find( + ([, event]) => + event.phase !== 'compile' && event.total && event.loaded < event.total, + ) || + entries.find(([, event]) => event.phase === 'compile') || + entries[0]; + const [key, event] = active; + const verb = + event.phase === 'cache' + ? 'LOADING' + : event.phase === 'compile' + ? 'COMPILING' + : 'DOWNLOADING'; + const name = + key === 'llm' + ? models.llm?.label || 'model' + : key === 'stt' + ? models.stt?.label || 'speech recognition' + : 'natural voice'; + const percent = + event.total > 0 + ? Math.round(Math.min(1, event.loaded / event.total) * 100) + : null; + const amount = + event.total > 0 && event.phase !== 'compile' + ? ` · ${formatBytes(event.loaded)} / ${formatBytes(event.total)}` + : ''; + return { label: `${verb} ${name.toUpperCase()}${amount}`, percent }; +} + +/** + * Derives the voice tray's controls from one provider snapshot. + * @param {object} state Local voice store snapshot. + * @param {{llmModels?: object[], sttModels?: object[]}} [catalog] + */ +export function localVoiceTrayView( + state, + { + llmModels = LLM_MODELS, + sttModels = STT_MODELS, + ttsEngines = TTS_ENGINES, + } = {}, +) { + const busy = BUSY_PHASES.has(state.phase); + const llm = llmModels.find((model) => model.id === state.llm) || null; + const stt = sttModels.find((model) => model.id === state.stt) || null; + const unsupported = state.support && state.support.ok === false; + const progress = busy ? progressLine(state.progress, { llm, stt }) : null; + let status; + if (unsupported) status = state.support.reason; + else if (state.phase === 'error') status = state.message || 'Voice failed'; + else if (state.phase === 'checking') status = 'CHECKING THIS DEVICE'; + else if (state.phase === 'preparing') status = 'PREPARING MODEL'; + else if (state.phase === 'loading') status = 'LOADING MODELS'; + else if (state.phase === 'ready') + status = `READY · ${shortModelLabel(llm)} · HOLD SPACE`; + else status = 'OFF · FIRST START DOWNLOADS THE MODEL'; + const pending = + state.phase === 'ready' && + ((state.loadedLlm && state.loadedLlm !== state.llm) || + (state.loadedStt && state.loadedStt !== state.stt) || + (state.loadedTts && state.loadedTts !== state.tts)); + return { + badge: `ON-DEVICE ${shortModelLabel(llm)}`, + open: busy || unsupported || state.phase === 'error', + modelChips: llmModels.map((model) => ({ + id: model.id, + label: shortModelLabel(model), + title: `${model.label} · ${formatBytes(model.bytes)}${model.recommended ? ' · recommended' : ''}${model.experimental ? ' · experimental' : ''}`, + active: model.id === state.llm, + disabled: busy, + state: busy && model.id === state.llm ? 'loading' : 'idle', + })), + sttChips: sttModels.map((model) => ({ + id: model.id, + label: model.label.toUpperCase(), + title: `${model.label} · ${formatBytes(model.approxBytes)}`, + active: model.id === state.stt, + disabled: busy, + })), + ttsChips: ttsEngines + .filter( + (engine) => engine.id !== 'kokoro' || state.naturalVoice !== false, + ) + .map((engine) => ({ + id: engine.id, + label: engine.label.toUpperCase(), + title: engine.note, + active: engine.id === state.tts, + disabled: busy, + })), + notice: + state.tts === 'kokoro' && + state.naturalVoice !== false && + state.naturalNotice + ? NATURAL_VOICE_NOTICE + : null, + clearLabel: state.clearing ? 'DELETING…' : 'DELETE DOWNLOADED MODELS', + clearDisabled: Boolean(state.clearing) || busy || state.phase === 'ready', + storageMessage: state.storageMessage || null, + progressHidden: !progress, + progressLabel: progress?.label || '', + progressPercent: progress?.percent ?? 0, + status, + statusError: Boolean(unsupported || state.phase === 'error'), + note: pending + ? 'Model change applies when voice restarts' + : state.speechNote + ? state.speechNote + : state.microphone === 'unavailable' + ? 'Microphone unavailable · typed commands still work' + : state.storageNote || null, + }; +} diff --git a/src/voice/local/localVoiceStore.js b/src/voice/local/localVoiceStore.js new file mode 100644 index 000000000..3915995c9 --- /dev/null +++ b/src/voice/local/localVoiceStore.js @@ -0,0 +1,58 @@ +/** + * Observable snapshot of the on-device voice provider: selected and loaded + * models, load progress, device support and the last exchange. The session + * adapter writes it; the voice tray renders it. + */ +export function createLocalVoiceStore(initial = {}) { + let state = { + phase: 'idle', + llm: null, + stt: null, + tts: 'kokoro', + loadedLlm: null, + loadedStt: null, + progress: {}, + support: null, + message: null, + microphone: 'unknown', + storageNote: null, + storageMessage: null, + clearing: false, + lastUser: null, + lastReply: null, + ...initial, + }; + const listeners = new Set(); + return { + getState: () => state, + update(patch) { + const next = { ...state, ...patch }; + if (Object.keys(patch).every((key) => Object.is(next[key], state[key]))) + return state; + state = next; + for (const listener of [...listeners]) { + try { + listener(state); + } catch { + /* observers cannot break the provider */ + } + } + return state; + }, + /** Records progress for one component (llm, stt or tts). */ + progress(key, event) { + return this.update({ + progress: event + ? { ...state.progress, [key]: event } + : Object.fromEntries( + Object.entries(state.progress).filter(([name]) => name !== key), + ), + }); + }, + subscribe(listener) { + listeners.add(listener); + listener(state); + return () => listeners.delete(listener); + }, + }; +} diff --git a/src/voice/local/localVoiceUi.test.mjs b/src/voice/local/localVoiceUi.test.mjs new file mode 100644 index 000000000..971029622 --- /dev/null +++ b/src/voice/local/localVoiceUi.test.mjs @@ -0,0 +1,418 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +import { railFixture } from '../../ui/railTestFixture.mjs'; +import { VoiceEngineControls } from '../voiceEngineControls.js'; +import { LocalVoiceControls } from './localVoiceControls.js'; +import { naturalVoiceAvailable } from './naturalVoiceSetting.js'; +import { + localVoiceTrayView, + shortModelLabel, +} from './localVoicePresentation.js'; +import { + LOCAL_VOICE_MODEL_STORAGE_KEY, + LOCAL_VOICE_STT_STORAGE_KEY, + LOCAL_VOICE_TTS_STORAGE_KEY, + localWebVoiceRequested, + readLocalVoiceSettings, + writeLocalVoiceModel, + writeLocalVoiceStt, + writeLocalVoiceTts, +} from './localVoicePreferences.js'; +import { createLocalVoiceStore } from './localVoiceStore.js'; +import { LLM_MODELS, findLlmModel } from './modelCatalog.js'; + +function memoryStorage(initial = {}) { + const values = new Map(Object.entries(initial)); + return { + getItem: (key) => (values.has(key) ? values.get(key) : null), + setItem: (key, value) => values.set(key, String(value)), + values, + }; +} + +test('the provider flag is opt-in through the URL', () => { + assert.equal(localWebVoiceRequested('?voice=local-web'), true); + assert.equal(localWebVoiceRequested('?voice=openai'), false); + assert.equal(localWebVoiceRequested(''), false); +}); + +test('settings resolve URL, then stored choice, then device recommendation', () => { + const storage = memoryStorage({ + [LOCAL_VOICE_MODEL_STORAGE_KEY]: 'gemma-4-e4b', + [LOCAL_VOICE_STT_STORAGE_KEY]: 'whisper-base', + }); + assert.deepEqual( + readLocalVoiceSettings({ search: '', storage, deviceMemoryGB: 32 }), + { + llm: 'gemma-4-e4b', + stt: 'whisper-base', + tts: 'kokoro', + naturalVoice: true, + }, + ); + assert.equal( + readLocalVoiceSettings({ + search: '?voiceModel=gemma-4-12b', + storage, + }).llm, + 'gemma-4-12b', + ); + const empty = memoryStorage({ [LOCAL_VOICE_MODEL_STORAGE_KEY]: 'retired' }); + assert.deepEqual( + readLocalVoiceSettings({ search: '', storage: empty, deviceMemoryGB: 32 }), + { + llm: 'gemma-4-12b', + stt: 'moonshine-base', + tts: 'kokoro', + naturalVoice: true, + }, + ); + assert.equal( + readLocalVoiceSettings({ search: '', storage: null, deviceMemoryGB: 8 }) + .llm, + 'gemma-4-e4b', + ); +}); + +test('only known models are persisted', () => { + const storage = memoryStorage(); + assert.equal(writeLocalVoiceModel('gemma-4-12b', storage), 'gemma-4-12b'); + assert.equal(writeLocalVoiceModel('nope', storage), null); + assert.equal(writeLocalVoiceStt('whisper-base', storage), 'whisper-base'); + assert.equal( + storage.values.get(LOCAL_VOICE_MODEL_STORAGE_KEY), + 'gemma-4-12b', + ); + assert.equal(storage.values.get(LOCAL_VOICE_STT_STORAGE_KEY), 'whisper-base'); + const throwing = { + getItem() { + throw new Error('denied'); + }, + setItem() { + throw new Error('denied'); + }, + }; + assert.equal(writeLocalVoiceModel('gemma-4-e4b', throwing), 'gemma-4-e4b'); + assert.equal( + readLocalVoiceSettings({ search: '', storage: throwing }).stt, + 'moonshine-base', + ); +}); + +test('the store publishes changes and progress per component', () => { + const store = createLocalVoiceStore({ llm: 'gemma-4-e4b' }); + const seen = []; + const unsubscribe = store.subscribe((state) => seen.push(state.phase)); + store.update({ phase: 'loading' }); + store.update({ phase: 'loading' }); + store.progress('llm', { loaded: 1, total: 2, phase: 'download' }); + store.progress('stt', { loaded: 1, total: 1, phase: 'download' }); + store.progress('llm', null); + assert.deepEqual(Object.keys(store.getState().progress), ['stt']); + unsubscribe(); + store.update({ phase: 'ready' }); + assert.deepEqual(seen, ['idle', 'loading', 'loading', 'loading', 'loading']); +}); + +test('the tray view reports download progress and device problems', () => { + assert.equal(shortModelLabel(findLlmModel('gemma-4-26b-a4b')), '26B A4B'); + const loading = localVoiceTrayView({ + phase: 'loading', + llm: 'gemma-4-12b', + stt: 'moonshine-base', + progress: { + llm: { loaded: 3e9, total: 6e9, phase: 'download' }, + tts: { loaded: 1, total: 1, phase: 'download' }, + }, + }); + assert.equal(loading.open, true); + assert.equal(loading.progressHidden, false); + assert.equal(loading.progressPercent, 50); + assert.match( + loading.progressLabel, + /^DOWNLOADING GEMMA 4 12B · 3\.0 GB \/ 6\.0 GB$/, + ); + assert.ok(loading.modelChips.every((chip) => chip.disabled)); + assert.equal(loading.modelChips.find((chip) => chip.active).state, 'loading'); + assert.equal(loading.badge, 'ON-DEVICE 12B'); + + const unsupported = localVoiceTrayView({ + phase: 'error', + llm: 'gemma-4-12b', + support: { ok: false, reason: 'Local voice needs WebGPU.' }, + }); + assert.equal(unsupported.open, true); + assert.equal(unsupported.statusError, true); + assert.equal(unsupported.status, 'Local voice needs WebGPU.'); + + const ready = localVoiceTrayView({ + phase: 'ready', + llm: 'gemma-4-12b', + loadedLlm: 'gemma-4-e4b', + stt: 'moonshine-base', + loadedStt: 'moonshine-base', + lastUser: 'Take me to Tokyo', + lastReply: 'Flying to Tokyo.', + }); + assert.equal(ready.open, false); + assert.equal(ready.progressHidden, true); + assert.equal(ready.note, 'Model change applies when voice restarts'); + assert.equal('lastReply' in ready, false, 'captions live in the voice card'); + assert.equal(ready.modelChips.length, LLM_MODELS.length); +}); + +function mountControls({ engine = 'on-device' } = {}) { + const fixture = railFixture(); + const { document } = fixture; + const root = document.createElement('div'); + root.id = 'gev-voice-control'; + const cost = document.createElement('div'); + cost.className = 'gev-voice-cost'; + root.appendChild(cost); + const selected = []; + const engineControls = new VoiceEngineControls({ + root, + engine, + onSelect: (id) => selected.push(id), + probe: async () => ({ + secure: true, + webgpu: true, + adapter: true, + vendor: 'apple', + architecture: 'metal-3', + maxBufferSize: 4 * 1024 ** 3, + deviceMemoryGB: 16, + }), + }); + const store = createLocalVoiceStore({ + llm: 'gemma-4-e4b', + stt: 'moonshine-base', + }); + const calls = []; + const controls = new LocalVoiceControls({ + root, + store, + engineControls, + actions: { + selectModel: (id) => { + calls.push(['model', id]); + store.update({ llm: id }); + }, + selectStt: (id) => calls.push(['stt', id]), + }, + }); + return { + fixture, + root, + cost, + store, + controls, + engineControls, + calls, + selected, + }; +} + +test('the model settings mount in the engine tray and follow the store', () => { + const { root, cost, store, controls, engineControls, calls } = + mountControls(); + assert.equal(cost.children[0], engineControls.toggle); + assert.equal(controls.section.parent, engineControls.slot); + assert.equal(root.dataset.voiceProvider, 'local-web'); + assert.equal(engineControls.toggle.textContent, 'ON-DEVICE E4B'); + assert.equal(root.dataset.localTray, undefined); + + store.update({ + phase: 'loading', + progress: { llm: { loaded: 1e9, total: 4e9, phase: 'download' } }, + }); + assert.equal(root.dataset.localTray, 'open', 'loading opens the tray'); + assert.equal(controls.elements.progress.hidden, false); + assert.equal(controls.elements.progressBar.style.width, '25%'); + + store.update({ phase: 'ready', progress: {} }); + assert.equal(root.dataset.localTray, undefined); + engineControls.toggle.click(); + assert.equal(root.dataset.voiceTray, 'open'); + assert.equal(engineControls.toggle.getAttribute('aria-expanded'), 'true'); + + const chip = controls.elements.models.children.find( + (node) => node.dataset.chipId === 'gemma-4-12b', + ); + chip.click(); + assert.deepEqual(calls, [['model', 'gemma-4-12b']]); + assert.equal(engineControls.toggle.textContent, 'ON-DEVICE 12B'); + assert.equal(chip.getAttribute('aria-pressed'), 'true'); +}); + +test('the natural voice notice shows until natural voice first starts', () => { + const { store, controls } = mountControls(); + store.update({ tts: 'kokoro', naturalNotice: true }); + assert.equal(controls.elements.notice.hidden, false); + assert.match( + controls.elements.notice.textContent, + /open-source speech component \(GPL\)/, + ); + store.update({ naturalNotice: false }); + assert.equal(controls.elements.notice.hidden, true); + store.update({ tts: 'system', naturalNotice: true }); + assert.equal(controls.elements.notice.hidden, true, 'only for natural voice'); +}); + +test('a build without natural voice offers system voices and text only', () => { + const view = localVoiceTrayView({ + phase: 'idle', + llm: 'gemma-4-e4b', + stt: 'moonshine-base', + tts: 'system', + naturalVoice: false, + naturalNotice: true, + }); + assert.deepEqual( + view.ttsChips.map((chip) => chip.id), + ['system', 'none'], + ); + assert.equal(view.notice, null); + const storage = memoryStorage({ [LOCAL_VOICE_TTS_STORAGE_KEY]: 'kokoro' }); + assert.equal( + readLocalVoiceSettings({ search: '', storage, naturalVoice: false }).tts, + 'system', + ); + assert.equal( + readLocalVoiceSettings({ + search: '?voiceTts=kokoro', + storage: null, + naturalVoice: false, + }).tts, + 'system', + ); + assert.equal(writeLocalVoiceTts('kokoro', storage, false), null); + assert.equal(naturalVoiceAvailable('off'), false); + assert.equal(naturalVoiceAvailable('0'), false); + assert.equal(naturalVoiceAvailable(''), true); + assert.equal(naturalVoiceAvailable(undefined), true); +}); + +test('natural voice falls back to a system voice when it cannot load', async () => { + const { createLocalWebSession } = await import('./localWebSession.js'); + const storage = memoryStorage(); + const session = createLocalWebSession({ + emit: () => {}, + runAction: async () => ({ ok: true }), + ui: null, + storage, + runtime: { + createEngine: () => ({ + prepared: true, + kokoroReady: false, + async ensure() { + return { + kokoroReady: false, + loads: [ + { + timings: { + ttsError: + 'Natural voice component failed its integrity check', + }, + }, + ], + }; + }, + terminate() {}, + }), + probeSupport: async () => ({ ok: true }), + checkStorage: async () => ({ note: null }), + openMicrophone: async () => { + throw new Error('no microphone in tests'); + }, + }, + }); + // No system voices in Node: the fallback ends at text only. + await session.start(); + const state = session.local.store.getState(); + assert.equal(state.tts, 'kokoro'); + assert.match(state.speechNote, /integrity check.*system voice/); + assert.equal(storage.values.size, 0, 'the notice stays until it succeeds'); + assert.equal(state.phase, 'ready'); + session.stop({ removeUi: true }); +}); + +test('destroying the settings removes their nodes, listeners and subscription', () => { + const { root, store, controls, engineControls, calls } = mountControls(); + const chip = controls.elements.models.children[0]; + controls.destroy(); + assert.equal(engineControls.slot.children.length, 0); + assert.equal(root.dataset.voiceProvider, undefined); + assert.equal(engineControls.toggle.textContent, 'ON-DEVICE'); + chip.click(); + store.update({ phase: 'loading' }); + assert.deepEqual(calls, []); + assert.equal(root.dataset.localTray, undefined); + controls.destroy(); + engineControls.destroy(); + assert.equal(root.dataset.voiceEngine, undefined); +}); + +test('model styles load with the provider; the engine tray with every page', () => { + const styles = readFileSync( + new URL('../../../style.css', import.meta.url), + 'utf8', + ); + assert.doesNotMatch(styles, /local-voice\.css/); + assert.match(styles, /voice-engine\.css/); + const provider = readFileSync( + new URL('./localWebProvider.js', import.meta.url), + 'utf8', + ); + assert.match( + provider, + /import\('\.\.\/\.\.\/ui\/styles\/local-voice\.css'\)/, + ); + const css = readFileSync( + new URL('../../ui/styles/voice-engine.css', import.meta.url), + 'utf8', + ); + assert.match( + css, + /#gev-voice-control\[data-voice-tray='open'\] \.gev-voice-engine-tray/, + ); + assert.match( + css, + /#gev-voice-control\[data-local-tray='open'\] \.gev-voice-engine-tray/, + ); + assert.match(css, /dock-has-pinned-tray \.gev-voice-engine-tray/); + assert.match( + css, + /\[data-local-tray='open'\] \.gev-voice-card \{\s*display: none;/, + ); +}); + +test('URL speech override and load verbs', () => { + assert.equal( + readLocalVoiceSettings({ search: '?voiceStt=whisper-base', storage: null }) + .stt, + 'whisper-base', + ); + const base = { phase: 'loading', llm: 'gemma-4-e4b', stt: 'moonshine-base' }; + assert.match( + localVoiceTrayView({ + ...base, + progress: { llm: { loaded: 1, total: 4, phase: 'cache' } }, + }).progressLabel, + /^LOADING GEMMA 4 E4B/, + ); + assert.equal( + localVoiceTrayView({ + ...base, + progress: { llm: { loaded: 0, total: 4, phase: 'compile' } }, + }).progressLabel, + 'COMPILING GEMMA 4 E4B', + ); + assert.equal( + localVoiceTrayView({ ...base, phase: 'ready', storageNote: 'Low space' }) + .note, + 'Low space', + ); +}); diff --git a/src/voice/local/localWebFlag.js b/src/voice/local/localWebFlag.js new file mode 100644 index 000000000..7f7213243 --- /dev/null +++ b/src/voice/local/localWebFlag.js @@ -0,0 +1,10 @@ +export const LOCAL_WEB_VOICE = 'local-web'; + +/** Whether the page asked for the in-browser voice tier (?voice=local-web). */ +export function localWebVoiceRequested(search = globalThis.location?.search) { + try { + return new URLSearchParams(search || '').get('voice') === LOCAL_WEB_VOICE; + } catch { + return false; + } +} diff --git a/src/voice/local/localWebProvider.js b/src/voice/local/localWebProvider.js new file mode 100644 index 000000000..06c245efc --- /dev/null +++ b/src/voice/local/localWebProvider.js @@ -0,0 +1,71 @@ +export { LOCAL_WEB_VOICE, localWebVoiceRequested } from './localWebFlag.js'; + +const cancelled = () => + new DOMException('On-device voice was stopped', 'AbortError'); + +/** + * Session adapter factory for the on-device tier that loads the provider on + * first use, so pages without the flag never download it. A stop issued + * while the module loads cancels the pending start; after removal the + * provider is never constructed and every later start is refused. + */ +export function createLocalWebSessionLoader( + hooks, + { + load = () => + Promise.all([ + import('./localWebSession.js'), + import('../../ui/styles/local-voice.css'), + ]).then(([module]) => module), + } = {}, +) { + let adapter = null; + let bound = false; + let disposed = false; + let stopEpoch = 0; + const isDisposed = () => disposed || Boolean(hooks.signal?.aborted); + const ready = Promise.resolve() + .then(load) + .then((module) => { + if (isDisposed()) return null; + adapter = module.createLocalWebSession(hooks); + if (bound) adapter.bindControls(); + return adapter; + }); + ready.catch(() => {}); + return { + capabilities: { costControls: false, pushToTalk: true, local: true }, + get local() { + return adapter?.local || null; + }, + ready, + async start(options) { + if (isDisposed()) throw cancelled(); + const epoch = stopEpoch; + const loaded = await ready; + if (!loaded || isDisposed()) throw cancelled(); + if (epoch !== stopEpoch) return undefined; + return loaded.start(options); + }, + stop(options = {}) { + stopEpoch++; + if (options.removeUi) disposed = true; + adapter?.stop(options); + }, + sendText(text) { + if (isDisposed()) return false; + void ready.then((loaded) => { + if (loaded && !isDisposed()) loaded.sendText(text); + }); + return true; + }, + sendMapEvent() { + return false; + }, + ignoreButtonClick: () => Boolean(adapter?.ignoreButtonClick()), + bindControls() { + bound = true; + if (!isDisposed()) adapter?.bindControls(); + }, + }; +} diff --git a/src/voice/local/localWebProvider.test.mjs b/src/voice/local/localWebProvider.test.mjs new file mode 100644 index 000000000..48275067c --- /dev/null +++ b/src/voice/local/localWebProvider.test.mjs @@ -0,0 +1,144 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { createVoiceSession } from '../session.js'; +import { + createLocalWebSessionLoader, + localWebVoiceRequested, +} from './localWebProvider.js'; + +function delayedProvider() { + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const created = []; + const load = async () => { + await gate; + return { + createLocalWebSession(hooks) { + const calls = []; + const adapter = { + hooks, + calls, + local: { metrics: [] }, + start: async (options) => calls.push(['start', options]), + stop: (options) => calls.push(['stop', options]), + sendText: (text) => calls.push(['text', text]), + bindControls: () => calls.push(['bind']), + ignoreButtonClick: () => false, + }; + created.push(adapter); + return adapter; + }, + }; + }; + return { load, release, created }; +} + +test('the loader exposes capabilities before the provider module arrives', async () => { + const provider = delayedProvider(); + const loader = createLocalWebSessionLoader({}, { load: provider.load }); + assert.deepEqual(loader.capabilities, { + costControls: false, + pushToTalk: true, + local: true, + }); + assert.equal(loader.local, null); + assert.equal(loader.ignoreButtonClick(), false); + assert.equal(loader.sendMapEvent({}), false); + loader.bindControls(); + assert.equal(loader.sendText('hello'), true); + provider.release(); + const adapter = await loader.ready; + await Promise.resolve(); + assert.deepEqual(adapter.calls, [['bind'], ['text', 'hello']]); + assert.equal(loader.local, adapter.local); +}); + +test('a stop while the module loads cancels the pending start', async () => { + const provider = delayedProvider(); + const loader = createLocalWebSessionLoader({}, { load: provider.load }); + const starting = loader.start({ pushToTalk: true }); + loader.stop(); + provider.release(); + await starting; + const [adapter] = provider.created; + assert.equal( + adapter.calls.some(([name]) => name === 'start'), + false, + ); + await loader.start({ pushToTalk: false }); + assert.deepEqual(adapter.calls.at(-1), ['start', { pushToTalk: false }]); +}); + +test('removal while loading never constructs the provider and refuses starts', async () => { + const provider = delayedProvider(); + const loader = createLocalWebSessionLoader({}, { load: provider.load }); + const starting = loader.start({}); + loader.bindControls(); + loader.stop({ removeUi: true }); + provider.release(); + await assert.rejects(starting, (error) => error.name === 'AbortError'); + assert.equal(await loader.ready, null); + assert.equal(provider.created.length, 0); + await assert.rejects( + loader.start({}), + (error) => error.name === 'AbortError', + ); + assert.equal(loader.sendText('late'), false); +}); + +test('destroying the voice session while loading leaves nothing running', async () => { + const provider = delayedProvider(); + const session = createVoiceSession({ + runner: async () => ({ ok: true }), + createAdapter: (hooks) => + createLocalWebSessionLoader(hooks, { load: provider.load }), + }); + const starting = session.start({ pushToTalk: false }); + session.destroy(); + provider.release(); + await starting; + assert.equal(provider.created.length, 0); + assert.equal(session.disposed, true); + await assert.rejects( + session.adapter.start({}), + (error) => error.name === 'AbortError', + ); +}); + +test('a stopped session can start again once the module has loaded', async () => { + const provider = delayedProvider(); + const session = createVoiceSession({ + runner: async () => ({ ok: true }), + createAdapter: (hooks) => + createLocalWebSessionLoader(hooks, { load: provider.load }), + }); + const starting = session.start({}); + session.stop(); + provider.release(); + await starting; + const [adapter] = provider.created; + assert.equal(adapter.calls.filter(([name]) => name === 'start').length, 0); + await session.start({}); + assert.equal(adapter.calls.filter(([name]) => name === 'start').length, 1); + session.destroy(); +}); + +test('the real provider module satisfies the adapter contract without a UI', async () => { + const loader = createLocalWebSessionLoader( + { emit: () => {}, runAction: async () => ({ ok: true }), ui: null }, + { load: () => import('./localWebSession.js') }, + ); + const adapter = await loader.ready; + for (const method of ['start', 'stop', 'sendText', 'sendMapEvent']) + assert.equal(typeof adapter[method], 'function'); + assert.ok(loader.local?.store); + loader.stop({ removeUi: true }); +}); + +test('the provider is only requested by the URL flag', () => { + assert.equal(localWebVoiceRequested('?voice=local-web&welcome=0'), true); + assert.equal(localWebVoiceRequested('?welcome=0'), false); +}); diff --git a/src/voice/local/localWebSession.js b/src/voice/local/localWebSession.js new file mode 100644 index 000000000..c06a1a13c --- /dev/null +++ b/src/voice/local/localWebSession.js @@ -0,0 +1,772 @@ +import { RealtimeInput } from '../realtimeInput.js'; +import { + createMarkupFilter, + parseGemmaToolCalls, + stripGemmaMarkup, +} from './gemmaToolCalls.js'; +import { createMicrophoneCapture } from './localAudio.js'; +import { + checkModelStorage, + clearDownloadedModels, + localVoiceEngine, + probeLocalVoiceSupport, +} from './localEngine.js'; +import { LOCAL_SYSTEM_PROMPT } from './localPrompt.js'; +import { createLocalRadioHandoff, pendingRadioPlayback } from './localRadio.js'; +import { LocalVoiceControls } from './localVoiceControls.js'; +import { + markNaturalVoiceNoticeSeen, + naturalVoiceNoticeSeen, + readLocalVoiceSettings, + writeLocalVoiceModel, + writeLocalVoiceStt, + writeLocalVoiceTts, +} from './localVoicePreferences.js'; +import { createLocalVoiceStore } from './localVoiceStore.js'; +import { findLlmModel, findSttModel } from './modelCatalog.js'; +import { + actionReply, + combineReplies, + compactResultForModel, + fallbackAnswer, + isMaterialReply, + needsContinuation, + needsSpokenAnswer, + progressReply, +} from './replies.js'; +import { + createKokoroOutput, + createSilentOutput, + createSystemOutput, + loadSystemVoices, + pickOnDeviceVoice, +} from './speechOutput.js'; +import { createSentenceSplitter, normalizeForSpeech } from './speechText.js'; +import { + ACTION_SCHEMA_BY_NAME, + buildLocalTools, + coerceArguments, +} from './toolset.js'; + +const MIN_UTTERANCE_SAMPLES = 16000 * 0.25; +const READY_HINT = 'Hold Space to talk'; +const SLOW_ACTION_MS = 700; +const MAX_TOOL_ROUNDS = 2; + +function rootMeanSquare(samples) { + let sum = 0; + for (const sample of samples) sum += sample * sample; + return samples.length ? Math.sqrt(sum / samples.length) : 0; +} + +/** + * Voice session adapter for the zero-install tier: push-to-talk capture, + * speech recognition, a Gemma 4 tool-calling model and a swappable speech + * output, all on this device. Actions run through the shared session runner. + */ +export function createLocalWebSession({ + emit, + runAction, + ui, + radioLayer = null, + viewer = null, + storage, + runtime = {}, +}) { + // The on-device runtime; tests substitute an engine, probes and outputs. + const { + createEngine = localVoiceEngine, + probeSupport = probeLocalVoiceSupport, + checkStorage = checkModelStorage, + openMicrophone = createMicrophoneCapture, + createOutput = null, + } = runtime; + const settings = readLocalVoiceSettings({ storage }); + const store = createLocalVoiceStore({ + llm: settings.llm, + stt: settings.stt, + tts: settings.tts, + naturalVoice: settings.naturalVoice, + // The one-line download notice shows until natural voice first starts. + naturalNotice: !naturalVoiceNoticeSeen(storage), + }); + const metrics = []; + let engine = null; + let status = 'idle'; + let active = false; + let startGeneration = 0; + let turnGeneration = 0; + let current = null; + let mic = null; + let speechChain = Promise.resolve(); + let lastUtterance = null; + + const radio = createLocalRadioHandoff({ radioLayer }); + let speech = createSilentOutput(); + const speakerHooks = { + onSpeaking: () => ui && input.setVoiceSpeaker('ai'), + onIdle: () => { + if (ui && !mic?.recording) input.setVoiceSpeaker('idle'); + }, + }; + + const input = new RealtimeInput({ + readUi: () => ui, + readStream: () => mic?.stream || null, + readStatus: () => status, + operations: { + isActive: () => active, + setStatus, + start: (options) => startFromGesture(options), + pauseRadioForVoice: () => radio.silenceForVoice(), + setMicrophoneEnabled: (enabled) => { + if (enabled) beginRecording(); + else void endRecording(); + }, + }, + }); + + let controls = null; + const actions = { + selectModel(id) { + const chosen = writeLocalVoiceModel(id, storage); + if (chosen) store.update({ llm: chosen }); + }, + selectStt(id) { + const chosen = writeLocalVoiceStt(id, storage); + if (chosen) store.update({ stt: chosen }); + }, + selectTts(id) { + const chosen = writeLocalVoiceTts(id, storage); + if (chosen) store.update({ tts: chosen }); + }, + async clearModels() { + if (active || store.getState().clearing) return; + store.update({ clearing: true, storageMessage: null }); + engine?.terminate(); + engine = null; + const removed = await clearDownloadedModels().catch(() => 0); + store.update({ + clearing: false, + loadedLlm: null, + loadedStt: null, + loadedTts: null, + storageNote: null, + storageMessage: removed + ? 'Downloaded models deleted.' + : 'No downloaded models found.', + }); + }, + }; + + async function createSpeech(ttsEngine) { + if (createOutput) return createOutput(ttsEngine, speakerHooks); + if (ttsEngine === 'kokoro' && engine?.kokoroReady && engine.kokoro) + return createKokoroOutput({ client: engine.kokoro, ...speakerHooks }); + // Natural voice that could not load falls back to a system voice. + if (ttsEngine === 'system' || ttsEngine === 'kokoro') { + const voice = pickOnDeviceVoice(await loadSystemVoices()); + if (voice) return createSystemOutput({ voice, ...speakerHooks }); + store.update({ + storageMessage: 'No on-device system voice; replies are text only.', + }); + } + return createSilentOutput(); + } + + function setStatus(state, detail) { + status = state; + emit({ type: 'state', state, detail }); + } + + function isCurrentTurn(id) { + return active && id === turnGeneration; + } + + async function ensureModels(attempt) { + const { llm, stt, tts } = store.getState(); + const llmModel = findLlmModel(llm); + const sttModel = findSttModel(stt); + const current = () => attempt === startGeneration && active; + engine = createEngine(); + store.update({ phase: 'loading', progress: {} }); + const loaded = await engine.ensure({ + llmModel, + sttModel, + ttsEngine: tts, + system: LOCAL_SYSTEM_PROMPT, + tools: buildLocalTools(), + onProgress: (key, event) => { + if (!current()) return; + store.progress(key, event); + if (event.total) + setStatus( + 'connecting', + `Loading ${Math.round((event.loaded / event.total) * 100)}%`, + ); + }, + onPhase: (phase) => { + if (!current()) return; + store.update({ phase, progress: {} }); + setStatus('connecting', 'Preparing model'); + }, + }); + if (!current()) return engine; + const timings = loaded.loads.at(-1)?.timings || {}; + const cache = timings.llm?.cache; + if (tts === 'kokoro' && timings.ttsError) + store.update({ + speechNote: `Natural voice unavailable (${timings.ttsError}); using a system voice.`, + }); + else if (tts === 'kokoro' && loaded.kokoroReady) { + markNaturalVoiceNoticeSeen(storage); + store.update({ speechNote: null }); + } + store.update({ + loadedLlm: llmModel.id, + loadedStt: sttModel.id, + loadedTts: tts, + progress: {}, + ...(cache && !cache.ok + ? { + storageMessage: `${llmModel.label} was not kept for next time: ${cache.error}`, + } + : {}), + }); + return engine; + } + + async function start({ pushToTalk = false } = {}) { + if (active) return; + const attempt = ++startGeneration; + active = true; + input.pushToTalkMode = true; + setStatus('connecting', 'Checking this device'); + store.update({ phase: 'checking', message: null, support: null }); + const support = await probeSupport(findLlmModel(store.getState().llm)); + if (attempt !== startGeneration) return; + store.update({ support }); + if (!support.ok) { + active = false; + store.update({ phase: 'error', message: support.reason }); + throw new Error(support.reason); + } + setStatus('connecting', 'Loading local models'); + try { + const { llm, stt, tts } = store.getState(); + const storageCheck = await checkStorage({ + llm: findLlmModel(llm), + stt: findSttModel(stt), + ttsEngine: tts, + }); + if (attempt !== startGeneration) return; + store.update({ storageNote: storageCheck.note, storageMessage: null }); + await ensureModels(attempt); + const output = await createSpeech(store.getState().tts); + if (attempt !== startGeneration || !active) { + output.close(); + return; + } + speech.close(); + speech = output; + } catch (error) { + if (attempt !== startGeneration) return; + store.update({ + phase: 'error', + message: error?.message || String(error), + progress: {}, + }); + throw error; + } + if (attempt !== startGeneration || !active) return; + try { + mic = await openMicrophone(); + input.startVoiceVisualizer(mic.stream); + if (speech.stream) input.startAssistantVoiceVisualizer(speech.stream); + store.update({ microphone: 'ready' }); + } catch { + mic = null; + store.update({ microphone: 'unavailable' }); + } + if (attempt !== startGeneration || !active) { + mic?.close(); + mic = null; + return; + } + store.update({ phase: 'ready' }); + setStatus('listening', READY_HINT); + if (ui) input.updateVoiceButtonLabel(); + if (pushToTalk && input.pushToTalkKeyHeld) beginRecording(); + } + + async function startFromGesture(options) { + const attempt = startGeneration + 1; + try { + await start(options); + } catch (error) { + if (attempt !== startGeneration) return; + stop({ preserveStatus: true }); + setStatus('error', error?.message || String(error)); + } + } + + function interrupt() { + const running = current; + current = null; + turnGeneration++; + if (running?.llmTurnId) engine?.llm.cancel(running.llmTurnId); + radio.cancel(); + speech.stop(); + speechChain = Promise.resolve(); + if (running) emit({ type: 'interruption' }); + } + + function beginRecording() { + if (!active || !mic || status === 'connecting') return; + interrupt(); + mic.start(); + if (ui?.root) ui.root.dataset.microphone = 'active'; + if (ui) input.setVoiceSpeaker('user'); + setStatus('listening', 'Release Space to send'); + } + + async function endRecording() { + if (!mic?.recording) return; + const endOfInput = performance.now(); + const audio = await mic.stop(); + if (ui?.root) ui.root.dataset.microphone = 'muted'; + if (ui) input.setVoiceSpeaker('idle'); + lastUtterance = { samples: audio.length, rms: rootMeanSquare(audio) }; + if (!active) return; + if (audio.length < MIN_UTTERANCE_SAMPLES) { + setStatus('listening', READY_HINT); + return; + } + await transcribeAndRun(audio, endOfInput); + } + + async function transcribeAndRun(audio, endOfInput = performance.now()) { + if (!active || !engine?.prepared) return null; + interrupt(); + const turn = turnGeneration; + setStatus('executing', 'Transcribing'); + let transcript; + try { + transcript = await engine.stt.transcribe(audio); + } catch (error) { + if (isCurrentTurn(turn)) setStatus('listening', 'Could not transcribe'); + return null; + } + if (!isCurrentTurn(turn)) return null; + if (!transcript.text) { + setStatus('listening', 'Nothing heard, try again'); + return null; + } + return runTurn(transcript.text, { + endOfInput, + stt: { ms: transcript.ms, audioMs: transcript.audioMs }, + source: 'voice', + }); + } + + function speak(text, record, turn) { + const clean = normalizeForSpeech(text); + if (!clean) return; + const splitter = createSentenceSplitter({ minChars: 8 }); + for (const sentence of [...splitter.push(clean), ...splitter.flush()]) + queueSentence(sentence, record, turn); + } + + function queueSentence(sentence, record, turn) { + speechChain = speechChain.then(async () => { + if (!isCurrentTurn(turn)) return; + const spoken = await speech.speak(sentence).catch(() => null); + if (!spoken || !isCurrentTurn(turn)) return; + record.tts.push(Math.round(spoken.ms)); + if (record.firstAudioMs == null && spoken.delayMs != null) + record.firstAudioMs = Math.round( + performance.now() + spoken.delayMs - record.endOfInput, + ); + }); + return speechChain; + } + + function countFrames() { + const scene = viewer?.scene; + if (!scene?.postRender) return () => null; + let frames = 0; + const started = performance.now(); + const remove = scene.postRender.addEventListener(() => { + frames++; + }); + return () => { + remove(); + const ms = performance.now() - started; + return ms > 0 ? Math.round((frames * 10000) / ms) / 10 : null; + }; + } + + async function decode(promise, record) { + const stopCounting = countFrames(); + try { + return await promise; + } finally { + const fps = stopCounting(); + if (record && fps != null) record.decodeFps.push(fps); + } + } + + async function runTurn( + text, + { endOfInput = performance.now(), stt = null, source = 'text' } = {}, + ) { + const trimmed = String(text || '').trim(); + if (!trimmed || !active || !engine?.prepared) return null; + interrupt(); + const turn = turnGeneration; + const record = { + id: metrics.length + 1, + text: trimmed, + source, + model: store.getState().loadedLlm, + stt, + endOfInput, + actions: [], + tts: [], + firstToolCallMs: null, + firstActionMs: null, + firstAudioMs: null, + decodeFps: [], + answers: [], + }; + metrics.push(record); + current = { turn, record, llmTurnId: null }; + emit({ type: 'transcript', role: 'user', text: trimmed, final: true }); + store.update({ lastUser: trimmed, lastReply: null }); + setStatus('executing', 'Thinking'); + + const executed = []; + let chain = Promise.resolve(); + const execute = async (call) => { + if (!isCurrentTurn(turn)) return; + const name = call.name; + let args = call.arguments; + if (!ACTION_SCHEMA_BY_NAME.has(name)) { + executed.push({ name, reply: 'I can’t do that.' }); + return; + } + args = coerceArguments(name, args); + const entry = { + name, + args, + atMs: Math.round(performance.now() - endOfInput), + }; + record.actions.push(entry); + if (record.firstActionMs == null) record.firstActionMs = entry.atMs; + setStatus('executing', 'Running command'); + const pending = runAction(name, args); + // Slow actions (a layer loading its feed) get an immediate spoken + // acknowledgement; the result then only adds words if it failed. + const slow = await Promise.race([ + pending.then( + () => false, + () => false, + ), + new Promise((resolve) => setTimeout(resolve, SLOW_ACTION_MS, true)), + ]); + const progress = slow ? progressReply(name, args) : null; + if (progress && isCurrentTurn(turn)) speak(progress, record, turn); + let result; + try { + result = await pending; + } catch (error) { + if (error?.name === 'AbortError') return; + result = { ok: false, error: error?.message || String(error) }; + } + entry.ok = result?.ok !== false; + entry.ms = Math.round(performance.now() - endOfInput) - entry.atMs; + let reply = needsSpokenAnswer(name, result) + ? null + : actionReply(name, args, result); + // The acknowledgement already said what is happening; the result adds + // words only when it corrects or qualifies that ("…but its feed is + // unavailable", "at least 500"). + if (progress && entry.ok && !isMaterialReply(name, args, result)) + reply = null; + executed.push({ name, args, result, reply, progress }); + }; + // Each model message lists its calls cumulatively; dispatch new ones in + // order as soon as they stream in. + const dispatcher = () => { + let dispatched = 0; + return (calls) => { + if (record.firstToolCallMs == null) + record.firstToolCallMs = Math.round(performance.now() - endOfInput); + for (const call of calls.slice(dispatched)) { + dispatched++; + chain = chain.then(() => execute(call)); + } + }; + }; + const onToolCalls = dispatcher(); + + // Query results go back to the model for one spoken answer. The model + // may instead call more actions (look up, then track); those run and + // their results feed the next round. + const answerResults = async () => { + setStatus('executing', 'Answering'); + const started = performance.now(); + let pending = executed.slice(); + let answer = ''; + for (let round = 0; round < MAX_TOOL_ROUNDS && pending.length; round++) { + const results = pending + .filter((entry) => entry.result) + .map((entry) => ({ + name: entry.name, + response: compactResultForModel(entry.result), + })); + if (!results.length) break; + const before = executed.length; + const splitter = createSentenceSplitter({ minChars: 8 }); + const markup = createMarkupFilter(); + const say = (sentences) => { + for (const sentence of sentences) { + const spoken = normalizeForSpeech(sentence); + if (spoken) queueSentence(spoken, record, turn); + } + }; + let text = ''; + let answered; + try { + answered = await decode( + engine.llm.continueTurn(handle.turnId, results, { + onToolCalls: dispatcher(), + onDelta: (delta) => { + text += delta; + say(splitter.push(markup.push(delta))); + }, + }), + record, + ); + } catch { + break; + } + if (answered.cancelled || !isCurrentTurn(turn)) return answer; + say([...splitter.push(markup.flush()), ...splitter.flush()]); + record.answers.push({ + round, + ms: Math.round(performance.now() - started), + stats: answered.stats, + }); + answer = combineReplies([answer, stripGemmaMarkup(text)]); + await chain; + if (!isCurrentTurn(turn)) return answer; + pending = executed.slice(before); + const confirmations = combineReplies( + pending.map((entry) => entry.reply), + ); + if (confirmations) { + speak(confirmations, record, turn); + answer = combineReplies([answer, confirmations]); + } + if ( + !pending.some((entry) => needsContinuation(entry.name, entry.result)) + ) + break; + } + if (answer.trim()) return answer; + // No words from the model: state the most recent lookup only. + const last = executed + .filter( + (entry) => + needsSpokenAnswer(entry.name, entry.result) && entry.result, + ) + .at(-1); + const fallback = last ? fallbackAnswer(last.name, last.result) : ''; + speak(fallback, record, turn); + return fallback; + }; + + const handle = engine.llm.turn(trimmed, { onToolCalls }); + current.llmTurnId = handle.turnId; + let done; + try { + done = await decode(handle.done, record); + } catch (error) { + engine.llm.release(handle.turnId); + if (!isCurrentTurn(turn)) return record; + record.error = error?.message || String(error); + setStatus('listening', 'Model error'); + store.update({ lastReply: 'Model error: ' + record.error }); + emit({ + type: 'transcript', + role: 'assistant', + text: 'Model error: ' + record.error, + final: true, + }); + return record; + } + record.llm = done.stats; + if (done.cancelled || !isCurrentTurn(turn)) { + engine.llm.release(handle.turnId); + return record; + } + let modelText = done.text || ''; + if (!done.calls?.length && /<\|tool_call>/.test(modelText)) { + const parsed = parseGemmaToolCalls(modelText); + modelText = parsed.text; + if (parsed.calls.length) onToolCalls(parsed.calls); + } + await chain; + if (!isCurrentTurn(turn)) { + engine.llm.release(handle.turnId); + return record; + } + const confirmations = combineReplies( + executed.filter((entry) => entry.reply).map((entry) => entry.reply), + ); + const acknowledged = combineReplies( + executed.map((entry) => entry.progress), + ); + if (confirmations) speak(confirmations, record, turn); + let reply = combineReplies([acknowledged, confirmations]); + // Results the model must see continue the turn: answers to put into + // words, or a lookup the next call depends on. + if ( + executed.some( + (entry) => entry.result && needsContinuation(entry.name, entry.result), + ) + ) + reply = combineReplies([reply, await answerResults()]); + else if (!executed.length) { + reply = stripGemmaMarkup(modelText) || 'Sorry, I missed that.'; + speak(reply, record, turn); + } + if (!isCurrentTurn(turn)) { + engine.llm.release(handle.turnId); + return record; + } + record.calls = executed.map(({ name, args, result }) => ({ + name, + args, + ok: result ? result.ok !== false : false, + })); + record.reply = reply; + store.update({ lastReply: reply }); + emit({ type: 'transcript', role: 'assistant', text: reply, final: true }); + engine.llm.release(handle.turnId); + await speechChain; + const radioPlayback = pendingRadioPlayback(executed); + if (radioPlayback && isCurrentTurn(turn)) { + // Radio starts only after the confirmation has been heard. + await speech.idle(); + if (isCurrentTurn(turn)) + await handOffToRadio(radioPlayback, turn, record); + } + if (!isCurrentTurn(turn)) return record; + current = null; + emit({ type: 'completion', status: 'completed' }); + setStatus('listening', READY_HINT); + return record; + } + + async function handOffToRadio(result, turn, record) { + setStatus('executing', 'Starting Radio'); + const outcome = await radio.start(result, { + isCurrent: () => isCurrentTurn(turn), + stopVoice: () => { + current = null; + stop({ preserveRadioPlayback: true }); + emit({ type: 'state', state: 'idle', detail: 'Radio playing' }); + }, + }); + record.radio = outcome.result?.audioState || null; + if (outcome.result?.ok || outcome.cancelled || !isCurrentTurn(turn)) return; + const correction = 'The Radio station could not start. Voice is still on.'; + store.update({ lastReply: correction }); + emit({ + type: 'transcript', + role: 'assistant', + text: correction, + final: true, + }); + speak(correction, record, turn); + await speechChain; + if (isCurrentTurn(turn)) setStatus('listening', 'Radio did not start'); + } + + function stop({ removeUi = false, preserveRadioPlayback = false } = {}) { + startGeneration++; + if (preserveRadioPlayback) { + turnGeneration++; + current = null; + speech.stop(); + speechChain = Promise.resolve(); + } else if (active || current) interrupt(); + else radio.cancel(); + radio.release(); + active = false; + current = null; + mic?.close(); + mic = null; + input.stopVoiceVisualizer(); + input.resetSession(); + if (ui) input.updateVoiceButtonLabel(); + if (store.getState().phase !== 'error') + store.update({ phase: 'idle', progress: {} }); + if (removeUi) { + input.detachBindings(); + controls?.destroy(); + controls = null; + speech.close(); + engine?.terminate(); + engine = null; + } + } + + const local = { + settings, + store, + metrics, + get status() { + return status; + }, + get engine() { + return engine; + }, + /** Runs one typed turn through the local pipeline and resolves with its metrics. */ + runText: (text) => runTurn(text, { source: 'text' }), + /** Transcribes 16 kHz samples and runs the resulting turn. */ + runAudio: (samples) => transcribeAndRun(samples), + speechIdle: () => speechChain, + get lastUtterance() { + return lastUtterance; + }, + }; + + return { + capabilities: { costControls: false, pushToTalk: true, local: true }, + local, + start, + stop, + sendText(text) { + void runTurn(text, { source: 'text' }); + return true; + }, + sendMapEvent() { + return false; + }, + ignoreButtonClick: () => Boolean(input.spaceKeyHeld), + bindControls() { + input.bindPushToTalkShortcut(); + if (ui) input.updateVoiceButtonLabel(); + if (ui?.root && !controls) + controls = new LocalVoiceControls({ + root: ui.root, + store, + actions, + engineControls: ui.engineControls || null, + }); + }, + }; +} diff --git a/src/voice/local/localWorkflow.test.mjs b/src/voice/local/localWorkflow.test.mjs new file mode 100644 index 000000000..4f721383f --- /dev/null +++ b/src/voice/local/localWorkflow.test.mjs @@ -0,0 +1,245 @@ +// The on-device tier driving a scripted model: dependent calls continue +// (lookup, then track) and speech honors the shared say envelope. +// +// Run with: npm test +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createLocalWebSession } from './localWebSession.js'; +import { + actionReply, + fallbackAnswer, + isMaterialReply, + needsContinuation, + needsSpokenAnswer, +} from './replies.js'; +import { buildLocalTools } from './toolset.js'; + +/** A scripted model: `turn(text)` and `next(results)` return {calls, text}. */ +function scriptedEngine({ turn, next }) { + let ids = 0; + const seen = []; + return { + seen, + engine: { + prepared: true, + async ensure() { + return { loads: [] }; + }, + llm: { + turn(text, { onToolCalls }) { + const turnId = ++ids; + const done = (async () => { + const step = (await turn(text)) || {}; + if (step.calls?.length) onToolCalls(step.calls); + return { + text: step.text || '', + calls: step.calls || [], + stats: {}, + }; + })(); + return { turnId, done }; + }, + async continueTurn(turnId, results, { onToolCalls, onDelta }) { + seen.push(results); + const step = (await next(results)) || {}; + if (step.calls?.length) onToolCalls(step.calls); + if (step.text) onDelta(step.text); + return { stats: {} }; + }, + release() {}, + cancel() {}, + }, + terminate() {}, + }, + }; +} + +async function localSession(runAction, script) { + const spoken = []; + const model = scriptedEngine(script); + const session = createLocalWebSession({ + emit: () => {}, + runAction, + ui: null, + storage: { getItem: () => null, setItem() {}, removeItem() {} }, + runtime: { + createEngine: () => model.engine, + probeSupport: async () => ({ ok: true }), + checkStorage: async () => ({ note: null }), + openMicrophone: async () => { + throw new Error('no microphone in tests'); + }, + createOutput: () => ({ + id: 'test', + stream: null, + speak: async (sentence) => { + spoken.push(sentence); + return { ms: 1, delayMs: 0 }; + }, + idle: async () => {}, + stop() {}, + close() {}, + }), + }, + }); + await session.start(); + return { session, spoken, seen: model.seen }; +} + +const byName = (results, name) => + results.find((entry) => entry.name === name)?.response; + +test('"follow the nearest aircraft" looks up, then continues to track', async () => { + const ran = []; + const { session, seen } = await localSession( + async (name, args) => { + ran.push(name); + if (name === 'analyst_query') + return { + ok: true, + count: 1, + scopeLabel: 'in view', + items: [{ callsign: 'UAL428', layerId: 'flights' }], + }; + return { ok: true, say: `Tracking ${args.query}.` }; + }, + { + turn: () => ({ + calls: [ + { + name: 'analyst_query', + arguments: { layers: ['flights'], sortBy: 'distance', limit: 1 }, + }, + ], + }), + next: (results) => + byName(results, 'analyst_query') + ? { + calls: [{ name: 'track_entity', arguments: { query: 'UAL428' } }], + } + : {}, + }, + ); + const record = await session.local.runText('follow the nearest aircraft'); + await session.local.speechIdle(); + assert.deepEqual(ran, ['analyst_query', 'track_entity']); + assert.equal(seen.length, 1, 'only the lookup result went back to the model'); + assert.match(record.reply, /Tracking UAL428/); +}); + +test('local speech says the envelope line, including after an acknowledgement', async () => { + const unavailable = { + ok: true, + layerId: 'flights', + enabled: true, + feedState: 'unavailable', + say: 'Flights on, but its feed is unavailable.', + }; + assert.equal( + actionReply('set_layer_visibility', { layerId: 'flights' }, unavailable), + 'Flights on, but its feed is unavailable.', + ); + assert.equal( + isMaterialReply( + 'set_layer_visibility', + { layerId: 'flights' }, + unavailable, + ), + true, + ); + assert.equal( + isMaterialReply( + 'set_layer_visibility', + { layerId: 'flights' }, + { ...unavailable, feedState: 'nominal', say: 'Flights on.' }, + ), + false, + 'a plain confirmation adds nothing after "Turning on flights"', + ); + const { session, spoken } = await localSession( + async () => { + // Slower than the acknowledgement threshold. + await new Promise((resolve) => setTimeout(resolve, 800)); + return unavailable; + }, + { + turn: () => ({ + calls: [ + { + name: 'set_layer_visibility', + arguments: { layerId: 'flights', enabled: true }, + }, + ], + }), + next: () => ({}), + }, + ); + await session.local.runText('turn on flights'); + await session.local.speechIdle(); + assert.deepEqual(spoken, [ + 'Turning on flights.', + 'Flights on, but its feed is unavailable.', + ]); +}); + +test('lower bounds and partial answers survive the deterministic fallback', () => { + assert.equal( + fallbackAnswer('analyst_query', { + ok: true, + count: 500, + complete: false, + scopeLabel: 'inside Kathmandu', + items: [], + }), + 'At least 500 inside Kathmandu.', + ); + assert.match( + fallbackAnswer('analyst_query', { + ok: true, + count: 3, + complete: true, + partial: true, + unanswered: ['military'], + scopeLabel: 'anywhere in the loaded data', + items: [], + }), + /military not answered/, + ); + assert.ok(needsContinuation('analyst_query', { ok: true })); + assert.ok(!needsContinuation('analyst_query', { cancelled: true })); + assert.ok(needsSpokenAnswer('get_entity_context')); + assert.ok(!needsSpokenAnswer('set_layer_visibility')); +}); + +test('the on-device profile never offers pointing and lists only voice core actions', () => { + const tools = buildLocalTools().map((tool) => tool.function); + assert.equal(JSON.stringify(tools).includes('"pointer"'), false); + const names = tools.map((tool) => tool.name); + for (const name of ['resolve_area', 'find_imagery', 'osm_query']) + assert.equal(names.includes(name), false, name); + assert.equal(JSON.stringify(tools).includes('areaId'), false); +}); + +test('compact annotations offer only types their arguments can express', () => { + const tools = buildLocalTools().map((tool) => tool.function); + const annotate = tools.find((tool) => tool.name === 'annotate_map'); + assert.deepEqual( + annotate.parameters.properties.annotations.items.properties.type.enum, + ['pin', 'highlight', 'area', 'label'], + ); +}); + +test('natural voice drops audio that arrives after speech was stopped', async () => { + const { createKokoroOutput } = await import('./speechOutput.js'); + let answer; + const output = createKokoroOutput({ + client: { + synthesize: () => new Promise((resolve) => (answer = resolve)), + flush() {}, + }, + }); + const spoken = output.speak('Zoomed in.'); + output.stop(); + answer({ samples: new Float32Array(10), sampleRate: 24000, ms: 1 }); + assert.equal(await spoken, null, 'nothing is queued after stop()'); +}); diff --git a/src/voice/local/modelCache.test.mjs b/src/voice/local/modelCache.test.mjs new file mode 100644 index 000000000..ed0563228 --- /dev/null +++ b/src/voice/local/modelCache.test.mjs @@ -0,0 +1,217 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; + +import { createSha256 } from './sha256.js'; +import { teeWithBackpressure } from './streamTee.js'; +import { openModelStream } from './workers/modelCache.js'; + +const sha = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +function chunkedSource(chunks, counter = { reads: 0 }) { + let index = 0; + return new ReadableStream( + { + pull(controller) { + counter.reads++; + if (index < chunks.length) controller.enqueue(chunks[index++]); + else controller.close(); + }, + }, + { highWaterMark: 0 }, + ); +} + +async function drain(stream) { + const parts = []; + for await (const part of stream) parts.push(part); + return Buffer.concat(parts); +} + +function fakeCaches({ failOpen = false, failPut = false } = {}) { + const stored = new Map(); + const cache = { + async match(url) { + return stored.has(url) + ? new Response(stored.get(url), { + headers: { 'content-length': String(stored.get(url).length) }, + }) + : undefined; + }, + async put(url, response) { + const body = Buffer.from(await response.arrayBuffer()); + if (failPut) throw new Error('QuotaExceededError'); + stored.set(url, body); + }, + }; + return { + stored, + open: async () => { + if (failOpen) throw new Error('SecurityError'); + return cache; + }, + }; +} + +const payload = Buffer.from(Array.from({ length: 5000 }, (_, i) => i % 251)); +const chunks = () => + Array.from({ length: 50 }, (_, i) => + payload.subarray(i * 100, i * 100 + 100), + ); +const fetchOf = + (parts, length = payload.length) => + async () => + new Response(chunkedSource(parts), { + headers: { 'content-length': String(length) }, + }); + +test('streaming SHA-256 matches the platform digest across chunk sizes', () => { + for (const size of [0, 1, 55, 56, 63, 64, 65, 127, 1000, 4096 + 7]) { + const bytes = Buffer.from( + Array.from({ length: size }, (_, i) => (i * 7) % 256), + ); + const hash = createSha256(); + for (let offset = 0; offset < size; offset += 13) + hash.update(bytes.subarray(offset, offset + 13)); + assert.equal(hash.digest(), sha(bytes), `size ${size}`); + } +}); + +test('the bounded tee never reads far ahead of a stalled consumer', async () => { + const counter = { reads: 0 }; + const parts = Array.from({ length: 1000 }, () => new Uint8Array(1024)); + const { primary, secondary } = teeWithBackpressure( + chunkedSource(parts, counter), + { highWaterMark: 8 * 1024 }, + ); + const reader = primary.getReader(); + for (let i = 0; i < 20; i++) { + const race = await Promise.race([ + reader.read(), + new Promise((resolve) => setTimeout(resolve, 20, 'blocked')), + ]); + if (race === 'blocked') break; + } + assert.ok(counter.reads <= 12, `read ${counter.reads} chunks`); + const secondaryReader = secondary.getReader(); + await secondaryReader.cancel(); + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + } + assert.ok(total > 900 * 1024); +}); + +test('a download is cached once verified and later read from the cache', async () => { + const caches = fakeCaches(); + const first = await openModelStream('m', { + cachesApi: caches, + fetchImpl: fetchOf(chunks()), + sha256: sha(payload), + }); + assert.equal(first.fromCache, false); + assert.deepEqual(await drain(first.stream), payload); + await first.finish(); + assert.deepEqual(await first.cacheWrite, { ok: true }); + const second = await openModelStream('m', { + cachesApi: caches, + fetchImpl: () => assert.fail('cached models are not downloaded'), + }); + assert.equal(second.fromCache, true); + assert.deepEqual(await drain(second.stream), payload); +}); + +test('altered or truncated downloads fail and are not cached', async () => { + const caches = fakeCaches(); + const altered = await openModelStream('m', { + cachesApi: caches, + fetchImpl: fetchOf(chunks()), + sha256: '0'.repeat(64), + }); + await assert.rejects(drain(altered.stream), /integrity/); + await assert.rejects(altered.finish(), /integrity/); + assert.equal((await altered.cacheWrite).ok, false); + const short = await openModelStream('m', { + cachesApi: caches, + fetchImpl: fetchOf(chunks().slice(0, 10)), + }); + await assert.rejects(drain(short.stream), /ended early/); + assert.equal((await short.cacheWrite).ok, false); + assert.equal(caches.stored.size, 0); +}); + +test('without a usable cache the model still loads and says it was not kept', async () => { + const closed = await openModelStream('m', { + cachesApi: fakeCaches({ failOpen: true }), + fetchImpl: fetchOf(chunks()), + }); + assert.deepEqual(await drain(closed.stream), payload); + assert.match((await closed.cacheWrite).error, /SecurityError/); + const full = await openModelStream('m', { + cachesApi: fakeCaches({ failPut: true }), + fetchImpl: fetchOf(chunks()), + }); + assert.deepEqual(await drain(full.stream), payload); + assert.match((await full.cacheWrite).error, /Quota/); +}); + +test('a loader that stops early still gets the file verified and cached', async () => { + const caches = fakeCaches(); + const opened = await openModelStream('m', { + cachesApi: caches, + fetchImpl: fetchOf(chunks()), + sha256: sha(payload), + }); + const reader = opened.stream.getReader(); + for (let i = 0; i < 5; i++) await reader.read(); + await opened.finish(); + assert.deepEqual(await opened.cacheWrite, { ok: true }); + assert.deepEqual(caches.stored.get('m'), payload); +}); + +test('an early-stopping loader of an altered file is told to discard it', async () => { + const opened = await openModelStream('m', { + cachesApi: fakeCaches({ failOpen: true }), + fetchImpl: fetchOf(chunks()), + sha256: '0'.repeat(64), + }); + const reader = opened.stream.getReader(); + await reader.read(); + await assert.rejects(opened.finish(), /integrity/); + assert.equal((await opened.cacheWrite).ok, false); +}); + +test('a cache write that fails without reading never stalls the loader', async () => { + // Larger than the tee's 32 MiB read-ahead, so an unread branch would block. + const big = Buffer.alloc(2 * 1024 * 1024, 7); + const many = () => Array.from({ length: 40 }, () => big); + const whole = Buffer.concat(many()); + const refusing = { + open: async () => ({ + match: async () => undefined, + put: async () => { + throw new Error('QuotaExceededError'); + }, + }), + }; + const full = await openModelStream('m', { + cachesApi: refusing, + fetchImpl: fetchOf(many(), whole.length), + sha256: sha(whole), + }); + assert.equal((await drain(full.stream)).length, whole.length); + await full.finish(); + assert.match((await full.cacheWrite).error, /Quota/); + + const early = await openModelStream('m', { + cachesApi: refusing, + fetchImpl: fetchOf(many(), whole.length), + sha256: sha(whole), + }); + const reader = early.stream.getReader(); + await reader.read(); + await early.cacheWrite; + await early.finish(); +}); diff --git a/src/voice/local/modelCatalog.js b/src/voice/local/modelCatalog.js new file mode 100644 index 000000000..d08dd0c12 --- /dev/null +++ b/src/voice/local/modelCatalog.js @@ -0,0 +1,215 @@ +const HF = 'https://huggingface.co'; +const litert = (name, revision) => + `${HF}/litert-community/${name}-litert-lm/resolve/${revision}/${name}-web.litertlm`; + +/** + * Gemma 4 web builds for LiteRT-LM, pinned to a repository revision and + * checked against the published SHA-256 when downloaded. + */ +export const LLM_MODELS = Object.freeze([ + { + id: 'gemma-4-e2b', + label: 'Gemma 4 E2B', + url: litert('gemma-4-E2B-it', 'b3ca0d2f076785a8f4b2219ddbd2bdb99954eae1'), + sha256: '3a08e8d94e23b814ae5414469c370c503813949acb8ceaa17e4ebf8a35af35b5', + bytes: 2008432640, + minMemoryGB: 8, + }, + { + id: 'gemma-4-e4b', + label: 'Gemma 4 E4B', + url: litert('gemma-4-E4B-it', '2eee7ac325f20eb8c9ac1d0e972f7c84663062da'), + sha256: '3904d826d5dddd25ea173e85204caec09e68ba038116e9b992b69cbdc94f57a0', + bytes: 2969059328, + minMemoryGB: 8, + }, + { + id: 'gemma-4-12b', + label: 'Gemma 4 12B', + url: litert('gemma-4-12B-it', '7a0b1ce0ea821bcd01c5f72af84155e02191152f'), + sha256: 'd37f9392b4f093b470b50b72624c9a752cc0b288ed549b769037cbbd04024449', + bytes: 5986074624, + minMemoryGB: 16, + recommended: true, + }, + { + id: 'gemma-4-26b-a4b', + label: 'Gemma 4 26B A4B', + url: litert( + 'gemma-4-26B-A4B-it', + 'a6268756268ac3eab06333eb28f279cff56e9ea6', + ), + sha256: '4523b2de695a7f22dc675b716253ba9a8512ca9f4852fb6a682fe4c2eb859c16', + bytes: 15786524672, + minMemoryGB: 32, + experimental: true, + }, +]); + +/** Speech recognition models for transformers.js on WebGPU. */ +export const STT_MODELS = Object.freeze([ + { + id: 'moonshine-base', + label: 'Moonshine Base', + repo: 'onnx-community/moonshine-base-ONNX', + revision: 'b1e9b6aae3c3c7298f10c3798393fdf38e8fbbad', + dtype: { encoder_model: 'fp32', decoder_model_merged: 'q4' }, + approxBytes: 160e6, + }, + { + id: 'whisper-base', + label: 'Whisper Base (English)', + repo: 'onnx-community/whisper-base.en', + revision: '51eefc0af78b103839eda9e7e4f4186acc6517fe', + dtype: { encoder_model: 'fp32', decoder_model_merged: 'q4' }, + approxBytes: 210e6, + }, + { + id: 'whisper-large-v3-turbo', + label: 'Whisper Large v3 Turbo', + repo: 'onnx-community/whisper-large-v3-turbo', + revision: '360ebcde2559d60bb474678be3c1de9ef347d01a', + dtype: { encoder_model: 'fp16', decoder_model_merged: 'q4f16' }, + approxBytes: 1.5e9, + }, +]); + +export const TTS_MODEL = Object.freeze({ + id: 'kokoro-82m', + repo: 'onnx-community/Kokoro-82M-v1.0-ONNX', + revision: '1939ad2a8e416c0acfeecc08a694d14ef25f2231', + dtype: 'fp32', + voice: 'af_heart', + sampleRate: 24000, + approxBytes: 330e6, +}); + +/** + * The phonemizer Kokoro turns text into phonemes with. It is not part of + * this build: the browser downloads this exact file from the public npm CDN + * when natural voice first starts and checks its SHA-256 before using it. + */ +export const PHONEMIZER_RUNTIME = Object.freeze({ + name: 'phonemizer', + version: '1.2.1', + url: 'https://cdn.jsdelivr.net/npm/phonemizer@1.2.1/dist/phonemizer.js', + sha256: '193481f474f7c1ea81df3195d18b45df8ef7254dbdccb3f193d60215c4897bec', + bytes: 1322380, +}); + +/** Speech output engines. */ +export const TTS_ENGINES = Object.freeze([ + { + id: 'kokoro', + label: 'Natural', + approxBytes: TTS_MODEL.approxBytes + PHONEMIZER_RUNTIME.bytes, + note: 'Kokoro-82M. Downloads an open-source speech component (GPL) to this browser.', + }, + { + id: 'system', + label: 'System', + approxBytes: 0, + note: 'Browser speech synthesis limited to voices on this device', + }, + { id: 'none', label: 'Text only', approxBytes: 0, note: 'No speech' }, +]); + +export const DEFAULT_STT_ID = 'moonshine-base'; +export const DEFAULT_TTS_ID = 'kokoro'; + +export function findTtsEngine(id) { + return TTS_ENGINES.find((engine) => engine.id === id) || null; +} + +export function findLlmModel(id) { + return LLM_MODELS.find((model) => model.id === id) || null; +} + +export function findSttModel(id) { + return STT_MODELS.find((model) => model.id === id) || null; +} + +/** + * Picks the largest recommended model that fits the reported memory. + * navigator.deviceMemory is capped (Chrome reports at most 32), so the + * largest tier is never chosen automatically. + */ +export function recommendLlmModel({ deviceMemoryGB } = {}) { + const memory = Number(deviceMemoryGB); + if (!Number.isFinite(memory) || memory <= 0) + return findLlmModel('gemma-4-e4b'); + if (memory >= 16) return findLlmModel('gemma-4-12b'); + return findLlmModel('gemma-4-e4b'); +} + +/** + * Decides whether this browser can run the local tier. + * @param {{hasWebGpu: boolean, adapter?: boolean, maxBufferSize?: number, + * deviceMemoryGB?: number, isSecureContext?: boolean, model?: object}} env + * @returns {{ok: boolean, reason?: string}} + */ +export function checkLocalVoiceSupport(env = {}) { + if (env.isSecureContext === false) + return { ok: false, reason: 'On-device voice needs HTTPS or localhost.' }; + if (!env.hasWebGpu) + return { + ok: false, + reason: 'On-device voice needs WebGPU. Use a current Chrome or Edge.', + }; + if (env.adapter === false) + return { ok: false, reason: 'No WebGPU graphics adapter is available.' }; + if ( + Number.isFinite(env.maxBufferSize) && + env.maxBufferSize < 1024 * 1024 * 1024 + ) + return { + ok: false, + reason: 'This GPU cannot allocate the buffers a local model needs.', + }; + const model = env.model; + const memory = Number(env.deviceMemoryGB); + if (model && Number.isFinite(memory) && memory > 0 && memory < 32) { + if (memory < model.minMemoryGB) + return { + ok: false, + reason: `${model.label} needs about ${model.minMemoryGB} GB of memory; this device reports ${memory} GB.`, + }; + } + return { ok: true }; +} + +/** + * Warns when the browser cannot keep the models between sessions. Returns + * null when everything still to download fits and storage is persistent. + * @param {{quota?: number, usage?: number}} estimate StorageManager estimate. + * @param {Array<{label: string, bytes: number}>} pending Models not yet cached. + * @param {{persisted?: boolean}} [options] Whether storage is persistent. + */ +export function storageAdvice(estimate, pending = [], { persisted } = {}) { + const needed = pending.reduce((sum, model) => sum + (model.bytes || 0), 0); + const quota = Number(estimate?.quota); + const usage = Number(estimate?.usage) || 0; + const notes = []; + if (needed > 0 && Number.isFinite(quota) && quota > 0) { + const free = Math.max(0, quota - usage); + if (free < needed * 1.05) + notes.push( + `Browser storage has ${formatBytes(free)} free but ${pending + .map((model) => model.label) + .join( + ', ', + )} need ${formatBytes(needed)}; they will download again next session.`, + ); + } + if (persisted === false && needed > 0) + notes.push('The browser may clear downloaded models when space runs low.'); + return notes.length ? notes.join(' ') : null; +} + +/** Human-readable byte count for download progress. */ +export function formatBytes(bytes) { + const value = Number(bytes) || 0; + if (value >= 1e9) return `${(value / 1e9).toFixed(1)} GB`; + if (value >= 1e6) return `${Math.round(value / 1e6)} MB`; + return `${Math.round(value / 1e3)} KB`; +} diff --git a/src/voice/local/naturalVoice.test.mjs b/src/voice/local/naturalVoice.test.mjs new file mode 100644 index 000000000..d00dccde0 --- /dev/null +++ b/src/voice/local/naturalVoice.test.mjs @@ -0,0 +1,224 @@ +// Natural voice delivery: the phonemizer kokoro-js needs is never bundled; +// the browser downloads the pinned file at runtime and checks its SHA-256. +// +// Run with: npm test +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { + ESPEAK_MARKERS, + findEspeakChunks, + onDeviceRuntimePlugin, + phonemizerLoaderPath, +} from '../../../build/onDeviceRuntime.js'; +import { createBrowserViteConfig } from '../../../build/vite.js'; +import { PHONEMIZER_RUNTIME } from './modelCatalog.js'; +import { + loadPhonemizer, + resetPhonemizer, + sha256Hex, +} from './workers/phonemizerRuntime.js'; + +const encoder = new TextEncoder(); +const genuine = encoder.encode('export const phonemize = async () => ["x"];'); +const runtime = { ...PHONEMIZER_RUNTIME, sha256: sha256Hex(genuine) }; + +function memoryCaches() { + const entries = new Map(); + return { + entries, + async open() { + return { + match: async (url) => + entries.has(url) ? new Response(entries.get(url)) : undefined, + put: async (url, response) => + entries.set(url, new Uint8Array(await response.arrayBuffer())), + }; + }, + }; +} + +const serve = (bytes, calls) => async (url, init) => { + calls.push({ url, init }); + return new Response(bytes); +}; + +const fakeImport = async () => ({ + phonemize: async () => ['x'], + list_voices: async () => [], +}); + +test('the pinned phonemizer matches the package the lockfile installs', () => { + const require = createRequire(import.meta.url); + const file = require.resolve('phonemizer'); + const pkg = JSON.parse( + readFileSync(path.join(path.dirname(file), '..', 'package.json'), 'utf8'), + ); + assert.equal(pkg.version, PHONEMIZER_RUNTIME.version); + assert.equal( + PHONEMIZER_RUNTIME.url, + `https://cdn.jsdelivr.net/npm/phonemizer@${pkg.version}/dist/phonemizer.js`, + ); + const bytes = readFileSync(path.join(path.dirname(file), 'phonemizer.js')); + assert.equal(bytes.length, PHONEMIZER_RUNTIME.bytes); + assert.equal(sha256Hex(new Uint8Array(bytes)), PHONEMIZER_RUNTIME.sha256); +}); + +test('a verified download is imported and kept; the next load reads the copy', async () => { + resetPhonemizer(); + const calls = []; + const caches = memoryCaches(); + const imported = []; + const first = await loadPhonemizer({ + runtime, + fetchImpl: serve(genuine, calls), + cachesApi: caches, + importModule: async (bytes) => { + imported.push(bytes.length); + return fakeImport(); + }, + }); + assert.equal(first.source, 'network'); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, PHONEMIZER_RUNTIME.url); + assert.equal(calls[0].init.credentials, 'omit'); + assert.deepEqual(imported, [genuine.length]); + assert.ok(caches.entries.has(PHONEMIZER_RUNTIME.url)); + + resetPhonemizer(); + const second = await loadPhonemizer({ + runtime, + fetchImpl: serve(genuine, calls), + cachesApi: caches, + importModule: fakeImport, + }); + assert.equal(second.source, 'cache'); + assert.equal(calls.length, 1, 'no second download'); +}); + +test('a download that fails the hash check is never imported or kept', async () => { + resetPhonemizer(); + const caches = memoryCaches(); + let imported = false; + await assert.rejects( + loadPhonemizer({ + runtime, + fetchImpl: serve(encoder.encode('export const phonemize = 1;//x'), []), + cachesApi: caches, + importModule: async () => { + imported = true; + return fakeImport(); + }, + }), + /failed its integrity check/, + ); + assert.equal(imported, false); + assert.equal(caches.entries.size, 0); + + // A tampered kept copy is replaced by a verified download. + resetPhonemizer(); + caches.entries.set(PHONEMIZER_RUNTIME.url, encoder.encode('tampered')); + const calls = []; + const loaded = await loadPhonemizer({ + runtime, + fetchImpl: serve(genuine, calls), + cachesApi: caches, + importModule: fakeImport, + }); + assert.equal(loaded.source, 'network'); + assert.equal(calls.length, 1); +}); + +test('an HTTP error is reported and a later load retries', async () => { + resetPhonemizer(); + await assert.rejects( + loadPhonemizer({ + runtime, + fetchImpl: async () => new Response('', { status: 503 }), + cachesApi: null, + importModule: fakeImport, + }), + /HTTP 503/, + ); + const loaded = await loadPhonemizer({ + runtime, + fetchImpl: serve(genuine, []), + cachesApi: null, + importModule: fakeImport, + }); + assert.equal(loaded.source, 'network'); + resetPhonemizer(); +}); + +test('a stalled download times out instead of holding voice start', async () => { + resetPhonemizer(); + await assert.rejects( + loadPhonemizer({ + runtime, + cachesApi: null, + importModule: fakeImport, + timeoutMs: 20, + fetchImpl: (url, { signal }) => + new Promise((resolve, reject) => + signal.addEventListener('abort', () => + reject(new DOMException('aborted', 'AbortError')), + ), + ), + }), + /download timed out/, + ); + resetPhonemizer(); +}); + +test('the build resolves `phonemizer` to the runtime loader, in pages and workers', () => { + const config = createBrowserViteConfig(); + const alias = config.resolve.alias.find((entry) => + entry.find.test('phonemizer'), + ); + assert.equal(alias.replacement, phonemizerLoaderPath()); + assert.equal(alias.find.test('phonemizer/extra'), false); + const plugin = onDeviceRuntimePlugin(); + assert.equal(plugin.resolveId('phonemizer'), phonemizerLoaderPath()); + assert.equal(plugin.resolveId('kokoro-js'), null); + assert.equal( + config.worker.plugins()[0].resolveId('phonemizer'), + phonemizerLoaderPath(), + ); +}); + +test('a build that bundles eSpeak NG code fails', () => { + const clean = { + 'assets/tts.worker.js': { code: 'import("blob:");' }, + 'assets/app.css': { source: 'body{}' }, + }; + assert.deepEqual(findEspeakChunks(clean), []); + const leaked = { + ...clean, + 'assets/kokoro.js': { code: `var x=new A.${ESPEAK_MARKERS[0]}` }, + 'assets/data.bin': { source: encoder.encode('espeak-ng-data/voices') }, + }; + assert.deepEqual(findEspeakChunks(leaked), [ + 'assets/kokoro.js', + 'assets/data.bin', + ]); + const plugin = onDeviceRuntimePlugin(); + assert.throws( + () => + plugin.generateBundle.call( + { + error(message) { + throw new Error(message); + }, + }, + {}, + leaked, + ), + /eSpeak NG phonemizer code must not be bundled/, + ); + // The shipped package really carries the markers the check looks for. + const require = createRequire(import.meta.url); + const shipped = readFileSync(require.resolve('phonemizer'), 'utf8'); + for (const marker of ESPEAK_MARKERS) assert.ok(shipped.includes(marker)); +}); diff --git a/src/voice/local/naturalVoiceSetting.js b/src/voice/local/naturalVoiceSetting.js new file mode 100644 index 000000000..d7ab48db2 --- /dev/null +++ b/src/voice/local/naturalVoiceSetting.js @@ -0,0 +1,17 @@ +function buildSetting() { + try { + // Replaced at build time (build/vite.js); undefined outside Vite. + return import.meta.env.GEV_NATURAL_VOICE; + } catch { + return undefined; + } +} + +/** + * Whether this build offers natural (Kokoro) voice. GEV_NATURAL_VOICE=off + * (or 0, false, no) removes it; replies then use the browser's on-device + * voices or text. + */ +export function naturalVoiceAvailable(value = buildSetting()) { + return !/^(off|0|false|no)$/i.test(String(value ?? '').trim()); +} diff --git a/src/voice/local/onDeviceRuntime.test.mjs b/src/voice/local/onDeviceRuntime.test.mjs new file mode 100644 index 000000000..b524597b0 --- /dev/null +++ b/src/voice/local/onDeviceRuntime.test.mjs @@ -0,0 +1,72 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { onDeviceRuntimePlugin } from '../../../build/onDeviceRuntime.js'; +import { createBrowserViteConfig } from '../../../build/vite.js'; + +const root = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..', +); +const assetsModule = path.join( + root, + 'src/voice/local/workers/runtimeAssets.js', +); + +test('the build serves every runtime file from installed packages', () => { + const plugin = onDeviceRuntimePlugin({ root }); + const source = plugin.load(assetsModule); + const imports = [...source.matchAll(/from "([^"]+)"/g)].map( + (match) => match[1].split('?')[0], + ); + assert.equal(imports.length, 8); + for (const file of imports) { + assert.ok(existsSync(file), file); + assert.ok(file.includes(`${path.sep}node_modules${path.sep}`), file); + } + assert.doesNotMatch(source, /https?:/); + assert.match(source, /export const litertRuntimes/); + assert.match(source, /export const onnxWasmPaths/); + assert.equal(plugin.load(path.join(root, 'src/main.js')), null); +}); + +test('only the LiteRT-LM loaders gain a module export', () => { + const plugin = onDeviceRuntimePlugin({ root }); + const loader = [ + ...plugin + .load(assetsModule) + .matchAll(/import internalFactory from "([^"]+)"/g), + ][0][1]; + const code = readFileSync(loader, 'utf8'); + assert.match( + plugin.transform(code, loader).code, + /export default ModuleFactory;\n$/, + ); + assert.equal(plugin.transform('x', path.join(root, 'src/main.js')), null); +}); + +test('workers and pages both get the runtime plugin', () => { + const config = createBrowserViteConfig(); + assert.ok( + config.plugins.some((plugin) => plugin?.name === 'gev-on-device-runtime'), + ); + assert.ok( + config.worker + .plugins() + .some((plugin) => plugin.name === 'gev-on-device-runtime'), + ); +}); + +test('workers load runtimes only through the build-provided module', () => { + for (const name of ['llm', 'stt', 'tts']) { + const code = readFileSync( + path.join(root, `src/voice/local/workers/${name}.worker.js`), + 'utf8', + ); + assert.match(code, /from '\.\/runtimeAssets\.js'/); + assert.doesNotMatch(code, /\beval\b|importScripts|cdn\.jsdelivr/); + } +}); diff --git a/src/voice/local/replies.js b/src/voice/local/replies.js new file mode 100644 index 000000000..641526c09 --- /dev/null +++ b/src/voice/local/replies.js @@ -0,0 +1,323 @@ +import { QUERY_ACTIONS } from './toolset.js'; + +const LAYER_NAMES = { + flights: 'Flights', + military: 'Military flights', + earthquakes: 'Earthquakes', + satellites: 'Satellites', + 'rocket-launches': 'Space missions', + traffic: 'Traffic', + cctv: 'Traffic cameras', + radio: 'Radio', + bikeshare: 'Bike share', + 'ais-live-vessels': 'Ships', + 'local-datacenters': 'Datacenters', + 'local-dams': 'Dams', + 'telegeography-submarine-cables': 'Undersea cables', + 'local-firms': 'Fires', + 'alpr-cameras': 'Plate readers', + 'local-adsb': 'Local receiver', +}; + +const STYLE_NAMES = { + normal: 'Normal view', + surveillance: 'Night vision', + thermal: 'Thermal', + retro: 'Retro', + anime: 'Anime', + noir: 'Noir', + snow: 'Snow', +}; + +const STACK_NAMES = { + photoreal: 'Google 3D', + 'bing-aerial': 'Bing aerial', + 'bing-labels': 'Aerial with labels', + 'esri-imagery': 'Esri imagery', + osm: 'OpenStreetMap', +}; + +/** + * Whether a completed action needs a model pass to be put into words. + * @param {string} name + */ +export function needsSpokenAnswer(name) { + return QUERY_ACTIONS.includes(name); +} + +/** + * Whether the model gets another round with this result, to answer it or to + * make the call that depends on it ("track the nearest aircraft"). + * @param {string} name + * @param {object} [result] + */ +export function needsContinuation(name, result = null) { + if (result?.cancelled) return false; + return QUERY_ACTIONS.includes(name); +} + +/** The envelope's spoken line, when the action composed one. */ +function envelopeSay(result) { + if (!result || result.cancelled) return null; + return typeof result.say === 'string' && result.say.trim() + ? result.say.trim() + : null; +} + +const comparable = (text) => + String(text || '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim(); + +/** + * Whether a result has something to say beyond the acknowledgement already + * spoken while it ran: an envelope line that differs from the plain + * confirmation ("Flights on, but its feed is unavailable", "At least 500 …"). + * @param {string} name + * @param {object} args + * @param {object} result + */ +export function isMaterialReply(name, args = {}, result = {}) { + const say = envelopeSay(result); + if (!say) return false; + return comparable(say) !== comparable(fixedReply(name, args, result)); +} + +/** + * Short spoken confirmation for an action result. The shared envelope's + * `say` is authoritative (it carries feed state, lower bounds and partial + * answers); otherwise a fixed line echoes the resulting state, never the + * request. + * @param {string} name + * @param {object} args + * @param {object} result + */ +export function actionReply(name, args = {}, result = {}) { + return envelopeSay(result) || fixedReply(name, args, result); +} + +function fixedReply(name, args = {}, result = {}) { + if (!result || result.ok === false) return failureReply(name, result); + switch (name) { + case 'set_layer_visibility': { + const label = + LAYER_NAMES[result.layerId || args.layerId] || result.label || 'Layer'; + return `${label} ${result.enabled === false || args.enabled === false ? 'off' : 'on'}.`; + } + case 'fly_to_location': + return `Flying to ${spokenPlace(result.label || args.query || 'there')}.`; + case 'adjust_camera_zoom': + return `Zoomed ${result.direction || args.direction || 'in'}.`; + case 'zoom_to_globe': + return 'Globe view.'; + case 'set_visual_style': + return `${STYLE_NAMES[result.style || args.style] || 'Style set'}.`; + case 'set_hud': { + const hud = result.hud || {}; + if (args.visible === 'off' || hud.visible === false) return 'HUD off.'; + if (args.layout) return `HUD ${args.layout} layout.`; + return 'HUD on.'; + } + case 'track_entity': + return `Tracking ${result.label || result.callsign || result.name || args.query || 'target'}.`; + case 'stop_tracking': + return 'Tracking stopped.'; + case 'select_nearest_aircraft': + return `Selected ${result.label || 'the nearest aircraft'}${result.location ? ` near ${spokenPlace(result.location)}` : ''}.`; + case 'annotate_map': { + const failed = result.failedLabels?.length || 0; + if (failed && result.partial) + return 'Marked, but I couldn’t place some of it.'; + return 'Marked.'; + } + case 'clear_annotations': + return 'Map cleared.'; + case 'set_map_stack': + return `${STACK_NAMES[result.requested || args.stack] || 'Basemap'} basemap.`; + case 'set_detection': + return result.enabled === false ? 'Detection off.' : 'Detection on.'; + case 'set_post_processing': + return 'Done.'; + case 'show_data_layers_menu': + return 'Data layers.'; + case 'set_panel_open': + return args.open === false ? 'Closed.' : 'Opened.'; + case 'move_camera': + return args.motion === 'stop' ? 'Camera stopped.' : 'Moving.'; + case 'control_radio': + if (result.radioPlaybackRequested) return 'Turning on the radio.'; + return 'Radio updated.'; + case 'frame_overhead': + return Number.isFinite(result.count) + ? `Framed ${result.count}.` + : 'Framed.'; + default: + return typeof result.say === 'string' && result.say.trim() + ? result.say + : 'Done.'; + } +} + +/** + * Present-tense acknowledgement spoken while a slow action is still running. + * Never claims the result; null means stay quiet until it finishes. + */ +export function progressReply(name, args = {}) { + if (name === 'set_layer_visibility') { + const label = (LAYER_NAMES[args.layerId] || 'the layer').toLowerCase(); + return `${args.enabled === false ? 'Turning off' : 'Turning on'} ${label}.`; + } + if (name === 'analyst_query' || name === 'get_entity_context') + return 'Checking.'; + if (name === 'fly_to_location') + return `Flying to ${spokenPlace(args.query || 'there')}.`; + if (name === 'annotate_map') return 'Marking it.'; + if (name === 'track_entity') return 'Looking for it.'; + return null; +} + +function failureReply(name, result = {}) { + if (result?.cancelled) return null; + const reason = shortReason(result?.error); + if (name === 'track_entity') return reason || 'Nothing matched.'; + if (name === 'fly_to_location') + return reason + ? `Couldn’t fly there. ${reason}` + : 'Couldn’t find that place.'; + return reason ? `That failed. ${reason}` : 'That failed.'; +} + +function shortReason(error) { + const text = String(error || '').trim(); + if (!text) return ''; + const sentence = text.split(/(?<=[.!?])\s/)[0].replace(/\s+/g, ' '); + const clipped = sentence.length > 90 ? sentence.slice(0, 87) + '…' : sentence; + return /[.!?…]$/.test(clipped) ? clipped : clipped + '.'; +} + +function spokenPlace(label) { + return String(label).split(',')[0].trim() || 'there'; +} + +/** Joins several confirmations into one short utterance. */ +export function combineReplies(replies) { + const parts = replies.filter(Boolean); + if (!parts.length) return ''; + return parts.join(' '); +} + +/** + * Deterministic words for a query result, used when the model pass is not + * available or fails. + */ +export function fallbackAnswer(name, result = {}) { + if (!result || result.ok === false) return failureReply(name, result); + if (name === 'analyst_query') { + const stale = + result.feedState && result.feedState !== 'nominal' + ? `, ${result.feedState}` + : ''; + const first = result.items?.[0]; + const example = first ? ` Top: ${itemName(first)}.` : ''; + // A capped count is a floor; a layer that could not answer is named. + const floor = result.complete === false ? 'At least ' : ''; + const missing = result.unanswered?.length + ? ` ${result.unanswered.join(', ')} not answered.` + : ''; + return `${floor}${result.count ?? 0} ${result.scopeLabel || 'found'}${stale}.${example}${missing}`; + } + if (name === 'get_entity_context') { + const place = + result.selected?.name || + result.basemap?.place?.name || + result.basemap?.place?.label || + result.basemap?.viewportPlaces?.dominantLocality; + return place ? `Looking at ${place}.` : 'I can’t tell from here.'; + } + if (typeof result.say === 'string' && result.say.trim()) return result.say; + if (name === 'get_current_view_state') return 'State read.'; + return 'Done.'; +} + +function itemName(item) { + return ( + item.callsign || + item.name || + item.label || + item.title || + item.id || + 'unnamed' + ); +} + +const DROP_KEYS = new Set([ + 'action', + 'coverage', + 'feedProvenance', + 'lifecycleState', + 'lifecycleUncertain', + 'viewportSamples', + 'debug', + 'geometry', + 'positions', + 'raw', +]); + +/** + * Shrinks a query result before it is prefilled into a small model: drops + * bookkeeping, then lowers depth, list length and string length until the + * JSON fits the character budget (about four characters per token). + */ +export function compactResultForModel(result, { budget = 1200 } = {}) { + const passes = [ + { maxDepth: 4, maxItems: 3, maxString: 160 }, + { maxDepth: 3, maxItems: 3, maxString: 100 }, + { maxDepth: 3, maxItems: 2, maxString: 80 }, + { maxDepth: 2, maxItems: 1, maxString: 60 }, + { maxDepth: 1, maxItems: 1, maxString: 60 }, + ]; + let compact = {}; + for (const pass of passes) { + compact = shrinkResult(result, pass); + if (JSON.stringify(compact).length <= budget) return compact; + } + return compact; +} + +function shrinkResult(result, { maxDepth, maxItems, maxString }) { + const seen = new WeakSet(); + const shrink = (value, depth) => { + if (value === null || typeof value !== 'object') { + if (typeof value === 'string' && value.length > maxString) + return value.slice(0, maxString) + '…'; + if (typeof value === 'number' && !Number.isInteger(value)) + return Math.round(value * 100) / 100; + return value; + } + if (seen.has(value) || depth > maxDepth) return undefined; + seen.add(value); + if (Array.isArray(value)) + return value + .slice(0, maxItems) + .map((item) => shrink(item, depth + 1)) + .filter((item) => item !== undefined); + const out = {}; + for (const [key, item] of Object.entries(value)) { + if (DROP_KEYS.has(key) || item === null || item === undefined) continue; + const next = shrink(item, depth + 1); + if (next === undefined) continue; + if (Array.isArray(next) && !next.length) continue; + if ( + next && + typeof next === 'object' && + !Array.isArray(next) && + !Object.keys(next).length + ) + continue; + out[key] = next; + } + return out; + }; + return shrink(result, 0) || {}; +} diff --git a/src/voice/local/sha256.js b/src/voice/local/sha256.js new file mode 100644 index 000000000..d7c627d27 --- /dev/null +++ b/src/voice/local/sha256.js @@ -0,0 +1,120 @@ +const K = new Uint32Array([ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, + 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, + 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, + 0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +]); + +/** + * Incremental SHA-256 for verifying multi-gigabyte downloads as they stream + * (WebCrypto only digests whole buffers). update() accepts Uint8Array chunks. + */ +export function createSha256() { + const state = new Uint32Array([ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, + 0x1f83d9ab, 0x5be0cd19, + ]); + const words = new Uint32Array(64); + const block = new Uint8Array(64); + let blockLength = 0; + let bytes = 0; + + function compress(data, offset) { + for (let i = 0; i < 16; i++) { + const j = offset + i * 4; + words[i] = + (data[j] << 24) | + (data[j + 1] << 16) | + (data[j + 2] << 8) | + data[j + 3]; + } + for (let i = 16; i < 64; i++) { + const a = words[i - 15]; + const b = words[i - 2]; + const s0 = ((a >>> 7) | (a << 25)) ^ ((a >>> 18) | (a << 14)) ^ (a >>> 3); + const s1 = + ((b >>> 17) | (b << 15)) ^ ((b >>> 19) | (b << 13)) ^ (b >>> 10); + words[i] = (words[i - 16] + s0 + words[i - 7] + s1) | 0; + } + let a = state[0]; + let b = state[1]; + let c = state[2]; + let d = state[3]; + let e = state[4]; + let f = state[5]; + let g = state[6]; + let h = state[7]; + for (let i = 0; i < 64; i++) { + const s1 = + ((e >>> 6) | (e << 26)) ^ + ((e >>> 11) | (e << 21)) ^ + ((e >>> 25) | (e << 7)); + const choice = (e & f) ^ (~e & g); + const t1 = (h + s1 + choice + K[i] + words[i]) | 0; + const s0 = + ((a >>> 2) | (a << 30)) ^ + ((a >>> 13) | (a << 19)) ^ + ((a >>> 22) | (a << 10)); + const majority = (a & b) ^ (a & c) ^ (b & c); + const t2 = (s0 + majority) | 0; + h = g; + g = f; + f = e; + e = (d + t1) | 0; + d = c; + c = b; + b = a; + a = (t1 + t2) | 0; + } + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + state[5] += f; + state[6] += g; + state[7] += h; + } + + return { + update(chunk) { + const data = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk); + bytes += data.length; + let offset = 0; + if (blockLength) { + const take = Math.min(64 - blockLength, data.length); + block.set(data.subarray(0, take), blockLength); + blockLength += take; + offset = take; + if (blockLength < 64) return; + compress(block, 0); + blockLength = 0; + } + for (; offset + 64 <= data.length; offset += 64) compress(data, offset); + block.set(data.subarray(offset), 0); + blockLength = data.length - offset; + }, + /** Returns the lowercase hex digest; the instance is spent afterwards. */ + digest() { + const tail = new Uint8Array(blockLength < 56 ? 64 : 128); + tail.set(block.subarray(0, blockLength)); + tail[blockLength] = 0x80; + const bits = bytes * 8; + const view = new DataView(tail.buffer); + view.setUint32(tail.length - 8, Math.floor(bits / 2 ** 32)); + view.setUint32(tail.length - 4, bits >>> 0); + for (let offset = 0; offset < tail.length; offset += 64) + compress(tail, offset); + return [...state] + .map((word) => word.toString(16).padStart(8, '0')) + .join(''); + }, + }; +} diff --git a/src/voice/local/speechOutput.js b/src/voice/local/speechOutput.js new file mode 100644 index 000000000..305e42844 --- /dev/null +++ b/src/voice/local/speechOutput.js @@ -0,0 +1,145 @@ +import { createSpeechPlayer } from './localAudio.js'; + +/** + * Speech output engines share one interface so the voice can be swapped: + * speak(text) → Promise<{delayMs, ms} | null> once the sentence is queued, + * idle() → Promise that settles when everything queued has been heard, + * stop() cancels queued and playing speech, close() releases resources. + */ + +/** Picks an English on-device voice, preferring US English. */ +export function pickOnDeviceVoice(voices = [], language = 'en-US') { + const local = voices.filter((voice) => voice.localService); + const base = language.split('-')[0].toLowerCase(); + return ( + local.find((voice) => voice.lang === language && voice.default) || + local.find((voice) => voice.lang === language) || + local.find((voice) => voice.lang?.toLowerCase().startsWith(base)) || + null + ); +} + +/** Kokoro through the synthesis worker and a gapless player. */ +export function createKokoroOutput({ client, onSpeaking, onIdle }) { + const player = createSpeechPlayer({ onSpeaking, onIdle }); + // Audio synthesized for a sentence queued before stop() or close() is + // dropped, even if the worker answers after the flush. + let generation = 0; + return { + id: 'kokoro', + get stream() { + return player.stream; + }, + async speak(text) { + const epoch = generation; + const audio = await client.synthesize(text).catch(() => null); + if (!audio || audio.skipped || epoch !== generation) return null; + return { + delayMs: player.enqueue(audio.samples, audio.sampleRate), + ms: audio.ms, + }; + }, + idle: () => player.idle(), + stop() { + generation++; + client.flush(); + player.stop(); + }, + close() { + generation++; + player.close(); + }, + }; +} + +/** + * The browser's speech synthesis restricted to voices that report + * `localService`, so text is not sent to a speech service. + */ +export function createSystemOutput({ + synthesis = globalThis.speechSynthesis, + Utterance = globalThis.SpeechSynthesisUtterance, + voice, + onSpeaking, + onIdle, +}) { + let pending = 0; + let generation = 0; + let waiters = []; + const settle = () => { + if (pending) return; + onIdle?.(); + const resolved = waiters; + waiters = []; + for (const resolve of resolved) resolve(); + }; + return { + id: 'system', + stream: null, + // The platform queues utterances itself, so speak() returns at once and + // audible timing is not reported. + async speak(text) { + const epoch = generation; + const utterance = new Utterance(text); + utterance.voice = voice; + utterance.lang = voice?.lang || 'en-US'; + const finish = () => { + if (epoch !== generation) return; + pending = Math.max(0, pending - 1); + settle(); + }; + utterance.onstart = () => { + if (epoch === generation) onSpeaking?.(); + }; + utterance.onend = finish; + utterance.onerror = finish; + pending++; + synthesis.speak(utterance); + return { delayMs: null, ms: 0 }; + }, + idle() { + return pending + ? new Promise((resolve) => waiters.push(resolve)) + : Promise.resolve(); + }, + stop() { + generation++; + pending = 0; + synthesis.cancel(); + settle(); + }, + close() { + this.stop(); + }, + }; +} + +/** No speech: replies appear on screen only. */ +export function createSilentOutput() { + return { + id: 'none', + stream: null, + speak: async () => null, + idle: () => Promise.resolve(), + stop() {}, + close() {}, + }; +} + +/** Waits for the browser's voice list, which loads asynchronously. */ +export function loadSystemVoices( + synthesis = globalThis.speechSynthesis, + { timeoutMs = 1500 } = {}, +) { + if (!synthesis) return Promise.resolve([]); + const voices = synthesis.getVoices(); + if (voices.length) return Promise.resolve(voices); + return new Promise((resolve) => { + const done = () => { + synthesis.removeEventListener?.('voiceschanged', done); + resolve(synthesis.getVoices()); + }; + synthesis.addEventListener?.('voiceschanged', done); + setTimeout(done, timeoutMs); + }); +} diff --git a/src/voice/local/speechText.js b/src/voice/local/speechText.js new file mode 100644 index 000000000..18f1f3217 --- /dev/null +++ b/src/voice/local/speechText.js @@ -0,0 +1,122 @@ +const ABBREVIATIONS = new Set([ + 'mr', + 'mrs', + 'ms', + 'dr', + 'st', + 'mt', + 'ft', + 'vs', + 'etc', + 'approx', + 'no', + 'jr', + 'sr', +]); + +const DIGIT_WORDS = [ + 'zero', + 'one', + 'two', + 'three', + 'four', + 'five', + 'six', + 'seven', + 'eight', + 'nine', +]; + +/** + * Incremental sentence splitter for streamed model text. + * push() returns sentences completed so far; flush() returns the remainder. + */ +export function createSentenceSplitter({ minChars = 12, maxChars = 220 } = {}) { + let buffer = ''; + const take = (final) => { + const out = []; + for (;;) { + const end = findBoundary(buffer, minChars); + if (end < 0) break; + out.push(buffer.slice(0, end).trim()); + buffer = buffer.slice(end); + } + if (buffer.length > maxChars) { + const cut = buffer.lastIndexOf(' ', maxChars); + const at = cut > minChars ? cut : maxChars; + out.push(buffer.slice(0, at).trim()); + buffer = buffer.slice(at); + } + if (final && buffer.trim()) { + out.push(buffer.trim()); + buffer = ''; + } + return out.filter(Boolean); + }; + return { + push(text) { + buffer += String(text || ''); + return take(false); + }, + flush() { + return take(true); + }, + reset() { + buffer = ''; + }, + }; +} + +function findBoundary(text, minChars) { + const pattern = /[.!?…]+["')\]]*(\s+|$)|\n+/g; + let match; + while ((match = pattern.exec(text))) { + const end = match.index + match[0].length; + if (end < minChars) continue; + if (match.index + match[0].length === text.length && !/\s$/.test(match[0])) + return -1; + if (text[match.index] === '.') { + const word = /([A-Za-z]+)$/.exec(text.slice(0, match.index))?.[1]; + if (word && ABBREVIATIONS.has(word.toLowerCase())) continue; + if (/\d$/.test(text.slice(0, match.index)) && /^\d/.test(text.slice(end))) + continue; + } + return end; + } + return -1; +} + +/** + * Prepares text for speech: strips markdown and markup, spells callsign-like + * tokens (letters followed by digits) and expands common units. + */ +export function normalizeForSpeech(text) { + return String(text || '') + .replace(/<[^>]{1,40}>/g, ' ') + .replace(/```[\s\S]*?```/g, ' ') + .replace(/[*_#`>|]+/g, ' ') + .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') + .replace(/\b([A-Z]{2,4})(\d{1,4}[A-Z]?)\b/g, (_, letters, digits) => + spellCallsign(letters, digits), + ) + .replace(/(\d)\s?km\b/g, '$1 kilometers') + .replace(/(\d)\s?m\/s\b/g, '$1 meters per second') + .replace(/(\d)\s?ft\b/g, '$1 feet') + .replace(/(\d)\s?kts?\b/g, '$1 knots') + .replace(/\s+/g, ' ') + .trim(); +} + +function spellCallsign(letters, digits) { + const spoken = [...digits].map((char) => + /\d/.test(char) ? DIGIT_WORDS[Number(char)] : char, + ); + return [...letters, ...spoken].join(' '); +} + +/** Clips a reply to a short spoken form; the full text stays on screen. */ +export function spokenExcerpt(text, maxSentences = 2) { + const splitter = createSentenceSplitter({ minChars: 1 }); + const sentences = [...splitter.push(String(text || '')), ...splitter.flush()]; + return sentences.slice(0, maxSentences).join(' '); +} diff --git a/src/voice/local/streamTee.js b/src/voice/local/streamTee.js new file mode 100644 index 000000000..dc0a3eefa --- /dev/null +++ b/src/voice/local/streamTee.js @@ -0,0 +1,105 @@ +/** + * Splits a byte stream in two with bounded buffering. Either branch pulls + * from the source, but a branch waits while the other holds more than + * `highWaterMark` unread bytes, so a slow consumer bounds memory instead of + * the stream buffering the whole file. + * + * A consumer that stops reading before the end (a model loader that has + * what it needs) calls `releasePrimary()`; the secondary then reads the + * rest alone. Cancelling a branch lets the other continue; cancelling both + * cancels the source. A source error errors both branches, so a partial + * copy is never completed. `releaseSecondary()` drops a secondary whose + * consumer failed, and `drain()` reads what is left of the source once + * neither branch will. + * + * @returns {{primary: ReadableStream, secondary: ReadableStream, + * releasePrimary: () => void, releaseSecondary: () => void, + * drain: () => Promise}} + */ +export function teeWithBackpressure( + source, + { highWaterMark = 32 * 1024 * 1024 } = {}, +) { + const reader = source.getReader(); + const branches = { primary: null, secondary: null }; + const open = { primary: true, secondary: true }; + let reading = null; + let ended = false; + let waiters = []; + const wake = () => { + const pending = waiters; + waiters = []; + for (const resolve of pending) resolve(); + }; + const settle = (name, error) => { + if (!open[name]) return; + open[name] = false; + try { + if (error) branches[name].error(error); + else branches[name].close(); + } catch { + /* already settled */ + } + }; + const pump = () => { + reading ||= reader.read().then( + ({ done, value }) => { + reading = null; + if (done) { + ended = true; + settle('primary'); + settle('secondary'); + } else { + for (const name of ['primary', 'secondary']) + if (open[name]) branches[name].enqueue(value); + } + wake(); + }, + (error) => { + reading = null; + ended = true; + settle('primary', error); + settle('secondary', error); + wake(); + }, + ); + return reading; + }; + const branch = (name, other) => + new ReadableStream( + { + start(controller) { + branches[name] = controller; + }, + async pull() { + while (open[other] && branches[other].desiredSize <= 0 && open[name]) + await new Promise((resolve) => waiters.push(resolve)); + if (open[name]) await pump(); + }, + cancel(reason) { + open[name] = false; + wake(); + if (!open[other]) return reader.cancel(reason); + return undefined; + }, + }, + new ByteLengthQueuingStrategy({ highWaterMark }), + ); + const primary = branch('primary', 'secondary'); + const secondary = branch('secondary', 'primary'); + return { + primary, + secondary, + releasePrimary() { + open.primary = false; + wake(); + }, + releaseSecondary() { + open.secondary = false; + wake(); + }, + async drain() { + while (!ended) await pump(); + }, + }; +} diff --git a/src/voice/local/toolset.js b/src/voice/local/toolset.js new file mode 100644 index 000000000..97e4f1475 --- /dev/null +++ b/src/voice/local/toolset.js @@ -0,0 +1,546 @@ +import { GEV_ACTION_SCHEMAS } from '../actionSchemas.js'; +import { ACTION_DESCRIPTIONS } from '../actionDescriptions.js'; +import { VOICE_LAYER_MANIFEST, voiceLayer } from '../layerManifest.js'; + +/** Actions with hand-tuned compact wording and argument projections. */ +export const CORE_ACTIONS = Object.freeze([ + 'set_layer_visibility', + 'fly_to_location', + 'adjust_camera_zoom', + 'zoom_to_globe', + 'set_visual_style', + 'set_hud', + 'track_entity', + 'stop_tracking', + 'analyst_query', + 'get_entity_context', + 'get_current_view_state', + 'annotate_map', +]); + +/** Actions whose result is answered in words rather than a fixed reply. */ +export const QUERY_ACTIONS = Object.freeze([ + 'analyst_query', + 'get_entity_context', + 'get_current_view_state', + 'next_iss_pass', + 'next_satellite_pass', +]); + +export const MORE_ACTIONS_TOOL = 'more_actions'; + +/** + * Actions left out of the on-device prompt to keep its cached prefix small: + * the Cyber sonar controls apply to one HUD theme only. + */ +export const LOCAL_EXCLUDED_ACTIONS = Object.freeze(['set_cyber_sonar']); + +/** Actions the on-device model can call. */ +export const LOCAL_ACTIONS = Object.freeze( + GEV_ACTION_SCHEMAS.map((schema) => schema.name).filter( + (name) => !LOCAL_EXCLUDED_ACTIONS.includes(name), + ), +); + +// Argument paths kept for the compact profile; everything else stays valid +// but is hidden from the model to keep the cached prompt short. +const COMPACT_ARGUMENTS = { + fly_to_location: ['query', 'rangeM'], + select_nearest_aircraft: ['layerId', 'locationQuery'], + control_radio: [ + 'action', + 'volumePct', + 'category', + 'locationId', + 'locationQuery', + 'country', + ], + next_iss_pass: [], + next_satellite_pass: ['target', 'visibleOnly'], + analyst_query: [ + 'layers', + 'scope.kind', + 'scope.name', + 'scope.km', + 'filters.field', + 'filters.op', + 'filters.value', + 'sortBy', + 'sortDir', + 'limit', + 'followUp', + ], + annotate_map: [ + 'annotations.type', + 'annotations.target', + 'annotations.label', + 'flyTo', + ], + get_entity_context: ['scope'], +}; + +// Enum values narrowed for the compact profile: routes and arrows need +// waypoint and destination arguments the compact projection leaves out. +const COMPACT_ENUMS = { + annotate_map: { 'annotations.type': ['pin', 'highlight', 'area', 'label'] }, +}; + +const COMPACT_REQUIRED = { + fly_to_location: ['query'], + select_nearest_aircraft: ['layerId', 'locationQuery'], +}; + +const COMPACT_DESCRIPTIONS = { + set_layer_visibility: { + description: 'Turn one map data layer on or off.', + layerId: + 'fires/wildfires=local-firms, ships/boats/vessels=ais-live-vessels, datacenters=local-datacenters, dams=local-dams, undersea cables=telegeography-submarine-cables, space missions/launches=rocket-launches, street traffic=traffic, traffic cameras=cctv, military aircraft=military, license plate readers=alpr-cameras, my receiver=local-adsb.', + }, + fly_to_location: { + description: + 'Fly the camera to a named place: city, country, landmark, street or address.', + query: 'The place exactly as the user named it, e.g. "Golden Gate Bridge".', + rangeM: 'Only when the user gives a height or distance in meters.', + }, + adjust_camera_zoom: { + description: 'Zoom the camera in or out from the current view.', + amount: 'little for "a bit", medium by default, lot for "way in/out".', + }, + zoom_to_globe: { + description: + 'Show the whole Earth: globe view, whole planet, zoom all the way out.', + }, + set_visual_style: { + description: + 'Change the visual filter. surveillance=night vision, thermal=heat/FLIR, retro=CRT, normal=default look.', + }, + set_hud: { + description: 'Show, hide or change the layout of the HUD overlay.', + visible: 'on or off.', + }, + track_entity: { + description: + 'Follow a specific aircraft (callsign), ship (name) or satellite (name) with the camera.', + query: 'Callsign, ship name or satellite name, e.g. "UAL428" or "ISS".', + }, + stop_tracking: { + description: 'Stop following the tracked aircraft, ship or satellite.', + }, + analyst_query: { + description: + 'Answer counts, lists and superlatives about loaded data: how many flights over Texas, biggest fire, fastest aircraft, ships headed to Oakland.', + layers: 'Layers to search. fires=local-firms, ships=ais-live-vessels.', + 'scope.kind': + 'view=near the camera (default), region=a named state/country/area, radius=within km, anywhere=global.', + 'scope.name': 'Region name for kind=region, e.g. "Texas".', + 'filters.field': + 'altitudeM, speedMps, onGround, destination, magnitude, frp, name.', + 'filters.value': 'Altitude is meters: 40,000 ft = 12192.', + sortBy: 'Field to sort by, e.g. altitudeM, speedMps, frp, distance.', + }, + get_entity_context: { + description: + 'Describe what the camera is looking at: the place, selected object and visible entities.', + }, + get_current_view_state: { + description: + 'Read the camera, style, HUD, tracked entity and enabled layers.', + }, + select_nearest_aircraft: { + description: + 'Only when the user names a place: turn on flights and select the nearest airborne aircraft to that place.', + locationQuery: 'Place name, e.g. "Austin".', + }, + show_data_layers_menu: { + description: + 'Open the data layers menu, optionally at one layer row. Does not turn the layer on.', + }, + set_panel_open: { description: 'Open or close a UI panel.' }, + set_context_mode: { + description: + 'Context modes: contacts (live contacts), space-missions, or off.', + }, + control_cockpit: { + description: 'Cockpit view: enter, exit, next or previous aircraft.', + }, + set_detection: { + description: + 'Object detection overlay: on/off, sparse/balanced/dense mode or density percent.', + }, + set_map_stack: { + description: + 'Switch the basemap only when named: Bing aerial=bing-aerial, aerial with labels=bing-labels, OSM/road map=osm, Esri=esri-imagery, Google 3D/photorealistic=photoreal.', + }, + set_post_processing: { + description: 'Bloom and sharpen effects: on/off and intensity percent.', + }, + control_scene: { description: 'Scenes: list, play, stop or next scene.' }, + control_cctv: { + description: + 'Traffic cameras: next, previous, nearest, select a named camera, coverage or viewsheds.', + }, + control_radio: { + description: + 'Radio: play (turn on the radio), pause, resume, stop, next, previous, volume with volumePct, or select a station by category and place.', + }, + frame_overhead: { + description: 'Frame the aircraft, ships or satellites overhead in view.', + }, + clear_annotations: { + description: + 'Remove all map marks. Only when the user asks to clear the map.', + }, + move_camera: { + description: 'Camera motion: orbit, pan, tilt, rotate, or stop moving.', + }, + fly_route: { description: 'Fly the camera along the drawn route.' }, + next_iss_pass: { description: 'When the ISS next passes over this view.' }, + next_satellite_pass: { + description: 'When a named satellite next passes over this view.', + }, + annotate_map: { + description: + 'Mark places on the map: a pin on a spot, or an area outline for a building, park or district.', + 'annotations.type': 'pin for a spot, area for a building/park/district.', + 'annotations.target': 'Place name to mark, e.g. "Texas State Capitol".', + 'annotations.label': 'Short label to show.', + flyTo: 'true only when the place is not already in view.', + }, +}; + +/** + * Pointing ("this", "here") needs the turn's pointer snapshot, which only the + * cloud adapter captures. The on-device profile therefore offers no pointer + * sentinel: a model that cannot point is never told it can. + */ +const LOCAL_UNSUPPORTED_ENUM_VALUES = Object.freeze(['pointer']); + +function withoutUnsupportedValues(schema) { + if (!schema || typeof schema !== 'object') return schema; + if (Array.isArray(schema.enum)) + schema.enum = schema.enum.filter( + (value) => !LOCAL_UNSUPPORTED_ENUM_VALUES.includes(value), + ); + for (const child of Object.values(schema.properties || {})) + withoutUnsupportedValues(child); + if (schema.items) withoutUnsupportedValues(schema.items); + return schema; +} + +/** Schemas by action name. */ +export const ACTION_SCHEMA_BY_NAME = new Map( + GEV_ACTION_SCHEMAS.map((schema) => [schema.name, schema]), +); + +/** + * Converts a canonical argument schema into the subset every local engine + * accepts: type, description, properties, required, items and enum. Untyped + * values become strings so strict chat templates cannot fail on them. + */ +export function normalizeSchema(schema) { + const source = schema && typeof schema === 'object' ? schema : {}; + const out = {}; + const type = source.type || (source.properties ? 'object' : 'string'); + out.type = type; + if (typeof source.description === 'string') + out.description = source.description; + if (Array.isArray(source.enum)) out.enum = source.enum.slice(); + if (type === 'object' && source.properties) { + out.properties = {}; + for (const [key, value] of Object.entries(source.properties)) + out.properties[key] = normalizeSchema(value); + const required = (source.required || []).filter( + (key) => key in out.properties, + ); + if (required.length) out.required = required; + } + if (type === 'array') out.items = normalizeSchema(source.items || {}); + return out; +} + +function projectSchema(schema, paths) { + if (!paths) return structuredClone(schema); + const tree = {}; + for (const path of paths) { + let node = tree; + for (const key of path.split('.')) node = node[key] ||= {}; + } + const project = (node, keep) => { + if (node.type === 'array') + return { ...node, items: project(node.items || {}, keep) }; + if (node.type !== 'object' || !node.properties) return { ...node }; + const properties = {}; + for (const [key, child] of Object.entries(keep)) { + if (!node.properties[key]) continue; + properties[key] = Object.keys(child).length + ? project(node.properties[key], child) + : { ...node.properties[key] }; + } + const required = (node.required || []).filter((key) => key in properties); + const result = { ...node, properties }; + if (required.length) result.required = required; + else delete result.required; + return result; + }; + return project(structuredClone(schema), tree); +} + +function describeArguments(schema, notes, prefix = '') { + if (schema.type === 'array' && schema.items) + describeArguments(schema.items, notes, prefix); + if (schema.type !== 'object' || !schema.properties) return; + for (const [key, child] of Object.entries(schema.properties)) { + const path = prefix ? `${prefix}.${key}` : key; + if (notes[path]) child.description = notes[path]; + else delete child.description; + describeArguments(child, notes, path); + } +} + +function applyFullDescriptions(schema, metadata) { + if (!metadata || typeof metadata !== 'object') return; + if (typeof metadata.description === 'string') + schema.description = metadata.description; + if (metadata.parameters && schema.parameters) + applyFullDescriptions(schema.parameters, metadata.parameters); + if (metadata.properties && schema.properties) { + for (const [key, child] of Object.entries(metadata.properties)) + if (schema.properties[key]) + applyFullDescriptions(schema.properties[key], child); + } + if (metadata.items && schema.items) + applyFullDescriptions(schema.items, metadata.items); +} + +/** + * Builds one engine-neutral function declaration + * ({name, description, parameters}) for an action. + * @param {string} name + * @param {{profile?: 'compact'|'full'}} [options] + */ +export function describeAction(name, { profile = 'compact' } = {}) { + const schema = ACTION_SCHEMA_BY_NAME.get(name); + if (!schema) throw new TypeError('Unknown action: ' + name); + if (profile === 'full') { + const full = structuredClone(schema); + applyFullDescriptions(full, ACTION_DESCRIPTIONS[name]); + return { + name, + description: full.description || '', + parameters: withoutUnsupportedValues(normalizeSchema(full.parameters)), + }; + } + const notes = COMPACT_DESCRIPTIONS[name] || {}; + const parameters = withoutUnsupportedValues( + normalizeSchema(projectSchema(schema.parameters, COMPACT_ARGUMENTS[name])), + ); + if (COMPACT_REQUIRED[name]) parameters.required = COMPACT_REQUIRED[name]; + for (const [path, values] of Object.entries(COMPACT_ENUMS[name] || {})) { + const node = schemaAt(parameters, path); + if (node?.enum) node.enum = node.enum.filter((v) => values.includes(v)); + } + describeArguments(parameters, notes); + return { + name, + description: + notes.description || + firstSentence(ACTION_DESCRIPTIONS[name]?.description || ''), + parameters, + }; +} + +function schemaAt(schema, path) { + let node = schema; + for (const key of path.split('.')) { + if (node?.type === 'array') node = node.items; + node = node?.properties?.[key]; + } + return node || null; +} + +/** One-line index entries for actions outside the core set. */ +export function describeMoreActions(core = CORE_ACTIONS) { + const names = LOCAL_ACTIONS.filter((name) => !core.includes(name)); + return { + name: MORE_ACTIONS_TOOL, + description: + 'Any other app control: ' + + names.map((name) => `${name} (${shortPurpose(name)})`).join('; ') + + '.', + parameters: { + type: 'object', + properties: { + action: { type: 'string', enum: names }, + request: { + type: 'string', + description: "The user's request, repeated verbatim.", + }, + }, + required: ['action', 'request'], + }, + }; +} + +/** + * Tools for a local model turn, as OpenAI-style function tools + * ({type:'function', function}) which LiteRT-LM and OpenAI-compatible + * servers both accept. + */ +export function buildLocalTools({ + core = LOCAL_ACTIONS, + profile = 'compact', + includeMore = core.length < LOCAL_ACTIONS.length, +} = {}) { + const tools = core.map((name) => describeAction(name, { profile })); + if (includeMore) tools.push(describeMoreActions(core)); + return tools.map((declaration) => ({ + type: 'function', + function: declaration, + })); +} + +const PURPOSES = { + select_nearest_aircraft: 'find the nearest aircraft to a place', + show_data_layers_menu: 'open the data layers menu', + set_panel_open: 'open or close a panel', + set_context_mode: 'contacts or space missions mode', + control_cockpit: 'cockpit view', + set_detection: 'object detection on/off/density', + set_map_stack: 'basemap: Bing aerial, OSM, Esri, Google 3D', + set_post_processing: 'bloom and sharpen', + control_scene: 'play or stop scenes', + control_cctv: 'traffic cameras, coverage, viewsheds', + control_radio: 'play, pause, stop, volume radio', + frame_overhead: 'frame aircraft, ships or satellites overhead', + clear_annotations: 'clear the map marks', + move_camera: 'orbit, pan, tilt or stop the camera', + fly_route: 'fly along a drawn route', + next_iss_pass: 'when the ISS passes over', + next_satellite_pass: 'when a satellite passes over', + set_layer_visibility: 'layers on or off', + fly_to_location: 'fly to a place', + adjust_camera_zoom: 'zoom in or out', + zoom_to_globe: 'globe view', + set_visual_style: 'visual filter', + set_hud: 'HUD', + track_entity: 'follow an entity', + stop_tracking: 'stop following', + analyst_query: 'count or rank loaded data', + get_entity_context: 'what is in view', + get_current_view_state: 'current app state', + annotate_map: 'mark places', +}; + +function shortPurpose(name) { + return ( + PURPOSES[name] || + firstSentence(ACTION_DESCRIPTIONS[name]?.description || name) + ); +} + +function firstSentence(text) { + const match = String(text).match(/^(.+?[.!?])(\s|$)/); + return (match ? match[1] : String(text)).trim(); +} + +/** + * Coerces model-produced arguments toward the canonical schema: numeric and + * boolean strings, case-insensitive enum values and unknown keys. The action + * runner remains the validation authority. + */ +export function coerceArguments(name, args) { + const schema = ACTION_SCHEMA_BY_NAME.get(name); + if (!schema) return args && typeof args === 'object' ? args : {}; + const coerced = coerceValue(schema.parameters, args ?? {}); + if (name === 'analyst_query' && Array.isArray(coerced.filters)) + coerced.filters = coerced.filters.map((filter) => + coerceFilterValue(filter, coerced.layers), + ); + return coerced; +} + +/** + * An analyst field's type from the voice layer manifest (number, text, flag + * or time), looking first at the queried layers. + */ +export function analystFieldKind(field, layerIds = []) { + const named = (Array.isArray(layerIds) ? layerIds : [layerIds]) + .map((id) => voiceLayer(id)) + .filter(Boolean); + for (const entry of [...named, ...VOICE_LAYER_MANIFEST]) { + const type = entry.query?.fields?.[field]?.type; + if (type) return type; + } + return null; +} + +// Filter values are untyped in the schema; their type follows the field, so +// an identifier such as icao24 "001234" stays text. +function coerceFilterValue(filter, layerIds) { + if (!filter || typeof filter !== 'object') return filter; + const kind = analystFieldKind(filter.field, layerIds); + const value = filter.value; + if (kind === 'text') return { ...filter, value: String(value) }; + if (kind === 'number' && typeof value === 'string') { + const number = Number(value.trim().replace(/,/g, '')); + return Number.isFinite(number) ? { ...filter, value: number } : filter; + } + if (kind === 'flag' && typeof value === 'string') { + const lowered = value.trim().toLowerCase(); + if (lowered === 'true' || lowered === 'false') + return { ...filter, value: lowered === 'true' }; + } + return filter; +} + +function coerceValue(schema, value) { + if (!schema || typeof schema !== 'object') return value; + const type = schema.type; + if (type === 'object' || (!type && schema.properties)) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return {}; + const out = {}; + for (const [key, child] of Object.entries(value)) { + if (child === null || child === undefined) continue; + if (schema.properties && key in schema.properties) + out[key] = coerceValue(schema.properties[key], child); + else if (schema.additionalProperties !== false) out[key] = child; + } + return out; + } + if (type === 'array') { + const list = Array.isArray(value) ? value : [value]; + return list.map((item) => coerceValue(schema.items, item)); + } + if (type === 'boolean') { + if (typeof value === 'string') { + const lowered = value.trim().toLowerCase(); + if (['true', 'on', 'yes', '1'].includes(lowered)) return true; + if (['false', 'off', 'no', '0'].includes(lowered)) return false; + } + return value; + } + if (type === 'number' || type === 'integer') { + if (typeof value === 'string' && value.trim() !== '') { + const number = Number(value.replace(/,/g, '')); + if (Number.isFinite(number)) return number; + } + return value; + } + if (type === 'string') { + const text = typeof value === 'string' ? value : String(value); + if (Array.isArray(schema.enum)) { + const exact = schema.enum.find((option) => option === text); + if (exact) return exact; + const folded = text + .trim() + .toLowerCase() + .replace(/[\s_]+/g, '-'); + const match = schema.enum.find( + (option) => option.toLowerCase().replace(/[\s_]+/g, '-') === folded, + ); + return match || text; + } + return text; + } + return value; +} diff --git a/src/voice/local/workerClient.js b/src/voice/local/workerClient.js new file mode 100644 index 000000000..b74811c74 --- /dev/null +++ b/src/voice/local/workerClient.js @@ -0,0 +1,73 @@ +// Must match the installed @litert-lm/core; a unit test pins them together. +export const LITERT_VERSION = '0.17.1'; + +/** + * Request/response and stream plumbing for one worker. Pending requests and + * open streams are rejected when the worker fails or is terminated. + */ +export function createWorkerClient(worker) { + let sequence = 0; + let failure = null; + const pending = new Map(); + const listeners = new Set(); + const failureListeners = new Set(); + const fail = (error) => { + failure = error; + for (const request of pending.values()) request.reject(error); + pending.clear(); + for (const listener of [...failureListeners]) listener(error); + failureListeners.clear(); + }; + worker.addEventListener('message', ({ data }) => { + for (const listener of [...listeners]) listener(data); + const request = data?.id != null ? pending.get(data.id) : null; + if (!request) return; + if (data.type === 'progress') { + request.onProgress?.(data); + return; + } + if (data.type === 'error') { + pending.delete(data.id); + request.reject(new Error(data.message)); + return; + } + if (request.done(data)) { + pending.delete(data.id); + request.resolve(data); + } + }); + worker.addEventListener('error', (event) => { + fail(new Error(event.message || 'Voice worker failed')); + }); + return { + worker, + request(message, { transfer, onProgress, done = () => true } = {}) { + if (failure) return Promise.reject(failure); + const id = ++sequence; + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject, onProgress, done }); + worker.postMessage({ ...message, id }, transfer || []); + }); + }, + post(message) { + if (!failure) worker.postMessage(message); + }, + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + /** Calls back once if the worker fails or stops; returns an unsubscribe. */ + onFailure(listener) { + if (failure) { + listener(failure); + return () => {}; + } + failureListeners.add(listener); + return () => failureListeners.delete(listener); + }, + terminate() { + worker.terminate(); + fail(new DOMException('Voice worker stopped', 'AbortError')); + }, + }; +} diff --git a/src/voice/local/workers/llm.worker.js b/src/voice/local/workers/llm.worker.js new file mode 100644 index 000000000..4a2ec7583 --- /dev/null +++ b/src/voice/local/workers/llm.worker.js @@ -0,0 +1,360 @@ +import { + Backend, + Engine, + LiteRtLm, + SamplerType, + hasGlobalLiteRtLm, + setGlobalLiteRtLm, + setGlobalLiteRtLmPromise, +} from '@litert-lm/core'; +import { litertRuntimes, requireRuntime } from './runtimeAssets.js'; +import { openModelStream } from './modelCache.js'; + +// Relaxed SIMD probe: (func (result v128) i32.const 1 i8x16.splat +// i32.const 2 i8x16.splat i8x16.relaxed_swizzle) +const RELAXED_SIMD_PROBE = new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 15, 1, + 13, 0, 65, 1, 253, 15, 65, 2, 253, 15, 253, 128, 2, 11, +]); + +// The runtime ships with the application build; nothing is fetched from a +// CDN or evaluated from text. +async function ensureRuntime() { + if (hasGlobalLiteRtLm()) return 0; + const started = performance.now(); + const modern = + 'Suspending' in WebAssembly && WebAssembly.validate(RELAXED_SIMD_PROBE); + const runtimes = requireRuntime(litertRuntimes, 'LiteRT-LM'); + const runtime = modern ? runtimes.internal : runtimes.compatAsyncify; + const module = await runtime.ModuleFactory({ + locateFile: () => runtime.wasmUrl, + }); + const liteRtLm = new LiteRtLm(module); + setGlobalLiteRtLm(liteRtLm); + setGlobalLiteRtLmPromise(Promise.resolve(liteRtLm)); + return performance.now() - started; +} + +let engine = null; +let engineModelUrl = null; +let base = null; +let baseKey = null; +// Cancellation can arrive before a queued turn starts. +const cancelledTurns = new Set(); +// Deleting the conversation most recently cloned from the base invalidates +// the base's task chain (LiteRT-LM 0.17). Keep a spare clone so that a turn +// is never the latest clone when it is released. +let spare = null; +let lastClone = null; +const turns = new Map(); + +const post = (message) => self.postMessage(message); + +// Work runs one message at a time so clones, releases and prepares never +// interleave; cancel bypasses the queue to stop a running stream. +let queue = Promise.resolve(); + +self.onmessage = ({ data }) => { + if (data?.type === 'cancel') { + cancel(data.turnId); + return; + } + queue = queue.then(() => handle(data)); +}; + +async function handle(data) { + const { type, id } = data || {}; + try { + if (type === 'load') await load(data); + else if (type === 'prepare') await prepare(data); + else if (type === 'turn') await turn(data); + else if (type === 'tool_result') await continueTurn(data); + else if (type === 'release') await release(data.turnId); + else if (type === 'unload') await unload(); + else return; + if (type === 'release' || type === 'unload') post({ type: 'ok', id }); + } catch (error) { + post({ + type: 'error', + id, + turnId: data?.turnId, + message: error?.message || String(error), + }); + } +} + +async function load({ id, modelUrl, sha256, bytes, maxNumTokens = 8192 }) { + if (engine && engineModelUrl === modelUrl) { + post({ type: 'loaded', id, ms: 0, fromCache: true, reused: true }); + return; + } + await unload(); + const started = performance.now(); + const wasmMs = await ensureRuntime(); + const { stream, fromCache, total, finish, cacheWrite } = + await openModelStream(modelUrl, { + sha256, + bytes, + onProgress: (loaded, size, phase) => + post({ type: 'progress', id, loaded, total: size, phase }), + }); + post({ type: 'progress', id, loaded: 0, total, phase: 'compile' }); + engine = await Engine.create({ + model: stream, + backend: Backend.GPU_ARTISAN, + mainExecutorSettings: { + maxNumTokens, + backendConfig: { + num_output_candidates: 1, + wait_for_weight_uploads: true, + num_decode_steps_per_sync: 1, + sequence_batch_size: 0, + supported_lora_ranks: [], + max_top_k: 1, + enable_decode_logits: false, + enable_external_embeddings: false, + use_submodel: true, + use_autosized_ringbuffers: true, + }, + }, + benchmarkEnabled: true, + }); + try { + await finish(); + } catch (error) { + // The engine may have read an altered or truncated file; discard it. + await unload(); + throw error; + } + engineModelUrl = modelUrl; + post({ + type: 'loaded', + id, + ms: performance.now() - started, + wasmMs, + fromCache, + bytes: total, + cache: await cacheWrite, + }); +} + +function conversationConfig({ system, tools, constrained, maxOutputTokens }) { + return { + sessionConfig: { + maxOutputTokens, + samplerParams: { type: SamplerType.GREEDY, temperature: 0, k: 1 }, + }, + preface: { + messages: [{ role: 'system', content: system }], + tools, + extra_context: { enable_thinking: false }, + }, + enableConstrainedDecoding: Boolean(constrained), + prefillPrefaceOnInit: true, + filterChannelContentFromKvCache: true, + }; +} + +async function prepare({ + id, + system, + tools, + constrained = false, + maxOutputTokens = 256, +}) { + if (!engine) throw new Error('Model is not loaded'); + const key = JSON.stringify({ system, tools, constrained, maxOutputTokens }); + if (base && baseKey === key) { + post({ type: 'prepared', id, ms: 0, reused: true }); + return; + } + const started = performance.now(); + await dropBase(); + base = await engine.createConversation( + conversationConfig({ system, tools, constrained, maxOutputTokens }), + ); + baseKey = key; + const tokens = await base.getTokenCount().catch(() => null); + const prefillMs = performance.now() - started; + await cloneBase(); + post({ + type: 'prepared', + id, + ms: performance.now() - started, + prefillMs, + tokens, + }); +} + +// LiteRT-LM 0.17 keeps one KV cache per engine: clones continue from the +// prepared prefix, while a second independent conversation would overwrite +// it. Every turn therefore runs on a clone of the single prepared base. +async function openTurnConversation() { + if (!engine) throw new Error('Model is not loaded'); + if (!base) throw new Error('Conversation is not prepared'); + if (!spare) await cloneBase(); + const conversation = spare; + spare = null; + return conversation; +} + +async function cloneBase() { + spare = await base.clone(); + lastClone = spare; +} + +async function dropBase() { + for (const turnId of [...turns.keys()]) await release(turnId, true); + await spare?.delete().catch(() => {}); + spare = null; + lastClone = null; + await base?.delete().catch(() => {}); + base = null; + baseKey = null; +} + +async function turn(data) { + const { turnId, text } = data; + const state = { + conversation: null, + cancelled: cancelledTurns.delete(turnId), + idle: true, + }; + turns.set(turnId, state); + if (state.cancelled) { + post({ type: 'done', turnId, cancelled: true }); + return; + } + const started = performance.now(); + const conversation = await openTurnConversation(); + if (state.cancelled || turns.get(turnId) !== state) { + // Unused clone of the prepared prefix: keep it as the next spare. + if (!spare) spare = conversation; + else await conversation.delete().catch(() => {}); + if (turns.get(turnId) === state) + post({ type: 'done', turnId, cancelled: true }); + return; + } + state.conversation = conversation; + const cloneMs = performance.now() - started; + await stream(turnId, state, { role: 'user', content: text }, { cloneMs }); +} + +async function continueTurn({ turnId, results }) { + const state = turns.get(turnId); + if (!state) throw new Error('Turn is no longer active'); + if (state.cancelled || !state.conversation) { + post({ type: 'done', turnId, cancelled: true }); + return; + } + const content = results.map(({ name, response }) => ({ + type: 'tool_response', + name, + response, + })); + await stream(turnId, state, { role: 'tool', content }, {}); +} + +async function stream(turnId, state, message, extra) { + if (state.cancelled) { + post({ type: 'done', turnId, cancelled: true }); + return; + } + state.idle = false; + const started = performance.now(); + let firstChunkMs = null; + let text = ''; + const calls = []; + const reader = state.conversation.sendMessageStreaming(message).getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done || state.cancelled) break; + if (firstChunkMs === null) firstChunkMs = performance.now() - started; + if (value?.tool_calls?.length) { + for (const call of value.tool_calls) { + calls.push({ + name: call.function?.name, + arguments: call.function?.arguments || {}, + }); + } + post({ + type: 'tool_calls', + turnId, + calls: calls.slice(), + ms: performance.now() - started, + }); + } + const delta = chunkText(value); + if (delta) { + text += delta; + post({ type: 'delta', turnId, text: delta }); + } + } + state.idle = true; + if (state.cancelled) { + await reader.cancel().catch(() => {}); + post({ type: 'done', turnId, cancelled: true }); + return; + } + const bench = await state.conversation.getBenchmarkInfo().catch(() => null); + post({ + type: 'done', + turnId, + text, + calls, + stats: { + ...extra, + totalMs: performance.now() - started, + firstChunkMs, + prefillTokens: bench?.lastPrefillTokenCount ?? null, + prefillTokensPerSecond: bench?.lastPrefillTokensPerSecond ?? null, + decodeTokens: bench?.lastDecodeTokenCount ?? null, + decodeTokensPerSecond: bench?.lastDecodeTokensPerSecond ?? null, + timeToFirstTokenS: bench?.timeToFirstTokenInSecond ?? null, + }, + }); +} + +function chunkText(message) { + if (!message?.content) return ''; + if (typeof message.content === 'string') return message.content; + return message.content + .filter((part) => part?.type === 'text') + .map((part) => part.text) + .join(''); +} + +function cancel(turnId) { + const state = turns.get(turnId); + if (!state) { + cancelledTurns.add(turnId); + return; + } + state.cancelled = true; + if (!state.idle) state.conversation?.cancel(); +} + +async function release(turnId, droppingBase = false) { + const state = turns.get(turnId); + turns.delete(turnId); + cancelledTurns.delete(turnId); + if (!state?.conversation) return; + if (!droppingBase && state.conversation === lastClone && base) + await cloneBase(); + if (!state.idle) { + state.cancelled = true; + try { + state.conversation.cancel(); + } catch { + /* already idle */ + } + } + await state.conversation.delete().catch(() => {}); +} + +async function unload() { + await dropBase(); + await engine?.delete().catch(() => {}); + engine = null; + engineModelUrl = null; +} diff --git a/src/voice/local/workers/modelCache.js b/src/voice/local/workers/modelCache.js new file mode 100644 index 000000000..780f9ea1e --- /dev/null +++ b/src/voice/local/workers/modelCache.js @@ -0,0 +1,180 @@ +import { createSha256 } from '../sha256.js'; +import { teeWithBackpressure } from '../streamTee.js'; + +export const MODEL_CACHE_NAME = 'gev-local-voice-models-v1'; + +/** + * Counts and optionally hashes a download. It errors instead of closing when + * the stream is shorter than expected or the digest differs, so a truncated + * or altered file is never cached or loaded. + */ +function verifiedStream(source, { total, sha256, phase, onProgress }) { + const reader = source.getReader(); + let resolveVerified; + let rejectVerified; + const verified = new Promise((resolve, reject) => { + resolveVerified = resolve; + rejectVerified = reject; + }); + verified.catch(() => {}); + const hash = sha256 ? createSha256() : null; + let loaded = 0; + let lastReport = 0; + const stream = new ReadableStream( + { + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + onProgress?.(loaded, total, phase); + if (total && loaded !== total) + throw new Error( + `Model download ended early (${loaded} of ${total} bytes)`, + ); + if (hash && hash.digest() !== sha256) + throw new Error('Model file failed its integrity check'); + controller.close(); + resolveVerified(); + return; + } + loaded += value.byteLength; + hash?.update(value); + const now = performance.now(); + if (now - lastReport > 250) { + lastReport = now; + onProgress?.(loaded, total, phase); + } + controller.enqueue(value); + } catch (error) { + controller.error(error); + rejectVerified(error); + reader.cancel(error).catch(() => {}); + } + }, + cancel(reason) { + rejectVerified(reason || new Error('Download cancelled')); + return reader.cancel(reason); + }, + }, + { highWaterMark: 0 }, + ); + return { stream, verified }; +} + +async function openCache(cachesApi) { + try { + return { cache: await cachesApi.open(MODEL_CACHE_NAME), error: null }; + } catch (error) { + return { cache: null, error }; + } +} + +/** + * Opens a model file as a stream: from the Cache API when present, otherwise + * from the network while writing the same bytes to the cache with bounded + * buffering. Returns `cacheWrite`, which settles with whether the model will + * be kept for the next session. + */ +export async function openModelStream( + url, + { + bytes = 0, + sha256 = null, + onProgress, + signal, + cachesApi = globalThis.caches, + fetchImpl = (...args) => fetch(...args), + } = {}, +) { + const { cache, error: cacheError } = cachesApi + ? await openCache(cachesApi) + : { cache: null, error: new Error('Cache API unavailable') }; + const cached = cache ? await cache.match(url).catch(() => null) : null; + if (cached?.body) { + const total = Number(cached.headers.get('content-length')) || bytes; + const { stream } = verifiedStream(cached.body, { + total, + phase: 'cache', + onProgress, + }); + return { + fromCache: true, + total, + stream, + finish: async () => {}, + cacheWrite: Promise.resolve({ ok: true, cached: true }), + }; + } + const response = await fetchImpl(url, { signal }); + if (!response.ok || !response.body) + throw new Error(`Model download failed (${response.status})`); + const total = Number(response.headers.get('content-length')) || bytes; + const download = verifiedStream(response.body, { + total, + sha256, + phase: 'download', + onProgress, + }); + // The loader may stop reading once it has what it needs; the second + // branch reads to the end so the file is always verified and cached. + const { primary, secondary, releasePrimary, releaseSecondary, drain } = + teeWithBackpressure(download.stream); + // A failed cache write stops reading its branch; it must not hold the + // loader behind backpressure or leave the download unverified. + let secondaryDropped = false; + let primaryReleased = false; + const cacheWrite = cache + ? cache + .put( + url, + new Response(secondary, { + headers: { + 'content-type': 'application/octet-stream', + ...(total ? { 'content-length': String(total) } : {}), + }, + }), + ) + .then( + () => ({ ok: true }), + (error) => { + secondaryDropped = true; + releaseSecondary(); + // Nobody reads the rest if the loader already finished. + if (primaryReleased) drain().catch(() => {}); + return { ok: false, error: error?.message || String(error) }; + }, + ) + : (async () => { + for await (const chunk of secondary) void chunk; + return { + ok: false, + error: `Model cache unavailable: ${cacheError?.message || cacheError}`, + }; + })().catch((error) => ({ ok: false, error: error?.message })); + return { + fromCache: false, + total, + stream: primary, + /** + * Call once the loader is done with the stream. Resolves when the whole + * download has been verified; rejects if it was cut short or altered. + */ + async finish() { + primaryReleased = true; + releasePrimary(); + if (secondaryDropped) await drain(); + await download.verified; + }, + cacheWrite, + }; +} + +/** Whether a model file is already cached. */ +export async function isModelCached(url, cachesApi = globalThis.caches) { + try { + const cache = await cachesApi.open(MODEL_CACHE_NAME); + return Boolean(await cache.match(url)); + } catch { + return false; + } +} diff --git a/src/voice/local/workers/phonemizerRuntime.js b/src/voice/local/workers/phonemizerRuntime.js new file mode 100644 index 000000000..c27a1822f --- /dev/null +++ b/src/voice/local/workers/phonemizerRuntime.js @@ -0,0 +1,119 @@ +import { PHONEMIZER_RUNTIME } from '../modelCatalog.js'; +import { createSha256 } from '../sha256.js'; +import { MODEL_CACHE_NAME } from './modelCache.js'; + +// kokoro-js imports `phonemizer`; the build resolves that import to this +// module (build/onDeviceRuntime.js), so the phonemizer is never bundled or +// served from this origin. The first call downloads the pinned file from +// the public npm CDN, checks its SHA-256 and only then imports it. + +let pending = null; + +// A stalled download must not hold voice start; natural voice then falls +// back to a system voice. +const DOWNLOAD_TIMEOUT_MS = 30_000; + +/** SHA-256 of a byte array as lowercase hex. */ +export function sha256Hex(bytes) { + const hash = createSha256(); + hash.update(bytes); + return hash.digest(); +} + +async function keptCopy(cachesApi, url) { + try { + const cache = await cachesApi?.open(MODEL_CACHE_NAME); + const response = await cache?.match(url); + return response ? new Uint8Array(await response.arrayBuffer()) : null; + } catch { + return null; + } +} + +async function keepCopy(cachesApi, url, bytes) { + try { + const cache = await cachesApi?.open(MODEL_CACHE_NAME); + await cache?.put( + url, + new Response(bytes, { headers: { 'content-type': 'text/javascript' } }), + ); + } catch { + /* storage refused; the next start downloads again */ + } +} + +/** + * Reads the kept copy or downloads the pinned phonemizer, verifies its + * SHA-256 and imports it. A copy that fails the check is never imported. + * @returns {Promise<{module: object, source: 'cache'|'network'}>} + */ +export function loadPhonemizer({ + runtime = PHONEMIZER_RUNTIME, + fetchImpl = globalThis.fetch?.bind(globalThis), + cachesApi = globalThis.caches, + importModule = importFromBytes, + timeoutMs = DOWNLOAD_TIMEOUT_MS, +} = {}) { + pending ||= (async () => { + let bytes = await keptCopy(cachesApi, runtime.url); + let source = 'cache'; + if (!bytes || sha256Hex(bytes) !== runtime.sha256) { + source = 'network'; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(runtime.url, { + credentials: 'omit', + referrerPolicy: 'no-referrer', + signal: controller.signal, + }); + if (!response.ok) + throw new Error( + `Natural voice component could not be downloaded (HTTP ${response.status})`, + ); + bytes = new Uint8Array(await response.arrayBuffer()); + } catch (error) { + if (!controller.signal.aborted) throw error; + throw new Error('Natural voice component download timed out'); + } finally { + clearTimeout(timer); + } + if (sha256Hex(bytes) !== runtime.sha256) + throw new Error('Natural voice component failed its integrity check'); + } + const module = await importModule(bytes); + if (typeof module?.phonemize !== 'function') + throw new Error('Natural voice component is not usable'); + if (source === 'network') await keepCopy(cachesApi, runtime.url, bytes); + return { module, source }; + })(); + pending.catch(() => { + pending = null; + }); + return pending; +} + +async function importFromBytes(bytes) { + const url = URL.createObjectURL( + new Blob([bytes], { type: 'text/javascript' }), + ); + try { + return await import(/* @vite-ignore */ url); + } finally { + URL.revokeObjectURL(url); + } +} + +/** The phonemizer API kokoro-js calls. */ +export async function phonemize(text, language) { + return (await loadPhonemizer()).module.phonemize(text, language); +} + +export async function list_voices(language) { + return (await loadPhonemizer()).module.list_voices(language); +} + +/** Forgets a loaded or failed component (tests). */ +export function resetPhonemizer() { + pending = null; +} diff --git a/src/voice/local/workers/runtimeAssets.js b/src/voice/local/workers/runtimeAssets.js new file mode 100644 index 000000000..0575a1a6c --- /dev/null +++ b/src/voice/local/workers/runtimeAssets.js @@ -0,0 +1,17 @@ +// The application build replaces this module (build/onDeviceRuntime.js) +// with the installed LiteRT-LM and ONNX Runtime Web loaders and wasm URLs, +// so on-device voice never fetches runtime code from a CDN. Builds without +// that plugin get these empty values and refuse to start. + +/** LiteRT-LM builds: {internal, compatAsyncify} of {ModuleFactory, wasmUrl}. */ +export const litertRuntimes = null; + +/** ONNX Runtime Web {mjs, wasm} URLs for each transformers.js copy. */ +export const onnxWasmPaths = { transformers: null, kokoro: null }; + +/** Throws when this build does not include the on-device runtimes. */ +export function requireRuntime(value, name) { + if (!value) + throw new Error(`On-device voice runtime missing from this build: ${name}`); + return value; +} diff --git a/src/voice/local/workers/stt.worker.js b/src/voice/local/workers/stt.worker.js new file mode 100644 index 000000000..90506a8e5 --- /dev/null +++ b/src/voice/local/workers/stt.worker.js @@ -0,0 +1,78 @@ +import { env, pipeline } from '@huggingface/transformers'; +import { onnxWasmPaths, requireRuntime } from './runtimeAssets.js'; + +env.allowLocalModels = false; +env.backends.onnx.wasm.wasmPaths = { + ...requireRuntime(onnxWasmPaths.transformers, 'ONNX Runtime'), +}; + +let recognizer = null; +let recognizerKey = null; +let recognizerLanguage = null; +let queue = Promise.resolve(); + +const post = (message, transfer) => self.postMessage(message, transfer || []); + +self.onmessage = ({ data }) => { + const { type, id } = data || {}; + queue = queue + .then(async () => { + if (type === 'load') await load(data); + else if (type === 'transcribe') await transcribe(data); + }) + .catch((error) => + post({ type: 'error', id, message: error?.message || String(error) }), + ); +}; + +async function load({ id, repo, revision, dtype, language = null }) { + const key = JSON.stringify({ repo, revision, dtype }); + if (recognizer && recognizerKey === key) { + recognizerLanguage = language; + post({ type: 'loaded', id, ms: 0, reused: true }); + return; + } + const started = performance.now(); + await recognizer?.dispose?.(); + const files = new Map(); + recognizer = await pipeline('automatic-speech-recognition', repo, { + device: 'webgpu', + dtype, + revision, + progress_callback: (event) => { + if (event.status !== 'progress' || !event.file) return; + files.set(event.file, { loaded: event.loaded, total: event.total }); + let loaded = 0; + let total = 0; + for (const file of files.values()) { + loaded += file.loaded || 0; + total += file.total || 0; + } + post({ type: 'progress', id, loaded, total, phase: 'download' }); + }, + }); + recognizerKey = key; + recognizerLanguage = language; + await recognizer(new Float32Array(16000), recognizerOptions()); + post({ type: 'loaded', id, ms: performance.now() - started }); +} + +function recognizerOptions() { + return recognizerLanguage + ? { language: recognizerLanguage, task: 'transcribe', max_new_tokens: 96 } + : { max_new_tokens: 96 }; +} + +async function transcribe({ id, audio }) { + if (!recognizer) throw new Error('Speech recognition is not loaded'); + const started = performance.now(); + const output = await recognizer(audio, recognizerOptions()); + const text = (Array.isArray(output) ? output[0]?.text : output?.text) || ''; + post({ + type: 'transcript', + id, + text: text.trim(), + ms: performance.now() - started, + audioMs: (audio.length / 16000) * 1000, + }); +} diff --git a/src/voice/local/workers/tts.worker.js b/src/voice/local/workers/tts.worker.js new file mode 100644 index 000000000..0225aacbe --- /dev/null +++ b/src/voice/local/workers/tts.worker.js @@ -0,0 +1,106 @@ +import { KokoroTTS, env } from 'kokoro-js'; +import { loadPhonemizer } from './phonemizerRuntime.js'; +import { onnxWasmPaths, requireRuntime } from './runtimeAssets.js'; + +env.wasmPaths = { ...requireRuntime(onnxWasmPaths.kokoro, 'ONNX Runtime') }; + +// kokoro-js loads the model and its voice vectors from the repository's +// main branch without a revision option; pin those requests to the +// catalogued revision. +const REPOSITORY_PATH = + /(\/onnx-community\/Kokoro-82M-v1\.0-ONNX\/resolve\/)main\//; +let pinnedRevision = null; +const baseFetch = self.fetch.bind(self); +self.fetch = (input, init) => { + const url = typeof input === 'string' ? input : input?.url; + if (pinnedRevision && url && REPOSITORY_PATH.test(url)) + return baseFetch( + url.replace(REPOSITORY_PATH, `$1${pinnedRevision}/`), + init, + ); + return baseFetch(input, init); +}; + +let tts = null; +let queue = Promise.resolve(); +let generation = 0; + +const post = (message, transfer) => self.postMessage(message, transfer || []); + +self.onmessage = ({ data }) => { + const { type, id } = data || {}; + if (type === 'flush') { + generation++; + post({ type: 'flushed', id }); + return; + } + const epoch = generation; + queue = queue + .then(async () => { + if (type === 'load') await load(data); + else if (type === 'speak') await speak(data, epoch); + }) + .catch((error) => + post({ type: 'error', id, message: error?.message || String(error) }), + ); +}; + +async function load({ id, repo, revision, dtype, voice = 'af_heart' }) { + if (tts) { + post({ type: 'loaded', id, ms: 0, reused: true }); + return; + } + const started = performance.now(); + // The phonemizer is downloaded and verified first, so a failed check + // stops natural voice before any model weights are fetched. + const phonemizer = await loadPhonemizer(); + const files = new Map(); + pinnedRevision = revision || null; + tts = await KokoroTTS.from_pretrained(repo, { + dtype, + device: 'webgpu', + progress_callback: (event) => { + if (event.status !== 'progress' || !event.file) return; + files.set(event.file, { loaded: event.loaded, total: event.total }); + let loaded = 0; + let total = 0; + for (const file of files.values()) { + loaded += file.loaded || 0; + total += file.total || 0; + } + post({ type: 'progress', id, loaded, total, phase: 'download' }); + }, + }); + await tts.generate('Ready.', { voice }); + post({ + type: 'loaded', + id, + ms: performance.now() - started, + phonemizer: phonemizer.source, + }); +} + +async function speak({ id, text, voice, speed }, epoch) { + if (epoch !== generation) { + post({ type: 'audio', id, skipped: true }); + return; + } + if (!tts) throw new Error('Speech synthesis is not loaded'); + const started = performance.now(); + const audio = await tts.generate(text, { voice, speed }); + if (epoch !== generation) { + post({ type: 'audio', id, skipped: true }); + return; + } + const samples = audio.audio; + post( + { + type: 'audio', + id, + samples, + sampleRate: audio.sampling_rate, + ms: performance.now() - started, + }, + [samples.buffer], + ); +} diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index d6a4160ef..236646f84 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -2,6 +2,7 @@ import { createVoiceControl } from './control.js'; import { createVoiceSession } from './session.js'; import { VoiceCardControls } from './voiceCard.js'; import { PointerReticle } from './pointerReticle.js'; +import { VoiceEngineControls } from './voiceEngineControls.js'; function createDefaultReticle(pointer) { const element = pointer?.element; @@ -28,9 +29,23 @@ export function createVoiceCommands({ pointer = null, referents = null, createReticle = createDefaultReticle, + engineChoice = null, + createEngineControls = (options) => new VoiceEngineControls(options), }) { window.__gevVoiceCommands?.stop?.({ removeUi: true }); const ui = createControl({ reset: true }); + // Cloud / On-device choice; the on-device provider mounts its model + // settings in this tray. + const engineControls = + engineChoice && ui.root + ? createEngineControls({ + root: ui.root, + engine: engineChoice.engine, + onSelect: engineChoice.select, + open: Boolean(engineChoice.openTray), + }) + : null; + ui.engineControls = engineControls; const session = createVoiceSession({ runner, signal, @@ -80,6 +95,7 @@ export function createVoiceCommands({ ? session.subscribe((event) => card.handle(event)) : null; controls.voiceCard = card; + controls.engineControls = engineControls; // Point-and-ask: the reticle follows turn events; pointer and referents are // session-scoped, so a session that ends forgets both. const reticle = pointer ? createReticle(pointer) : null; @@ -115,6 +131,7 @@ export function createVoiceCommands({ updatePointer(); reticle?.destroy(); card?.destroy(); + engineControls?.destroy(); runner.dispose?.(); ui.root.remove(); }, @@ -128,6 +145,7 @@ export function createVoiceCommands({ updatePointer(); reticle?.destroy(); card?.destroy(); + engineControls?.destroy(); runner.dispose?.(); ui.root.remove(); } else adapter.bindControls?.(); diff --git a/src/voice/voiceEngine.js b/src/voice/voiceEngine.js new file mode 100644 index 000000000..331291396 --- /dev/null +++ b/src/voice/voiceEngine.js @@ -0,0 +1,50 @@ +import { localWebVoiceRequested } from './local/localWebFlag.js'; +import { voiceStorage } from './realtimePreferences.js'; + +/** + * Where voice runs: the cloud (OpenAI Realtime through this server) or + * on this device (models in the browser). Chosen in the voice control and + * kept per browser (`godsEyeView..` convention). + */ +export const VOICE_ENGINES = Object.freeze(['cloud', 'on-device']); +export const DEFAULT_VOICE_ENGINE = 'cloud'; +export const VOICE_ENGINE_STORAGE_KEY = 'godsEyeView.voiceEngine.choice'; + +const known = (value) => (VOICE_ENGINES.includes(value) ? value : null); + +/** + * The engine for this page: `?voice=local-web` (or `?voice=cloud`) is a + * shortcut that wins, then the stored choice, then Cloud. + */ +export function readVoiceEngine({ + search = globalThis.location?.search, + storage, +} = {}) { + if (localWebVoiceRequested(search)) return 'on-device'; + try { + if (new URLSearchParams(search || '').get('voice') === 'cloud') + return 'cloud'; + } catch { + /* malformed query: use the stored choice */ + } + try { + return ( + known(voiceStorage(storage)?.getItem(VOICE_ENGINE_STORAGE_KEY)) || + DEFAULT_VOICE_ENGINE + ); + } catch { + return DEFAULT_VOICE_ENGINE; + } +} + +/** Persists an engine choice. Never throws; returns the stored value. */ +export function writeVoiceEngine(engine, storage) { + const chosen = known(engine); + if (!chosen) return null; + try { + voiceStorage(storage)?.setItem(VOICE_ENGINE_STORAGE_KEY, chosen); + } catch { + /* best effort */ + } + return chosen; +} diff --git a/src/voice/voiceEngine.test.mjs b/src/voice/voiceEngine.test.mjs new file mode 100644 index 000000000..4bac1fa68 --- /dev/null +++ b/src/voice/voiceEngine.test.mjs @@ -0,0 +1,209 @@ +// Cloud / On-device choice in the voice control: preference, hardware check +// and guidance, and the tray that switches engines. +// +// Run with: npm test +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { railFixture } from '../ui/railTestFixture.mjs'; +import { + DEFAULT_VOICE_ENGINE, + VOICE_ENGINE_STORAGE_KEY, + readVoiceEngine, + writeVoiceEngine, +} from './voiceEngine.js'; +import { VoiceEngineControls } from './voiceEngineControls.js'; +import { + ON_DEVICE_GUIDANCE, + deviceCheckView, + probeDevice, + voiceEngineView, +} from './voiceEnginePresentation.js'; + +function memoryStorage(initial = {}) { + const values = new Map(Object.entries(initial)); + return { + getItem: (key) => (values.has(key) ? values.get(key) : null), + setItem: (key, value) => values.set(key, String(value)), + values, + }; +} + +const appleProbe = { + secure: true, + webgpu: true, + adapter: true, + vendor: 'apple', + architecture: 'metal-3', + maxBufferSize: 4 * 1024 ** 3, + deviceMemoryGB: 16, +}; + +test('the engine choice persists; the URL flag stays a shortcut', () => { + const storage = memoryStorage(); + assert.equal(readVoiceEngine({ search: '', storage }), DEFAULT_VOICE_ENGINE); + assert.equal(DEFAULT_VOICE_ENGINE, 'cloud'); + assert.equal(writeVoiceEngine('on-device', storage), 'on-device'); + assert.equal(storage.values.get(VOICE_ENGINE_STORAGE_KEY), 'on-device'); + assert.equal(readVoiceEngine({ search: '', storage }), 'on-device'); + assert.equal(readVoiceEngine({ search: '?voice=cloud', storage }), 'cloud'); + assert.equal( + readVoiceEngine({ search: '?voice=local-web', storage: memoryStorage() }), + 'on-device', + ); + assert.equal(writeVoiceEngine('elsewhere', storage), null); + const corrupt = memoryStorage({ [VOICE_ENGINE_STORAGE_KEY]: 'nope' }); + assert.equal(readVoiceEngine({ search: '', storage: corrupt }), 'cloud'); + const throwing = { + getItem() { + throw new Error('denied'); + }, + setItem() { + throw new Error('denied'); + }, + }; + assert.equal(readVoiceEngine({ search: '', storage: throwing }), 'cloud'); + assert.equal(writeVoiceEngine('cloud', throwing), 'cloud'); +}); + +test('hardware guidance names the model and download for each tier', () => { + assert.deepEqual(ON_DEVICE_GUIDANCE, [ + 'Works best on 16 GB+ Apple Silicon: Gemma 4 12B (6 GB download)', + '8–16 GB: Gemma 4 E4B (3 GB download)', + 'Under 8 GB or no WebGPU: use Cloud', + ]); + const apple = deviceCheckView(appleProbe); + assert.equal(apple.state, 'supported'); + assert.equal(apple.gpu, 'WebGPU: available (apple metal-3)'); + assert.equal(apple.memory, 'Memory: browser reports at least 16 GB'); + assert.equal(apple.suggested, 'gemma-4-12b'); + assert.match(apple.verdict, /Suggested: Gemma 4 12B \(6 GB download\)/); + const eight = deviceCheckView({ ...appleProbe, deviceMemoryGB: 8 }); + assert.equal(eight.suggested, 'gemma-4-e4b'); + assert.equal(deviceCheckView(null).state, 'checking'); +}); + +test('an unsupported device is explained and pointed to Cloud', () => { + const noGpu = deviceCheckView({ ...appleProbe, webgpu: false }); + assert.equal(noGpu.state, 'unsupported'); + assert.equal(noGpu.gpu, 'WebGPU: not available'); + assert.match(noGpu.verdict, /needs WebGPU.*Use Cloud\.$/); + const small = deviceCheckView({ ...appleProbe, deviceMemoryGB: 4 }); + assert.equal(small.state, 'unsupported'); + assert.match(small.verdict, /needs about 8 GB/); + const view = voiceEngineView({ + engine: 'cloud', + probe: { ...appleProbe, webgpu: false }, + }); + assert.equal( + view.chips.find((chip) => chip.id === 'on-device').disabled, + true, + ); + // An on-device choice can always be undone, even when refused. + const chosen = voiceEngineView({ + engine: 'on-device', + probe: { ...appleProbe, webgpu: false }, + }); + assert.equal( + chosen.chips.find((chip) => chip.id === 'on-device').disabled, + false, + ); + assert.equal( + chosen.chips.find((chip) => chip.id === 'cloud').disabled, + undefined, + ); +}); + +test('the probe reads WebGPU adapter facts and the memory estimate', async () => { + const probe = await probeDevice({ + navigator: { + deviceMemory: 8, + gpu: { + requestAdapter: async () => ({ + info: { vendor: 'apple', architecture: 'metal-3' }, + limits: { maxBufferSize: 2 ** 32 }, + }), + }, + }, + secure: true, + }); + assert.deepEqual(probe, { + secure: true, + webgpu: true, + adapter: true, + vendor: 'apple', + architecture: 'metal-3', + maxBufferSize: 2 ** 32, + deviceMemoryGB: 8, + }); + const failing = await probeDevice({ + navigator: { + gpu: { + requestAdapter: async () => { + throw new Error('blocked'); + }, + }, + }, + }); + assert.equal(failing.adapter, false); + assert.equal((await probeDevice({ navigator: {} })).webgpu, false); +}); + +test('the engine tray nests in the voice control and switches engines', async () => { + const { document } = railFixture(); + const root = document.createElement('div'); + const cost = document.createElement('div'); + cost.className = 'gev-voice-cost'; + root.appendChild(cost); + const selected = []; + let probes = 0; + const controls = new VoiceEngineControls({ + root, + engine: 'cloud', + onSelect: (id) => selected.push(id), + probe: async () => { + probes++; + return appleProbe; + }, + }); + assert.equal(cost.children[0], controls.toggle); + assert.equal(controls.toggle.textContent, 'CLOUD'); + assert.equal(root.dataset.voiceEngine, 'cloud'); + assert.equal(probes, 0, 'a Cloud user is not probed until the tray opens'); + + controls.toggle.click(); + assert.equal(root.dataset.voiceTray, 'open'); + await controls.probing; + assert.equal(probes, 1); + assert.match(controls.elements.verdict.textContent, /Gemma 4 12B/); + assert.equal(controls.elements.guidance.children.length, 3); + + const chip = (id) => + controls.elements.chips.children.find((node) => node.dataset.chipId === id); + chip('cloud').click(); + assert.deepEqual(selected, [], 'the running engine is not re-selected'); + chip('on-device').click(); + assert.deepEqual(selected, ['on-device']); + + controls.toggle.click(); + assert.equal(root.dataset.voiceTray, undefined); + controls.destroy(); + assert.equal(cost.children.length, 0); + assert.equal(root.dataset.voiceEngine, undefined); +}); + +test('a saved on-device choice checks the hardware at once', async () => { + const { document } = railFixture(); + const root = document.createElement('div'); + const controls = new VoiceEngineControls({ + root, + engine: 'on-device', + probe: async () => ({ ...appleProbe, webgpu: false }), + }); + await controls.probing; + assert.equal(controls.toggle.textContent, 'ON-DEVICE'); + assert.match(controls.elements.verdict.textContent, /Use Cloud\.$/); + assert.ok(controls.elements.verdict.className.includes('error')); + controls.setBadge('ON-DEVICE E4B'); + assert.equal(controls.toggle.textContent, 'ON-DEVICE E4B'); + controls.destroy(); +}); diff --git a/src/voice/voiceEngineControls.js b/src/voice/voiceEngineControls.js new file mode 100644 index 000000000..5a2f29c14 --- /dev/null +++ b/src/voice/voiceEngineControls.js @@ -0,0 +1,185 @@ +import { syncChipGroup } from '../ui/chipGroup.js'; +import { UiLifetime } from '../ui/uiLifetime.js'; +import { probeDevice, voiceEngineView } from './voiceEnginePresentation.js'; + +export const VOICE_ENGINE_TRAY_ID = 'gev-voice-engine-tray'; + +const setText = (element, text) => { + if (element && element.textContent !== text) element.textContent = text; +}; + +/** + * Cloud / On-device choice nested in the voice control: a heading toggle + * and a tray with the two engines, a hardware check with plain guidance, + * and a slot where the on-device provider mounts its model settings. + * Choosing an engine calls `onSelect`; the caller persists it and restarts + * voice on the new engine. + */ +export class VoiceEngineControls { + /** + * @param {object} options + * @param {HTMLElement} options.root The voice control root. + * @param {'cloud'|'on-device'} options.engine The running engine. + * @param {(engine: string) => void} [options.onSelect] + * @param {() => Promise} [options.probe] Hardware probe. + * @param {boolean} [options.open] Start with the tray open. + */ + constructor({ + root, + engine, + onSelect = () => {}, + probe = probeDevice, + open = false, + }) { + this.root = root; + this.engine = engine; + this.onSelect = onSelect; + this.probeDevice = probe; + this.probe = null; + this.probing = null; + this.badge = null; + this.open = false; + this.lifetime = new UiLifetime(); + const document = root?.ownerDocument; + if (!document?.createElement) return; + const make = (tag, className, parent, text) => { + const node = document.createElement(tag); + if (className) node.className = className; + if (text) node.textContent = text; + parent?.appendChild(node); + return node; + }; + + this.toggle = make('button', 'gev-voice-tier-btn gev-voice-engine-toggle'); + this.toggle.id = 'gev-voice-engine-toggle'; + this.toggle.type = 'button'; + this.toggle.setAttribute('aria-controls', VOICE_ENGINE_TRAY_ID); + this.toggle.setAttribute('aria-expanded', 'false'); + const cluster = + root.querySelector('.gev-voice-cost') || + root.querySelector('.gev-voice-heading') || + root; + cluster.insertBefore(this.toggle, cluster.children?.[0] || null); + + const tray = make('div', 'gev-voice-engine-tray', root); + tray.id = VOICE_ENGINE_TRAY_ID; + tray.setAttribute('role', 'group'); + tray.setAttribute('aria-label', 'Where voice runs'); + const heading = make('div', 'gev-voice-local-heading', tray); + make('span', 'gev-voice-local-kicker', heading, 'VOICE RUNS ON'); + const chips = make('div', 'gev-voice-local-chips', tray); + chips.setAttribute('role', 'group'); + chips.setAttribute('aria-label', 'Voice engine'); + const description = make('p', 'gev-voice-local-hint', tray); + const device = make('div', 'gev-voice-engine-device', tray); + const deviceHeading = make('div', 'gev-voice-local-heading', device); + make('span', 'gev-voice-local-label', deviceHeading, 'THIS DEVICE'); + const gpu = make('span', 'gev-voice-engine-fact', device); + const memory = make('span', 'gev-voice-engine-fact', device); + const guidance = make('ul', 'gev-voice-engine-guidance', device); + guidance.setAttribute('aria-label', 'Hardware guidance'); + const verdict = make('p', 'gev-voice-engine-verdict', device); + verdict.setAttribute('role', 'status'); + // The on-device provider renders its model settings here. + this.slot = make('div', 'gev-voice-local-slot', tray); + this.tray = tray; + this.elements = { + chips, + description, + device, + gpu, + memory, + guidance, + verdict, + }; + root.dataset.voiceEngine = engine; + + this.lifetime.listen(this.toggle, 'click', (event) => { + event.stopPropagation?.(); + this.setOpen(!this.open); + }); + this.lifetime.listen(chips, 'click', (event) => { + const button = event.target?.closest?.('[data-chip-id]'); + const id = button?.dataset.chipId; + if (!id || button.disabled || id === this.engine) return; + this.onSelect(id); + }); + if (typeof document.addEventListener === 'function') + this.lifetime.listen(document, 'keydown', (event) => { + if (event.key === 'Escape' && this.open) this.setOpen(false); + }); + this.render(); + // A saved on-device choice checks the hardware straight away so the + // tray can explain a refusal before voice starts. + if (open) this.setOpen(true); + else if (engine === 'on-device') this.checkDevice(); + } + + /** Runs the hardware probe once; later calls reuse it. */ + checkDevice() { + if (this.probing || this.lifetime.destroyed) return this.probing; + this.probing = Promise.resolve() + .then(() => this.probeDevice()) + .catch(() => ({ webgpu: false, secure: true })) + .then((probe) => { + this.probe = probe; + this.render(); + return probe; + }); + return this.probing; + } + + setOpen(open) { + this.open = Boolean(open); + if (this.open) this.checkDevice(); + this.render(); + } + + /** Short label the on-device provider shows on the toggle (e.g. model). */ + setBadge(badge) { + this.badge = badge || null; + this.render(); + } + + render() { + if (this.lifetime.destroyed || !this.tray) return; + const view = voiceEngineView({ + engine: this.engine, + probe: this.probe, + badge: this.badge, + }); + const el = this.elements; + setText(this.toggle, view.toggleLabel); + this.toggle.title = view.toggleTitle; + this.toggle.setAttribute('aria-expanded', String(this.open)); + this.toggle.classList.toggle('active', this.open); + if (this.open) this.root.dataset.voiceTray = 'open'; + else delete this.root.dataset.voiceTray; + syncChipGroup(el.chips, view.chips); + setText(el.description, view.description); + setText(el.gpu, view.device.gpu); + el.memory.hidden = !view.device.memory; + setText(el.memory, view.device.memory); + if (el.guidance.children.length !== view.guidance.length) { + for (const item of [...el.guidance.children]) item.remove(); + for (const line of view.guidance) { + const item = this.root.ownerDocument.createElement('li'); + item.textContent = line; + el.guidance.appendChild(item); + } + } + el.verdict.hidden = !view.device.verdict; + setText(el.verdict, view.device.verdict); + el.verdict.classList.toggle('error', view.unsupported); + } + + destroy() { + this.lifetime.destroy(); + this.toggle?.remove(); + this.tray?.remove(); + if (this.root?.dataset) { + delete this.root.dataset.voiceTray; + delete this.root.dataset.voiceEngine; + } + } +} diff --git a/src/voice/voiceEnginePresentation.js b/src/voice/voiceEnginePresentation.js new file mode 100644 index 000000000..e7b6d8a73 --- /dev/null +++ b/src/voice/voiceEnginePresentation.js @@ -0,0 +1,145 @@ +import { + checkLocalVoiceSupport, + findLlmModel, + recommendLlmModel, +} from './local/modelCatalog.js'; + +const gigabytes = (bytes) => `${Math.round(bytes / 1e9)} GB`; + +const modelLine = (id) => { + const model = findLlmModel(id); + return `${model.label} (${gigabytes(model.bytes)} download)`; +}; + +/** Plain hardware guidance for the on-device engine, largest first. */ +export const ON_DEVICE_GUIDANCE = Object.freeze([ + `Works best on 16 GB+ Apple Silicon: ${modelLine('gemma-4-12b')}`, + `8–16 GB: ${modelLine('gemma-4-e4b')}`, + 'Under 8 GB or no WebGPU: use Cloud', +]); + +export const ENGINE_DESCRIPTIONS = Object.freeze({ + cloud: + "OpenAI Realtime through this app's server. Needs an OpenAI key; billed per use.", + 'on-device': + 'Runs in this browser after a one-time download. No key or per-use cost; audio stays on this device.', +}); + +/** + * Reads what the browser reports about this device: secure context, WebGPU + * adapter and limits, and the (rounded, capped) memory estimate. + */ +export async function probeDevice({ + navigator: nav = globalThis.navigator, + secure = globalThis.isSecureContext, +} = {}) { + const probe = { + secure: secure !== false, + webgpu: Boolean(nav?.gpu), + adapter: false, + vendor: '', + architecture: '', + maxBufferSize: null, + deviceMemoryGB: Number(nav?.deviceMemory) || null, + }; + if (!probe.webgpu) return probe; + try { + const adapter = await nav.gpu.requestAdapter(); + if (adapter) { + probe.adapter = true; + probe.vendor = adapter.info?.vendor || ''; + probe.architecture = adapter.info?.architecture || ''; + probe.maxBufferSize = adapter.limits?.maxBufferSize ?? null; + } + } catch { + probe.adapter = false; + } + return probe; +} + +/** + * Turns a device probe into the hardware check lines and a verdict. + * @param {object|null} probe Result of probeDevice, null while checking. + */ +export function deviceCheckView(probe) { + if (!probe) + return { + state: 'checking', + gpu: 'Checking WebGPU…', + memory: '', + verdict: '', + suggested: null, + }; + const support = checkLocalVoiceSupport({ + hasWebGpu: probe.webgpu, + adapter: probe.webgpu ? probe.adapter : undefined, + maxBufferSize: probe.maxBufferSize ?? undefined, + deviceMemoryGB: probe.deviceMemoryGB ?? undefined, + isSecureContext: probe.secure, + model: findLlmModel('gemma-4-e4b'), + }); + const gpuName = [probe.vendor, probe.architecture].filter(Boolean).join(' '); + const gpu = !probe.webgpu + ? 'WebGPU: not available' + : !probe.adapter + ? 'WebGPU: no graphics adapter' + : `WebGPU: available${gpuName ? ` (${gpuName})` : ''}`; + // Browsers round the estimate down and cap it, so it is a lower bound. + const memory = probe.deviceMemoryGB + ? `Memory: browser reports at least ${probe.deviceMemoryGB} GB` + : 'Memory: not reported by this browser'; + if (!support.ok) + return { + state: 'unsupported', + gpu, + memory, + verdict: `On-device voice can't run here: ${support.reason} Use Cloud.`, + suggested: null, + }; + const suggested = recommendLlmModel({ + deviceMemoryGB: probe.deviceMemoryGB, + }); + return { + state: 'supported', + gpu, + memory, + verdict: `This device can run on-device voice. Suggested: ${modelLine(suggested.id)}.`, + suggested: suggested.id, + }; +} + +/** + * Derives the engine choice controls from the current engine and probe. + * @param {{engine: string, probe?: object|null, badge?: string|null}} state + */ +export function voiceEngineView({ engine, probe = null, badge = null }) { + const device = deviceCheckView(probe); + const onDevice = engine === 'on-device'; + return { + toggleLabel: onDevice ? badge || 'ON-DEVICE' : 'CLOUD', + toggleTitle: onDevice + ? 'Voice runs on this device · change or see models' + : 'Voice runs in the cloud · choose Cloud or On-device', + chips: [ + { + id: 'cloud', + label: 'CLOUD', + title: ENGINE_DESCRIPTIONS.cloud, + active: !onDevice, + }, + { + id: 'on-device', + label: 'ON-DEVICE', + title: ENGINE_DESCRIPTIONS['on-device'], + active: onDevice, + // Stay selectable while checking or when already chosen, so the + // choice can always be undone. + disabled: device.state === 'unsupported' && !onDevice, + }, + ], + description: ENGINE_DESCRIPTIONS[engine] || ENGINE_DESCRIPTIONS.cloud, + device, + guidance: ON_DEVICE_GUIDANCE, + unsupported: device.state === 'unsupported', + }; +} diff --git a/style.css b/style.css index 8019ec3f4..5cefc314d 100644 --- a/style.css +++ b/style.css @@ -19,6 +19,7 @@ @import './src/ui/styles/command-dock-sliding.css'; @import './src/ui/styles/voice-cost.css'; @import './src/ui/styles/voice-card.css'; +@import './src/ui/styles/voice-engine.css'; @import './src/ui/styles/first-run.css'; @import './src/ui/styles/provider-settings.css';