mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 02:44:02 +00:00
Merge branch 'main' into upstream/cctv-tarktee-refresh
This commit is contained in:
62 files changed
+16223
-195
No files matched your search
@@ -0,0 +1,115 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { PassThrough } from 'node:stream';
|
||||
import { localProviderPlugins } from '../server/providers/local.js';
|
||||
import { standaloneVoiceTools } from '../server/standalone/voiceTools.js';
|
||||
import {
|
||||
KEY_SETUP_KEYS,
|
||||
keySetupStatus,
|
||||
validateKeySetupUpdates,
|
||||
upsertDotenvValues,
|
||||
} from './keySetupCore.mjs';
|
||||
|
||||
test('local hosts opt into Gemini explicitly and retain the default plugin order', (t) => {
|
||||
t.mock.method(globalThis, 'fetch', () => {
|
||||
throw Error('construction must not fetch');
|
||||
});
|
||||
const defaults = localProviderPlugins().map((plugin) => plugin.name);
|
||||
assert.ok(!defaults.includes('gemini-live-proxy'));
|
||||
const enabled = localProviderPlugins({ gemini: {} }).map(
|
||||
(plugin) => plugin.name,
|
||||
);
|
||||
assert.equal(
|
||||
enabled.filter((name) => name === 'gemini-live-proxy').length,
|
||||
1,
|
||||
);
|
||||
assert.deepEqual(
|
||||
enabled.filter((name) => name !== 'gemini-live-proxy'),
|
||||
defaults,
|
||||
);
|
||||
});
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`opted-in ${hook} locks the standalone catalog into the Gemini token`, async () => {
|
||||
const tools = standaloneVoiceTools();
|
||||
const calls = [];
|
||||
const plugin = localProviderPlugins({
|
||||
gemini: {
|
||||
tools,
|
||||
resolveApiKey: () => 'fixture-key',
|
||||
resolveModel: () => 'gemini-3.8-live',
|
||||
resolveHost: () => '',
|
||||
resolveAllowedHosts: () => '',
|
||||
resolveRateLimit: () => '0',
|
||||
fetchImpl: async (url, options) => {
|
||||
calls.push({ url, options });
|
||||
return Response.json({ name: 'auth_tokens/fixture' });
|
||||
},
|
||||
},
|
||||
}).find((entry) => entry.name === 'gemini-live-proxy');
|
||||
let handler;
|
||||
plugin[hook]({
|
||||
middlewares: {
|
||||
use(route, value) {
|
||||
assert.equal(route, '/api/gemini/token');
|
||||
handler = value;
|
||||
},
|
||||
},
|
||||
});
|
||||
const req = new PassThrough();
|
||||
req.method = 'POST';
|
||||
req.headers = {
|
||||
host: 'localhost:5173',
|
||||
origin: 'http://localhost:5173',
|
||||
'content-type': 'application/json',
|
||||
};
|
||||
req.socket = { remoteAddress: '127.0.0.1' };
|
||||
const res = new EventEmitter();
|
||||
res.setHeader = () => {};
|
||||
res.end = (body) => {
|
||||
res.body = JSON.parse(body);
|
||||
res.writableEnded = true;
|
||||
};
|
||||
req.end('{}');
|
||||
await handler(req, res);
|
||||
assert.equal(res.statusCode, 200);
|
||||
assert.equal(calls.length, 1);
|
||||
const locked = JSON.parse(calls[0].options.body).bidiGenerateContentSetup
|
||||
.tools[0].functionDeclarations;
|
||||
assert.deepEqual(
|
||||
locked.map(({ name }) => name),
|
||||
tools.map(({ name }) => name),
|
||||
);
|
||||
assert.deepEqual(
|
||||
locked.map(({ parametersJsonSchema }) => parametersJsonSchema),
|
||||
tools.map(({ parameters }) => parameters),
|
||||
);
|
||||
assert.ok(locked.every(({ behavior }) => behavior === 'BLOCKING'));
|
||||
assert.deepEqual(res.body.config.tools[0].functionDeclarations, locked);
|
||||
assert.ok(!JSON.stringify(res.body).includes('fixture-key'));
|
||||
});
|
||||
}
|
||||
|
||||
test('Gemini key setup uses the existing server-only status and save contract', () => {
|
||||
const entry = KEY_SETUP_KEYS.find(({ id }) => id === 'gemini');
|
||||
assert.deepEqual(entry.envVars, ['GEMINI_API_KEY']);
|
||||
assert.ok(!entry.clientExposed);
|
||||
assert.equal(
|
||||
keySetupStatus({}).keys.find(({ id }) => id === 'gemini').set,
|
||||
false,
|
||||
);
|
||||
const env = { GEMINI_API_KEY: 'fixture-private-key' };
|
||||
const status = keySetupStatus(env);
|
||||
assert.equal(status.keys.find(({ id }) => id === 'gemini').set, true);
|
||||
assert.ok(!JSON.stringify(status).includes(env.GEMINI_API_KEY));
|
||||
assert.equal(validateKeySetupUpdates(env).ok, true);
|
||||
assert.match(
|
||||
upsertDotenvValues('GEMINI_API_KEY=old\n', env),
|
||||
/^GEMINI_API_KEY=fixture-private-key\n$/,
|
||||
);
|
||||
assert.equal(
|
||||
validateKeySetupUpdates({ VITE_GEMINI_API_KEY: 'fixture-private-key' }).ok,
|
||||
false,
|
||||
);
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -2890,5 +2890,56 @@
|
||||
"USB work abandoned after a stalled transfer": "Trabajo USB cancelado tras una transferencia bloqueada",
|
||||
"SDR decoder did not start; receiver stopped": "El decodificador SDR no arrancó; receptor detenido",
|
||||
"SDR decoder stopped responding; receiver stopped": "El decodificador SDR dejó de responder; receptor detenido",
|
||||
"SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedó atrás; receptor detenido"
|
||||
"SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedó atrás; receptor detenido",
|
||||
"Gemini returned an oversized audio chunk": "Gemini devolvió un fragmento de audio demasiado grande",
|
||||
"Gemini returned an unsupported audio format": "Gemini devolvió un formato de audio no compatible",
|
||||
"Gemini returned an invalid audio sample rate": "Gemini devolvió una frecuencia de muestreo de audio no válida",
|
||||
"Gemini returned incomplete PCM audio": "Gemini devolvió audio PCM incompleto",
|
||||
"This browser does not support Gemini microphone audio": "Este navegador no admite audio del micrófono para Gemini",
|
||||
"Microphone audio processing failed": "Falló el procesamiento del audio del micrófono",
|
||||
"Gemini audio playback fell behind. Reconnect to continue.": "La reproducción de audio de Gemini se ha retrasado. Vuelve a conectar para continuar.",
|
||||
"Gemini token response did not include an ephemeral credential": "La respuesta de Gemini no incluyó una credencial temporal",
|
||||
"Invalid Gemini connection endpoint": "El destino de conexión de Gemini no es válido",
|
||||
"Voice command context is too large": "El contexto del comando de voz es demasiado grande",
|
||||
"Gemini reliable transport exceeded its queue limit. Start voice again.": "La cola de envío fiable de Gemini superó su límite. Vuelve a iniciar la voz.",
|
||||
"Gemini reliable transport did not drain. Start voice again.": "La cola de envío fiable de Gemini no se vació. Vuelve a iniciar la voz.",
|
||||
"Gemini token request failed (HTTP {status})": "Falló la solicitud de credencial de Gemini (HTTP {status})",
|
||||
"Gemini token response did not include a valid model": "La respuesta de Gemini no incluyó un modelo válido",
|
||||
"Gemini token response did not include session settings": "La respuesta de Gemini no incluyó la configuración de la sesión",
|
||||
"Gemini session settings do not match the input mode": "La configuración de la sesión de Gemini no coincide con el modo de entrada",
|
||||
"Gemini could not submit this input. Start voice again.": "Gemini no pudo enviar esta entrada. Vuelve a iniciar la voz.",
|
||||
"Connecting to Gemini": "Conectando con Gemini",
|
||||
"Gemini returned an oversized message": "Gemini devolvió un mensaje demasiado grande",
|
||||
"Gemini Live connection failed. Check network access, then start voice again.": "Falló la conexión con Gemini Live. Comprueba el acceso a la red y vuelve a iniciar la voz.",
|
||||
"Gemini Live connection ended. Start voice again for a new conversation.": "La conexión con Gemini Live terminó. Vuelve a iniciar la voz para una nueva conversación.",
|
||||
"Gemini Live could not start. Start voice again to request a fresh session.": "Gemini Live no pudo iniciarse. Vuelve a iniciar la voz para solicitar una nueva sesión.",
|
||||
"Start voice again for a fresh token and a new conversation.": "Vuelve a iniciar la voz para obtener una credencial nueva y comenzar otra conversación.",
|
||||
"Start voice again to request a fresh session credential.": "Vuelve a iniciar la voz para solicitar una nueva credencial de sesión.",
|
||||
"Microphone permission was denied. Allow microphone access and try again.": "Se denegó el permiso del micrófono. Permite el acceso al micrófono e inténtalo de nuevo.",
|
||||
"Gemini connection timed out. Try again.": "Se agotó el tiempo de conexión con Gemini. Inténtalo de nuevo.",
|
||||
"Gemini voice could not start": "No se pudo iniciar la voz de Gemini",
|
||||
"Gemini did not finish the interrupted reply. Start voice again.": "Gemini no terminó la respuesta interrumpida. Vuelve a iniciar la voz.",
|
||||
"Gemini session expiring; finishing the current reply": "La sesión de Gemini va a caducar; terminando la respuesta actual",
|
||||
"Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini rechazó la sesión de voz. Vuelve a iniciar la voz para solicitar una nueva sesión.",
|
||||
"Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "Vuelve a iniciar la voz para obtener una nueva credencial de sesión. Si se repite, comprueba el modelo de Gemini y el acceso a la API.",
|
||||
"Gemini action history is full. Start voice again for a fresh session.": "El historial de acciones de Gemini está lleno. Vuelve a iniciar la voz para una nueva sesión.",
|
||||
"Gemini requested too many simultaneous actions": "Gemini solicitó demasiadas acciones simultáneas",
|
||||
"Voice text command is too long": "El comando de voz escrito es demasiado largo",
|
||||
"Gemini voice is not connected": "La voz de Gemini no está conectada",
|
||||
"Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "La entrada interrumpida de Gemini superó el límite de búfer o de espera. Vuelve a iniciar la voz.",
|
||||
"Gemini Live connection is expiring. Start voice again for a new conversation.": "La conexión con Gemini Live va a caducar. Vuelve a iniciar la voz para una nueva conversación.",
|
||||
"Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "Las sesiones de voz de Gemini tienen un límite de tiempo. Cuando caduque la sesión, vuelve a iniciar la voz para una nueva conversación.",
|
||||
"Gemini Live voice control": "Control por voz con Gemini Live",
|
||||
"Voice provider": "Proveedor de voz",
|
||||
"Changing provider reconnects voice if it is on.": "Cambiar de proveedor vuelve a conectar la voz si está activada.",
|
||||
"Gemini voice could not connect. Try again.": "No se pudo conectar la voz de Gemini. Inténtalo de nuevo.",
|
||||
"Gemini voice requires a same-origin app request": "La voz de Gemini requiere una solicitud del mismo origen que la aplicación",
|
||||
"Add a Gemini API key in Provider Settings": "Añade una clave de API de Gemini en los ajustes de proveedores",
|
||||
"The Gemini Live model configuration is invalid": "La configuración del modelo de Gemini Live no es válida",
|
||||
"Gemini voice is starting too often. Try again shortly.": "La voz de Gemini se está iniciando con demasiada frecuencia. Vuelve a intentarlo en breve.",
|
||||
"Gemini quota is unavailable. Try again later.": "La cuota de Gemini no está disponible. Inténtalo más tarde.",
|
||||
"Gemini access was refused. Check the API key and model access.": "Se denegó el acceso a Gemini. Comprueba la clave de API y el acceso al modelo.",
|
||||
"Gemini Live is unavailable for the configured model.": "Gemini Live no está disponible para el modelo configurado.",
|
||||
"Token request is too large": "La solicitud de credencial es demasiado grande",
|
||||
"Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío"
|
||||
}
|
||||
@@ -2957,5 +2957,56 @@
|
||||
"USB work abandoned after a stalled transfer": "Работа с USB прервана: передача данных зависла",
|
||||
"SDR decoder did not start; receiver stopped": "Декодер SDR не запустился; приёмник остановлен",
|
||||
"SDR decoder stopped responding; receiver stopped": "Декодер SDR перестал отвечать; приёмник остановлен",
|
||||
"SDR decoder fell behind; receiver stopped": "Декодер SDR не успевает; приёмник остановлен"
|
||||
"SDR decoder fell behind; receiver stopped": "Декодер SDR не успевает; приёмник остановлен",
|
||||
"Gemini returned an oversized audio chunk": "Gemini вернул слишком большой фрагмент аудио",
|
||||
"Gemini returned an unsupported audio format": "Gemini вернул неподдерживаемый формат аудио",
|
||||
"Gemini returned an invalid audio sample rate": "Gemini вернул недопустимую частоту дискретизации аудио",
|
||||
"Gemini returned incomplete PCM audio": "Gemini вернул неполные аудиоданные PCM",
|
||||
"This browser does not support Gemini microphone audio": "Этот браузер не поддерживает звук с микрофона для Gemini",
|
||||
"Microphone audio processing failed": "Не удалось обработать звук с микрофона",
|
||||
"Gemini audio playback fell behind. Reconnect to continue.": "Воспроизведение звука Gemini отстаёт. Подключитесь заново, чтобы продолжить.",
|
||||
"Gemini token response did not include an ephemeral credential": "В ответе Gemini нет временных учётных данных",
|
||||
"Invalid Gemini connection endpoint": "Недопустимый адрес подключения к Gemini",
|
||||
"Voice command context is too large": "Контекст голосовой команды слишком велик",
|
||||
"Gemini reliable transport exceeded its queue limit. Start voice again.": "Очередь надёжной передачи Gemini превысила лимит. Запустите голосовой режим заново.",
|
||||
"Gemini reliable transport did not drain. Start voice again.": "Очередь надёжной передачи Gemini не освободилась. Запустите голосовой режим заново.",
|
||||
"Gemini token request failed (HTTP {status})": "Не удалось запросить токен Gemini (HTTP {status})",
|
||||
"Gemini token response did not include a valid model": "В ответе Gemini нет допустимой модели",
|
||||
"Gemini token response did not include session settings": "В ответе Gemini нет настроек сеанса",
|
||||
"Gemini session settings do not match the input mode": "Настройки сеанса Gemini не соответствуют режиму ввода",
|
||||
"Gemini could not submit this input. Start voice again.": "Gemini не удалось отправить эти данные. Запустите голосовой режим заново.",
|
||||
"Connecting to Gemini": "Подключение к Gemini",
|
||||
"Gemini returned an oversized message": "Gemini вернул слишком большое сообщение",
|
||||
"Gemini Live connection failed. Check network access, then start voice again.": "Не удалось подключиться к Gemini Live. Проверьте доступ к сети и запустите голосовой режим заново.",
|
||||
"Gemini Live connection ended. Start voice again for a new conversation.": "Соединение с Gemini Live завершено. Запустите голосовой режим заново для нового разговора.",
|
||||
"Gemini Live could not start. Start voice again to request a fresh session.": "Не удалось запустить Gemini Live. Запустите голосовой режим заново, чтобы запросить новый сеанс.",
|
||||
"Start voice again for a fresh token and a new conversation.": "Запустите голосовой режим заново, чтобы получить новый токен и начать новый разговор.",
|
||||
"Start voice again to request a fresh session credential.": "Запустите голосовой режим заново, чтобы запросить новые учётные данные сеанса.",
|
||||
"Microphone permission was denied. Allow microphone access and try again.": "Доступ к микрофону запрещён. Разрешите доступ и повторите попытку.",
|
||||
"Gemini connection timed out. Try again.": "Время ожидания подключения к Gemini истекло. Повторите попытку.",
|
||||
"Gemini voice could not start": "Не удалось запустить голосовой режим Gemini",
|
||||
"Gemini did not finish the interrupted reply. Start voice again.": "Gemini не завершил прерванный ответ. Запустите голосовой режим заново.",
|
||||
"Gemini session expiring; finishing the current reply": "Сеанс Gemini истекает; завершается текущий ответ",
|
||||
"Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini отклонил голосовой сеанс. Запустите голосовой режим заново, чтобы запросить новый сеанс.",
|
||||
"Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "Запустите голосовой режим заново для получения новых учётных данных. Если ошибка повторится, проверьте модель Gemini и доступ к API.",
|
||||
"Gemini action history is full. Start voice again for a fresh session.": "История действий Gemini заполнена. Запустите голосовой режим заново для нового сеанса.",
|
||||
"Gemini requested too many simultaneous actions": "Gemini запросил слишком много одновременных действий",
|
||||
"Voice text command is too long": "Текст голосовой команды слишком длинный",
|
||||
"Gemini voice is not connected": "Голосовой режим Gemini не подключён",
|
||||
"Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "Прерванный ввод Gemini превысил лимит буфера или ожидания. Запустите голосовой режим заново.",
|
||||
"Gemini Live connection is expiring. Start voice again for a new conversation.": "Соединение с Gemini Live истекает. Запустите голосовой режим заново для нового разговора.",
|
||||
"Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "Время голосовых сеансов Gemini ограничено. Когда сеанс истечёт, запустите голосовой режим заново для нового разговора.",
|
||||
"Gemini Live voice control": "Голосовое управление через Gemini Live",
|
||||
"Voice provider": "Поставщик голосового сервиса",
|
||||
"Changing provider reconnects voice if it is on.": "Смена поставщика переподключает голосовой режим, если он включён.",
|
||||
"Gemini voice could not connect. Try again.": "Не удалось подключить голосовой режим Gemini. Повторите попытку.",
|
||||
"Gemini voice requires a same-origin app request": "Голосовой режим Gemini требует запрос из того же источника, что и приложение",
|
||||
"Add a Gemini API key in Provider Settings": "Добавьте ключ API Gemini в настройках поставщиков",
|
||||
"The Gemini Live model configuration is invalid": "Недопустимая конфигурация модели Gemini Live",
|
||||
"Gemini voice is starting too often. Try again shortly.": "Голосовой режим Gemini запускается слишком часто. Повторите попытку чуть позже.",
|
||||
"Gemini quota is unavailable. Try again later.": "Квота Gemini недоступна. Повторите попытку позже.",
|
||||
"Gemini access was refused. Check the API key and model access.": "Доступ к Gemini запрещён. Проверьте ключ API и доступ к модели.",
|
||||
"Gemini Live is unavailable for the configured model.": "Gemini Live недоступен для выбранной модели.",
|
||||
"Token request is too large": "Запрос токена слишком велик",
|
||||
"Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON"
|
||||
}
|
||||
@@ -19,7 +19,7 @@ import { hasProxySignals } from './localRequestGate.mjs';
|
||||
/** Longest accepted key/token value. Real provider keys are all far shorter. */
|
||||
export const KEY_SETUP_VALUE_LIMIT = 512;
|
||||
|
||||
/** Most env vars accepted in one save. The registry defines ten. */
|
||||
/** Most env vars accepted in one save. */
|
||||
export const KEY_SETUP_UPDATE_LIMIT = 16;
|
||||
|
||||
/** Header line written above keys the panel appends to a .env file. */
|
||||
@@ -65,6 +65,14 @@ export const KEY_SETUP_KEYS = Object.freeze([
|
||||
envVars: Object.freeze(['OPENAI_API_KEY']),
|
||||
tier: 'metered',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'gemini',
|
||||
title: 'GEMINI', // i18n-ignore: provider product name
|
||||
unlocks: msg('Gemini Live voice control'),
|
||||
getUrl: 'https://aistudio.google.com/apikey',
|
||||
envVars: Object.freeze(['GEMINI_API_KEY']),
|
||||
tier: 'metered',
|
||||
}),
|
||||
Object.freeze({
|
||||
id: 'aisstream',
|
||||
title: 'AISSTREAM', // i18n-ignore: provider product name
|
||||
|
||||
+44
-10
@@ -3,6 +3,7 @@ import {
|
||||
FRAME_SIGNATURE_H,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
PROJECTION_MATTE_COLOR,
|
||||
PROJECTION_TEXTURE_SWAP_MS,
|
||||
ACTIVE_FRAME_REFRESH_MS,
|
||||
FRAME_ENDPOINT,
|
||||
@@ -12,6 +13,37 @@ import {
|
||||
PLACEHOLDER_REPAINT_MS,
|
||||
} from './policy.js';
|
||||
|
||||
/**
|
||||
* Contain-fits a source frame into the projection canvas: scaled to the
|
||||
* largest size that fits, centered, with the margins left for the matte fill.
|
||||
* Falls back to the full canvas when either dimension is missing or zero.
|
||||
*
|
||||
* @param {number} sourceW - Source width in pixels.
|
||||
* @param {number} sourceH - Source height in pixels.
|
||||
* @returns {{x:number,y:number,w:number,h:number}} Draw rectangle in canvas pixels.
|
||||
*/
|
||||
export function fitToProjection(sourceW, sourceH) {
|
||||
if (!sourceW || !sourceH)
|
||||
return {
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: PROJECTION_CANVAS_WIDTH,
|
||||
h: PROJECTION_CANVAS_HEIGHT,
|
||||
};
|
||||
const scale = Math.min(
|
||||
PROJECTION_CANVAS_WIDTH / sourceW,
|
||||
PROJECTION_CANVAS_HEIGHT / sourceH,
|
||||
);
|
||||
const w = Math.round(sourceW * scale);
|
||||
const h = Math.round(sourceH * scale);
|
||||
return {
|
||||
x: Math.round((PROJECTION_CANVAS_WIDTH - w) / 2),
|
||||
y: Math.round((PROJECTION_CANVAS_HEIGHT - h) / 2),
|
||||
w,
|
||||
h,
|
||||
};
|
||||
}
|
||||
|
||||
export function createFrames({ state: layerState, services, parts, source }) {
|
||||
/**
|
||||
* FNV-1a over the RGB channels of a downsampled frame. Pure (takes the raw
|
||||
@@ -182,7 +214,7 @@ export function createFrames({ state: layerState, services, parts, source }) {
|
||||
ctx.clearRect(0, 0, w, h);
|
||||
const g = ctx.createLinearGradient(0, 0, w, h);
|
||||
g.addColorStop(0, '#05111a');
|
||||
g.addColorStop(1, '#01070c');
|
||||
g.addColorStop(1, PROJECTION_MATTE_COLOR);
|
||||
ctx.fillStyle = g;
|
||||
ctx.fillRect(0, 0, w, h);
|
||||
|
||||
@@ -320,7 +352,14 @@ export function createFrames({ state: layerState, services, parts, source }) {
|
||||
// base color (planeMaterial color = WHITE, alpha .95) for the frame or
|
||||
// two Cesium needs to rebind, which IS the periodic white flash from the
|
||||
// owner field tests (2026-07-04 and 2026-07-30).
|
||||
const signature = projectionFrameSignature(runtime);
|
||||
const fit = fitToProjection(
|
||||
runtime.image.naturalWidth,
|
||||
runtime.image.naturalHeight,
|
||||
);
|
||||
// Pixels are hashed at a fixed size, so the fit carries the aspect.
|
||||
const pixels = projectionFrameSignature(runtime);
|
||||
const signature =
|
||||
pixels === null ? null : `${pixels}:${fit.w}x${fit.h}`;
|
||||
runtime.drawnImageStamp = runtime.imageStamp;
|
||||
if (signature !== null && signature === runtime.lastFrameSignature) {
|
||||
// Identical pixels — leave the canvas, and therefore the bound
|
||||
@@ -328,19 +367,14 @@ export function createFrames({ state: layerState, services, parts, source }) {
|
||||
return;
|
||||
}
|
||||
runtime.lastFrameSignature = signature;
|
||||
runtime.ctx.clearRect(
|
||||
0,
|
||||
0,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
);
|
||||
runtime.ctx.drawImage(
|
||||
runtime.image,
|
||||
runtime.ctx.fillStyle = PROJECTION_MATTE_COLOR;
|
||||
runtime.ctx.fillRect(
|
||||
0,
|
||||
0,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
);
|
||||
runtime.ctx.drawImage(runtime.image, fit.x, fit.y, fit.w, fit.h);
|
||||
runtime.canvasStamp = (runtime.canvasStamp || 0) + 1;
|
||||
runtime.lastPlaceholderPaintAt = 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
// src/layers/cctv/frames.test.mjs
|
||||
// Still-frame fit (#1201): stills draw at their source aspect, centered on the
|
||||
// 1920x1080 projection canvas, instead of being stretched to fill it.
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createFrames, fitToProjection } from './frames.js';
|
||||
import {
|
||||
FRAME_SIGNATURE_W,
|
||||
FRAME_SIGNATURE_H,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
PROJECTION_MATTE_COLOR,
|
||||
} from './policy.js';
|
||||
|
||||
// The expected rectangles below are written for a 1920x1080 canvas.
|
||||
assert.equal(PROJECTION_CANVAS_WIDTH, 1920);
|
||||
assert.equal(PROJECTION_CANVAS_HEIGHT, 1080);
|
||||
|
||||
const FULL_CANVAS = {
|
||||
x: 0,
|
||||
y: 0,
|
||||
w: PROJECTION_CANVAS_WIDTH,
|
||||
h: PROJECTION_CANVAS_HEIGHT,
|
||||
};
|
||||
|
||||
test('a 4:3 still is pillarboxed at full height', () => {
|
||||
assert.deepEqual(fitToProjection(640, 480), {
|
||||
x: 240,
|
||||
y: 0,
|
||||
w: 1440,
|
||||
h: 1080,
|
||||
});
|
||||
});
|
||||
|
||||
test('a source wider than 16:9 is letterboxed at full width', () => {
|
||||
assert.deepEqual(fitToProjection(3840, 1080), {
|
||||
x: 0,
|
||||
y: 270,
|
||||
w: 1920,
|
||||
h: 540,
|
||||
});
|
||||
});
|
||||
|
||||
test('a 16:9 still fills the canvas', () => {
|
||||
assert.deepEqual(fitToProjection(1920, 1080), FULL_CANVAS);
|
||||
});
|
||||
|
||||
test('a missing or zero dimension falls back to the full canvas', () => {
|
||||
assert.deepEqual(fitToProjection(0, 480), FULL_CANVAS);
|
||||
assert.deepEqual(fitToProjection(640, 0), FULL_CANVAS);
|
||||
assert.deepEqual(fitToProjection(640, undefined), FULL_CANVAS);
|
||||
assert.deepEqual(fitToProjection(undefined, undefined), FULL_CANVAS);
|
||||
});
|
||||
|
||||
// Draw path: the fit reaches drawImage, and the unchanged-frame skip keys on the
|
||||
// fitted size as well as the pixels (the 64x36 hash alone drops aspect).
|
||||
function stillHarness() {
|
||||
const calls = [];
|
||||
const ctx = {
|
||||
fillStyle: null,
|
||||
fillRect: (...rect) => calls.push(['fillRect', ctx.fillStyle, ...rect]),
|
||||
clearRect: () => {},
|
||||
drawImage: (_image, ...rect) => calls.push(['drawImage', ...rect]),
|
||||
};
|
||||
// Every frame downsamples to the same pixels, so only the size differs.
|
||||
const pixels = new Uint8ClampedArray(
|
||||
FRAME_SIGNATURE_W * FRAME_SIGNATURE_H * 4,
|
||||
).fill(7);
|
||||
const runtime = {
|
||||
mode: 'image',
|
||||
ctx,
|
||||
image: { naturalWidth: 0, naturalHeight: 0 },
|
||||
imageReady: true,
|
||||
// In flight, so refreshProjectionImage returns before touching image.src.
|
||||
imageLoading: true,
|
||||
imageStamp: 0,
|
||||
drawnImageStamp: -1,
|
||||
lastFrameSignature: null,
|
||||
canvasStamp: 0,
|
||||
signatureCtx: {
|
||||
clearRect: () => {},
|
||||
drawImage: () => {},
|
||||
getImageData: () => ({ data: pixels }),
|
||||
},
|
||||
};
|
||||
const record = { camera: { id: 'still' }, projection: runtime };
|
||||
const { drawProjectionFrame } = createFrames({
|
||||
state: { _healthById: new Map(), _activeCameraId: null },
|
||||
services: {},
|
||||
parts: {},
|
||||
source: {},
|
||||
});
|
||||
const load = (width, height) => {
|
||||
runtime.image = { naturalWidth: width, naturalHeight: height };
|
||||
runtime.imageStamp += 1;
|
||||
calls.length = 0;
|
||||
drawProjectionFrame(record);
|
||||
return calls.slice();
|
||||
};
|
||||
return { runtime, load };
|
||||
}
|
||||
|
||||
test('a still is drawn into its fitted rectangle over a full matte', () => {
|
||||
const { runtime, load } = stillHarness();
|
||||
assert.deepEqual(load(640, 480), [
|
||||
[
|
||||
'fillRect',
|
||||
PROJECTION_MATTE_COLOR,
|
||||
0,
|
||||
0,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
],
|
||||
['drawImage', 240, 0, 1440, 1080],
|
||||
]);
|
||||
assert.equal(runtime.canvasStamp, 1);
|
||||
});
|
||||
|
||||
test('same pixels at the same size skip the redraw', () => {
|
||||
const { runtime, load } = stillHarness();
|
||||
load(640, 480);
|
||||
assert.deepEqual(load(640, 480), []);
|
||||
assert.equal(runtime.canvasStamp, 1);
|
||||
});
|
||||
|
||||
test('same pixels at a new aspect redraw at the new fit', () => {
|
||||
const { runtime, load } = stillHarness();
|
||||
load(640, 480);
|
||||
assert.deepEqual(load(1920, 1080), [
|
||||
[
|
||||
'fillRect',
|
||||
PROJECTION_MATTE_COLOR,
|
||||
0,
|
||||
0,
|
||||
PROJECTION_CANVAS_WIDTH,
|
||||
PROJECTION_CANVAS_HEIGHT,
|
||||
],
|
||||
['drawImage', 0, 0, 1920, 1080],
|
||||
]);
|
||||
assert.equal(runtime.canvasStamp, 2);
|
||||
});
|
||||
@@ -35,6 +35,9 @@ export const PROJECTION_CANVAS_WIDTH = 1920;
|
||||
|
||||
export const PROJECTION_CANVAS_HEIGHT = 1080;
|
||||
|
||||
/** Matte behind fitted stills; the placeholder gradient's dark end. */
|
||||
export const PROJECTION_MATTE_COLOR = '#01070c';
|
||||
|
||||
// Downsample grid for the unchanged-frame signature (drawProjectionFrame).
|
||||
// 64x36 keeps the 16:9 aspect and reads ~9 KB per check versus the 8.3 MB a
|
||||
// full-resolution compare would touch.
|
||||
|
||||
@@ -17,6 +17,10 @@ export function createStandaloneTools(options) {
|
||||
},
|
||||
},
|
||||
...options,
|
||||
voice: { toolCatalog: loadToolCatalog, ...options?.voice },
|
||||
voice: {
|
||||
supportedProviders: ['openai', 'gemini'],
|
||||
toolCatalog: loadToolCatalog,
|
||||
...options?.voice,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVoiceSession } from '../voice/session.js';
|
||||
import { createGevActionRunner } from '../voice/gevActions.js';
|
||||
import { createGeminiSession } from '../voice/geminiSession.js';
|
||||
import { createNarrationScheduler } from '../voice/narration.js';
|
||||
|
||||
export const turnOfEventLoop = () =>
|
||||
new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
const promise = new Promise((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function ownedClock() {
|
||||
let now = 0;
|
||||
let nextId = 0;
|
||||
const timers = new Map();
|
||||
return {
|
||||
timers,
|
||||
now: () => now,
|
||||
setTimer(fn, delay = 0) {
|
||||
const id = ++nextId;
|
||||
timers.set(id, { fn, at: now + Math.max(0, delay) });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => timers.delete(id),
|
||||
advance(ms) {
|
||||
const until = now + ms;
|
||||
let callbacks = 0;
|
||||
for (;;) {
|
||||
const due = [...timers]
|
||||
.filter(([, timer]) => timer.at <= until)
|
||||
.sort((a, b) => a[1].at - b[1].at || a[0] - b[0])[0];
|
||||
if (!due) break;
|
||||
assert.ok(
|
||||
++callbacks <= 1000,
|
||||
'fixture timer queue must remain bounded',
|
||||
);
|
||||
timers.delete(due[0]);
|
||||
now = due[1].at;
|
||||
due[1].fn();
|
||||
}
|
||||
now = until;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Real common session -> real Gemini adapter -> real action runner.
|
||||
* Only devices, transport, passive UI and wall-clock facilities are fakes.
|
||||
* The optional result gate delays return AFTER the actual action commits.
|
||||
*/
|
||||
export function nativeIntegrationFixture(t) {
|
||||
const clock = ownedClock();
|
||||
const events = [];
|
||||
const sockets = [];
|
||||
const tracks = [];
|
||||
const audios = [];
|
||||
const requests = [];
|
||||
const runnerCalls = [];
|
||||
const runnerJobs = [];
|
||||
const resultGates = new Map();
|
||||
const acquisitions = { microphone: 0, audio: 0, token: 0, socket: 0 };
|
||||
const state = { height: 1000, renders: 0, zooms: [], earconStarts: 0 };
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: state.height },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(metres) {
|
||||
position.x -= metres;
|
||||
state.height -= metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms.push({ direction: 'in', metres });
|
||||
},
|
||||
zoomOut(metres) {
|
||||
position.x += metres;
|
||||
state.height += metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms.push({ direction: 'out', metres });
|
||||
},
|
||||
},
|
||||
scene: {
|
||||
canvas: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
},
|
||||
requestRender() {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
const dataManager = { layers: new Map() };
|
||||
const actualRunner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager,
|
||||
styleManager: {},
|
||||
});
|
||||
const runner = (name, args, options) => {
|
||||
runnerCalls.push({ name, args, options });
|
||||
const job = (async () => {
|
||||
// No fake action result: the camera and envelope come from production.
|
||||
const result = await actualRunner(name, args, options);
|
||||
const gate = resultGates.get(options.callId);
|
||||
if (gate) {
|
||||
gate.entered.resolve(result);
|
||||
await gate.release.promise;
|
||||
}
|
||||
return result;
|
||||
})();
|
||||
runnerJobs.push(job);
|
||||
return job;
|
||||
};
|
||||
class Socket {
|
||||
constructor(url) {
|
||||
acquisitions.socket++;
|
||||
this.url = url;
|
||||
this.readyState = 0;
|
||||
this.bufferedAmount = 0;
|
||||
this.sent = [];
|
||||
sockets.push(this);
|
||||
queueMicrotask(() => {
|
||||
if (this.closed) return;
|
||||
this.readyState = 1;
|
||||
this.onopen?.();
|
||||
});
|
||||
}
|
||||
send(raw) {
|
||||
assert.equal(
|
||||
this.readyState,
|
||||
1,
|
||||
'sending requires the owned open socket',
|
||||
);
|
||||
const message = JSON.parse(raw);
|
||||
this.sent.push(message);
|
||||
if (message.setup)
|
||||
queueMicrotask(() => this.message({ setupComplete: {} }));
|
||||
}
|
||||
message(message) {
|
||||
this.onmessage?.({ data: JSON.stringify(message) });
|
||||
}
|
||||
close() {
|
||||
this.closed = true;
|
||||
this.readyState = 3;
|
||||
}
|
||||
}
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
createAdapter: (hooks) =>
|
||||
createGeminiSession({
|
||||
...hooks,
|
||||
ui: { root: { remove() {} } },
|
||||
dataManager,
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
WebSocketClass: Socket,
|
||||
getUserMedia: async () => {
|
||||
acquisitions.microphone++;
|
||||
const track = {
|
||||
enabled: true,
|
||||
readyState: 'live',
|
||||
stopCalls: 0,
|
||||
stop() {
|
||||
this.stopCalls++;
|
||||
this.readyState = 'ended';
|
||||
},
|
||||
};
|
||||
tracks.push(track);
|
||||
return { getTracks: () => [track], getAudioTracks: () => [track] };
|
||||
},
|
||||
fetchImpl: async (url, init) => {
|
||||
acquisitions.token++;
|
||||
requests.push({ url, init });
|
||||
const manual = JSON.parse(init.body).inputMode === 'push-to-talk';
|
||||
return Response.json({
|
||||
token: 'auth_tokens/fixture-not-a-credential',
|
||||
model: 'gemini-3.8-live',
|
||||
websocketUrl:
|
||||
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained',
|
||||
config: {
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
...(manual
|
||||
? {
|
||||
realtimeInputConfig: {
|
||||
automaticActivityDetection: { disabled: true },
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
},
|
||||
createInput: ({ readStream, operations }) => ({
|
||||
...operations,
|
||||
setMicrophoneEnabled(enabled) {
|
||||
for (const track of readStream()?.getAudioTracks() || [])
|
||||
track.enabled = enabled;
|
||||
},
|
||||
updateVoiceButtonLabel() {},
|
||||
setVoiceSpeaker() {},
|
||||
startVoiceVisualizer() {},
|
||||
startAssistantVoiceVisualizer() {},
|
||||
cancelPushToTalkHold() {},
|
||||
stopVoiceVisualizer() {},
|
||||
detachBindings() {},
|
||||
bindPushToTalkShortcut() {},
|
||||
resetSession() {
|
||||
this.pushToTalkMode =
|
||||
this.pushToTalkKeyHeld =
|
||||
this.spaceKeyHeld =
|
||||
false;
|
||||
},
|
||||
}),
|
||||
createAudio: (callbacks) => {
|
||||
acquisitions.audio++;
|
||||
const audio = {
|
||||
pending: false,
|
||||
closed: false,
|
||||
outputStream: {},
|
||||
async initialize() {},
|
||||
connectMicrophone(stream, enabled, metadata) {
|
||||
this.stream = stream;
|
||||
this.setMicrophoneEnabled(enabled, metadata);
|
||||
},
|
||||
setMicrophoneEnabled(enabled, metadata) {
|
||||
this.enabled = enabled;
|
||||
this.metadata = metadata;
|
||||
},
|
||||
play() {
|
||||
this.pending = true;
|
||||
},
|
||||
clearPlayback() {
|
||||
this.pending = false;
|
||||
},
|
||||
drain() {
|
||||
this.pending = false;
|
||||
callbacks.onDrain();
|
||||
},
|
||||
diagnostics: () => ({ outputChunks: 0 }),
|
||||
async close() {
|
||||
this.closed = true;
|
||||
this.pending = false;
|
||||
return true;
|
||||
},
|
||||
};
|
||||
audios.push(audio);
|
||||
return audio;
|
||||
},
|
||||
createNarration: (options) =>
|
||||
createNarrationScheduler({
|
||||
...options,
|
||||
now: clock.now,
|
||||
setTimer: clock.setTimer,
|
||||
clearTimer: clock.clearTimer,
|
||||
}),
|
||||
createProgressEarcon: () => ({
|
||||
available: () => true,
|
||||
start() {
|
||||
state.earconStarts++;
|
||||
},
|
||||
stop() {},
|
||||
dispose() {},
|
||||
}),
|
||||
captureViewport: async () => null,
|
||||
}),
|
||||
});
|
||||
session.subscribe((event) => events.push(event));
|
||||
const controller = session.adapter.controller;
|
||||
t.after(async () => {
|
||||
session.destroy();
|
||||
for (const gate of resultGates.values()) gate.release.resolve();
|
||||
await Promise.allSettled(runnerJobs);
|
||||
actualRunner.dispose();
|
||||
assert.equal(clock.timers.size, 0, 'fixture must leave no owned timer');
|
||||
assert.ok(audios.every((audio) => audio.closed));
|
||||
assert.ok(tracks.every((track) => track.readyState === 'ended'));
|
||||
assert.ok(sockets.every((socket) => socket.closed));
|
||||
});
|
||||
return {
|
||||
session,
|
||||
controller,
|
||||
state,
|
||||
events,
|
||||
sockets,
|
||||
tracks,
|
||||
audios,
|
||||
requests,
|
||||
acquisitions,
|
||||
runnerCalls,
|
||||
clock,
|
||||
holdResult(callId) {
|
||||
const gate = { entered: deferred(), release: deferred() };
|
||||
resultGates.set(callId, gate);
|
||||
return { ready: gate.entered.promise, release: gate.release.resolve };
|
||||
},
|
||||
async start() {
|
||||
await session.start();
|
||||
assert.equal(session.state, 'listening');
|
||||
assert.equal(controller.getDiagnostics().connected, true);
|
||||
return sockets.at(-1);
|
||||
},
|
||||
activity(socket = sockets.at(-1)) {
|
||||
socket.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
socket.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
},
|
||||
tool(callId, socket = sockets.at(-1)) {
|
||||
socket.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: callId,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
// A deterministic settlement boundary, not a fake action dispatcher.
|
||||
return controller.toolQueue;
|
||||
},
|
||||
outputs(socket = sockets.at(-1)) {
|
||||
return socket.sent.flatMap(
|
||||
(message) => message.toolResponse?.functionResponses || [],
|
||||
);
|
||||
},
|
||||
complete(socket = sockets.at(-1)) {
|
||||
socket.message({ serverContent: { turnComplete: true } });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function assertSuccessfulZoom(h, socket, callId, beforeHeight) {
|
||||
assert.equal(h.state.height, beforeHeight * 0.75);
|
||||
assert.ok(h.state.renders > 0);
|
||||
const calls = h.events.filter(
|
||||
(event) => event.type === 'action-call' && event.callId === callId,
|
||||
);
|
||||
const results = h.events.filter(
|
||||
(event) => event.type === 'action-result' && event.callId === callId,
|
||||
);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(results.length, 1);
|
||||
assert.equal(calls[0].name, 'adjust_camera_zoom');
|
||||
assert.deepEqual(calls[0].arguments, { direction: 'in', amount: 'little' });
|
||||
assert.equal(results[0].result.ok, true);
|
||||
assert.equal(results[0].result.action, 'adjust_camera_zoom');
|
||||
assert.equal(results[0].result.beforeHeightM, Math.round(beforeHeight));
|
||||
assert.equal(results[0].result.afterHeightM, Math.round(h.state.height));
|
||||
const outputs = h.outputs(socket).filter((output) => output.id === callId);
|
||||
assert.equal(outputs.length, 1);
|
||||
assert.equal(outputs[0].name, 'adjust_camera_zoom');
|
||||
assert.equal(outputs[0].response.ok, true);
|
||||
assert.equal(outputs[0].response.action, 'adjust_camera_zoom');
|
||||
assert.equal(outputs[0].response.afterHeightM, Math.round(h.state.height));
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
import { GeminiSessionController } from '../voice/geminiSession.js';
|
||||
import { createNarrationScheduler } from '../voice/narration.js';
|
||||
|
||||
const endpoint =
|
||||
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
|
||||
const credential = {
|
||||
token: 'auth_tokens/test',
|
||||
model: 'gemini-3.8-live',
|
||||
websocketUrl: endpoint,
|
||||
config: { generationConfig: { responseModalities: ['AUDIO'] } },
|
||||
};
|
||||
const manualCredential = {
|
||||
...credential,
|
||||
config: {
|
||||
...credential.config,
|
||||
realtimeInputConfig: { automaticActivityDetection: { disabled: true } },
|
||||
},
|
||||
};
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve));
|
||||
function deferred() {
|
||||
let resolve;
|
||||
const promise = new Promise((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
function harness(overrides = {}) {
|
||||
const { autoSetup = true, ...controllerOverrides } = overrides;
|
||||
const events = [];
|
||||
const sockets = [];
|
||||
const media = [];
|
||||
const tracks = [];
|
||||
const captures = [];
|
||||
const calls = [];
|
||||
const requests = [];
|
||||
const state = { playing: 0, ducked: false, stopped: 0 };
|
||||
let clock = 0;
|
||||
let nextTimer = 0;
|
||||
const timers = new Map();
|
||||
const earcon = { playing: false, starts: 0, disposals: 0 };
|
||||
const advance = (ms) => {
|
||||
const target = clock + ms;
|
||||
for (;;) {
|
||||
const due = [...timers]
|
||||
.filter(([, timer]) => timer.at <= target)
|
||||
.sort((a, b) => a[1].at - b[1].at)[0];
|
||||
if (!due) break;
|
||||
timers.delete(due[0]);
|
||||
clock = due[1].at;
|
||||
due[1].fn();
|
||||
}
|
||||
clock = target;
|
||||
};
|
||||
let radioControl;
|
||||
const radio = {
|
||||
setVoiceDucked(value) {
|
||||
state.ducked = value;
|
||||
},
|
||||
pause() {},
|
||||
subscribePlaybackControls(callback) {
|
||||
radioControl = callback;
|
||||
return () => {
|
||||
radioControl = null;
|
||||
};
|
||||
},
|
||||
async playForVoice() {
|
||||
state.playing++;
|
||||
return true;
|
||||
},
|
||||
stopPlayback() {
|
||||
state.stopped++;
|
||||
},
|
||||
};
|
||||
class Socket {
|
||||
constructor(url) {
|
||||
this.url = url;
|
||||
this.readyState = 0;
|
||||
this.bufferedAmount = 0;
|
||||
this.sent = [];
|
||||
sockets.push(this);
|
||||
queueMicrotask(() => {
|
||||
this.readyState = 1;
|
||||
this.onopen?.();
|
||||
});
|
||||
}
|
||||
send(raw) {
|
||||
const value = JSON.parse(raw);
|
||||
this.sent.push(value);
|
||||
if (value.setup && autoSetup)
|
||||
queueMicrotask(() => this.message({ setupComplete: {} }));
|
||||
}
|
||||
message(value) {
|
||||
this.onmessage?.({ data: JSON.stringify(value) });
|
||||
}
|
||||
error() {
|
||||
this.onerror?.();
|
||||
}
|
||||
remoteClose() {
|
||||
this.readyState = 3;
|
||||
this.onclose?.();
|
||||
}
|
||||
close() {
|
||||
this.readyState = 3;
|
||||
this.closed = true;
|
||||
}
|
||||
}
|
||||
const makeStream = () => {
|
||||
const track = {
|
||||
enabled: true,
|
||||
readyState: 'live',
|
||||
stop() {
|
||||
this.readyState = 'ended';
|
||||
},
|
||||
};
|
||||
tracks.push(track);
|
||||
return { getTracks: () => [track], getAudioTracks: () => [track] };
|
||||
};
|
||||
const c = new GeminiSessionController({
|
||||
emit: (event) => events.push(event),
|
||||
runAction: async (name, args, options) => {
|
||||
calls.push({ name, args, options });
|
||||
return { ok: true, action: name };
|
||||
},
|
||||
ui: { root: { remove() {} } },
|
||||
radioLayer: radio,
|
||||
now: () => clock,
|
||||
setTimer: (fn, ms) => {
|
||||
const id = ++nextTimer;
|
||||
timers.set(id, { fn, at: clock + ms });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => timers.delete(id),
|
||||
WebSocketClass: Socket,
|
||||
fetchImpl: async (_url, init) => {
|
||||
requests.push(init);
|
||||
return Response.json(
|
||||
JSON.parse(init.body).inputMode === 'push-to-talk'
|
||||
? manualCredential
|
||||
: credential,
|
||||
);
|
||||
},
|
||||
getUserMedia: async () => makeStream(),
|
||||
createInput: ({ readStream, operations }) => ({
|
||||
...operations,
|
||||
setMicrophoneEnabled(enabled) {
|
||||
for (const track of readStream()?.getAudioTracks() || [])
|
||||
track.enabled = enabled;
|
||||
},
|
||||
updateVoiceButtonLabel() {},
|
||||
setVoiceSpeaker() {},
|
||||
startVoiceVisualizer() {},
|
||||
startAssistantVoiceVisualizer() {},
|
||||
cancelPushToTalkHold() {},
|
||||
stopVoiceVisualizer() {},
|
||||
detachBindings() {},
|
||||
bindPushToTalkShortcut() {},
|
||||
resetSession() {
|
||||
this.pushToTalkMode = false;
|
||||
this.pushToTalkKeyHeld = false;
|
||||
this.spaceKeyHeld = false;
|
||||
},
|
||||
}),
|
||||
createNarration: (options) =>
|
||||
createNarrationScheduler({
|
||||
...options,
|
||||
now: () => clock,
|
||||
setTimer: (fn, ms) => {
|
||||
const id = ++nextTimer;
|
||||
timers.set(id, { fn, at: clock + ms });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => timers.delete(id),
|
||||
}),
|
||||
createProgressEarcon: ({ isMicrophoneMuted }) => ({
|
||||
available: isMicrophoneMuted,
|
||||
start() {
|
||||
earcon.playing = true;
|
||||
earcon.starts++;
|
||||
},
|
||||
stop() {
|
||||
earcon.playing = false;
|
||||
},
|
||||
dispose() {
|
||||
this.stop();
|
||||
earcon.disposals++;
|
||||
},
|
||||
}),
|
||||
createAudio: (callbacks) => {
|
||||
const audio = {
|
||||
callbacks,
|
||||
pending: false,
|
||||
closed: false,
|
||||
played: [],
|
||||
async initialize() {},
|
||||
connectMicrophone(stream, enabled, metadata) {
|
||||
media.push({ stream, enabled });
|
||||
this.connected = true;
|
||||
this.setMicrophoneEnabled(enabled, metadata);
|
||||
},
|
||||
setMicrophoneEnabled(value, metadata) {
|
||||
if (!this.connected) return;
|
||||
if (value && !this.enabled) {
|
||||
this.metadata = metadata;
|
||||
callbacks.onAudioStart(metadata);
|
||||
}
|
||||
if (!value && this.enabled) callbacks.onAudioEnd(this.metadata);
|
||||
this.enabled = value;
|
||||
},
|
||||
play(data, mime) {
|
||||
this.pending = true;
|
||||
this.played.push({ data, mime });
|
||||
},
|
||||
drain() {
|
||||
this.pending = false;
|
||||
callbacks.onDrain();
|
||||
},
|
||||
clearPlayback() {
|
||||
this.pending = false;
|
||||
this.cleared = true;
|
||||
},
|
||||
diagnostics() {
|
||||
return { outputChunks: this.played.length };
|
||||
},
|
||||
close() {
|
||||
this.closed = true;
|
||||
},
|
||||
};
|
||||
captures.push(audio);
|
||||
return audio;
|
||||
},
|
||||
...controllerOverrides,
|
||||
});
|
||||
return {
|
||||
c,
|
||||
sockets,
|
||||
tracks,
|
||||
makeStream,
|
||||
events,
|
||||
media,
|
||||
captures,
|
||||
calls,
|
||||
requests,
|
||||
state,
|
||||
advance,
|
||||
timers,
|
||||
earcon,
|
||||
radioControl: (value) => radioControl?.(value),
|
||||
};
|
||||
}
|
||||
|
||||
export { harness, tick, deferred };
|
||||
@@ -88,8 +88,19 @@ test('root config retains existing named exports and standalone provider order',
|
||||
for (const [name, value] of Object.entries(providers))
|
||||
assert.equal(compatibility[name], value, name);
|
||||
const config = standaloneConfig({ mode: 'test' });
|
||||
const standaloneProviders = config.plugins
|
||||
.slice(3, -5)
|
||||
.map((plugin) => plugin.name);
|
||||
assert.equal(
|
||||
standaloneProviders.filter((name) => name === 'gemini-live-proxy').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
standaloneProviders.indexOf('gemini-live-proxy'),
|
||||
standaloneProviders.indexOf('openai-realtime-proxy') + 1,
|
||||
);
|
||||
assert.deepEqual(
|
||||
config.plugins.slice(3, -5).map((plugin) => plugin.name),
|
||||
standaloneProviders.filter((name) => name !== 'gemini-live-proxy'),
|
||||
providers.localProviderPlugins().map((plugin) => plugin.name),
|
||||
);
|
||||
assert.equal(config.plugins.at(-6).name, 'gev-key-setup');
|
||||
|
||||
@@ -140,3 +140,95 @@ html:not([lang='en'])
|
||||
white-space: normal;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
#gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: 0px;
|
||||
}
|
||||
|
||||
/* Provider choice belongs to the existing voice control, not a separate panel. */
|
||||
.gev-voice-provider-field {
|
||||
position: absolute;
|
||||
bottom: calc(100% + var(--gev-provider-lift, 0px));
|
||||
z-index: 9;
|
||||
left: 0;
|
||||
right: 0;
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
padding: 8px 6px;
|
||||
border: 1px solid var(--glass-border);
|
||||
border-radius: 8px;
|
||||
background: var(--bg-panel, #09121b);
|
||||
color: var(--text-secondary);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.5rem;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
.gev-voice-provider-field[hidden] {
|
||||
display: none;
|
||||
}
|
||||
.gev-voice-provider-field select {
|
||||
min-width: 0;
|
||||
width: 100%;
|
||||
min-height: 30px;
|
||||
color: var(--text-primary);
|
||||
background: #09121b;
|
||||
border: 1px solid var(--glass-border);
|
||||
border-radius: 4px;
|
||||
font: inherit;
|
||||
font-size: 0.65rem;
|
||||
}
|
||||
#gev-voice-control:hover .gev-voice-provider-field,
|
||||
#gev-voice-control:focus-within .gev-voice-provider-field {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
#command-dock
|
||||
#gev-voice-control[data-provider-choice='true']
|
||||
.gev-voice-help-tray,
|
||||
#command-dock #gev-voice-control[data-provider-choice='true'] .gev-voice-card,
|
||||
#command-dock
|
||||
#gev-voice-control[data-provider-choice='true']
|
||||
.gev-voice-error-tray {
|
||||
bottom: calc(100% + 4.8rem + var(--gev-provider-lift));
|
||||
}
|
||||
#command-dock.dock-has-pinned-tray
|
||||
#gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: calc(1.7rem + var(--dock-pinned-stack-height));
|
||||
}
|
||||
/* Pinned trays separate the chooser from its hover owner. Keep it reachable
|
||||
across that gap without requiring keyboard focus to hold it open. */
|
||||
#command-dock.dock-has-pinned-tray .gev-voice-provider-field {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
@media (max-width: 900px) {
|
||||
/* The attribution clearance also creates a gap for mouse users in a
|
||||
narrow window, so the raised chooser must not depend on root hover. */
|
||||
.gev-voice-provider-field {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
#command-dock #gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: 3rem;
|
||||
}
|
||||
#command-dock.dock-has-pinned-tray
|
||||
#gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: calc(3.5rem + var(--dock-pinned-stack-height));
|
||||
}
|
||||
}
|
||||
@media (max-width: 720px) {
|
||||
#command-dock #gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: calc(2vh + 2.5rem);
|
||||
}
|
||||
#command-dock.dock-has-pinned-tray
|
||||
#gev-voice-control[data-provider-choice='true'] {
|
||||
--gev-provider-lift: calc(2vh + 3rem + var(--dock-pinned-stack-height));
|
||||
}
|
||||
}
|
||||
@media (hover: none) {
|
||||
.gev-voice-provider-field {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,16 @@
|
||||
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
|
||||
import { createRealtimeSession } from './realtimeSession.js';
|
||||
import { createGeminiSession } from './geminiSession.js';
|
||||
import { createProviderCommands } from './providerCommands.js';
|
||||
|
||||
/** Default composition; callers may supply another session adapter factory. */
|
||||
export function createVoiceCommands(options) {
|
||||
export function createVoiceCommands(options = {}) {
|
||||
// Generic hosts retain the existing default; standalone opts in explicitly.
|
||||
if (!options.createSession && options.supportedProviders !== undefined)
|
||||
return createProviderCommands(options, {
|
||||
bind: bindVoiceCommands,
|
||||
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
|
||||
});
|
||||
return bindVoiceCommands({
|
||||
createSession: createRealtimeSession,
|
||||
...options,
|
||||
|
||||
@@ -39,6 +39,10 @@ export function createVoiceControl({ reset = false } = {}) {
|
||||
root.dataset.status = 'idle';
|
||||
root.dataset.speaker = 'idle';
|
||||
root.innerHTML = `
|
||||
<label id="gev-voice-provider-field" class="gev-voice-provider-field" hidden>
|
||||
<span>${t('Voice provider')}</span>
|
||||
<select id="gev-voice-provider" aria-label="${t('Voice provider')}" title="${t('Changing provider reconnects voice if it is on.')}"></select>
|
||||
</label>
|
||||
<div class="gev-voice-heading">
|
||||
<div class="gev-voice-kicker">${t('AI AGENT')}</div>
|
||||
<div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
|
||||
@@ -110,6 +114,8 @@ export function createVoiceControl({ reset = false } = {}) {
|
||||
}
|
||||
return {
|
||||
root,
|
||||
providerField: root.querySelector('#gev-voice-provider-field'),
|
||||
providerSelect: root.querySelector('#gev-voice-provider'),
|
||||
button: root.querySelector('#gev-voice-button'),
|
||||
buttonLabel: root.querySelector('.gev-mic-label'),
|
||||
status: root.querySelector('#gev-voice-status'),
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
deferred,
|
||||
tick,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
|
||||
function unreadResponse(bytes = null) {
|
||||
const state = { reads: 0, cancelled: 0 };
|
||||
const response = new Response(
|
||||
new ReadableStream(
|
||||
{
|
||||
pull(controller) {
|
||||
state.reads++;
|
||||
if (bytes) controller.enqueue(bytes);
|
||||
},
|
||||
cancel() {
|
||||
state.cancelled++;
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
),
|
||||
);
|
||||
return { response, state };
|
||||
}
|
||||
|
||||
test('already aborted token request does not acquire or read a response', async (t) => {
|
||||
let fetches = 0;
|
||||
const h = harness({
|
||||
fetchImpl: () => {
|
||||
fetches++;
|
||||
throw Error('must not fetch');
|
||||
},
|
||||
});
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' });
|
||||
assert.equal(fetches, 0);
|
||||
});
|
||||
|
||||
for (const phase of [
|
||||
'before response',
|
||||
'between resolution and continuation',
|
||||
]) {
|
||||
test(`aborted token acquisition closes the unread body ${phase}`, async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness({ fetchImpl: () => gate.promise });
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
const { response, state } = unreadResponse();
|
||||
const result = assert.rejects(h.c.requestToken(abort.signal), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
if (phase === 'before response') abort.abort();
|
||||
gate.resolve(response);
|
||||
if (phase !== 'before response') queueMicrotask(() => abort.abort());
|
||||
await result;
|
||||
await tick();
|
||||
assert.deepEqual(state, { reads: 0, cancelled: 1 });
|
||||
});
|
||||
}
|
||||
|
||||
test('token body cap cancels a native response stream', async (t) => {
|
||||
const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1));
|
||||
const h = harness({ fetchImpl: async () => response });
|
||||
t.after(() => h.c.stop());
|
||||
await assert.rejects(
|
||||
h.c.requestToken(new AbortController().signal),
|
||||
/too large|exceed|limit/i,
|
||||
);
|
||||
assert.equal(state.cancelled, 1);
|
||||
assert.equal(state.reads, 1);
|
||||
});
|
||||
|
||||
test('token body read ends promptly on caller abort and releases the reader', async (t) => {
|
||||
const { response, state } = unreadResponse();
|
||||
const h = harness({ fetchImpl: async () => response });
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
const result = assert.rejects(h.c.requestToken(abort.signal), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
await tick();
|
||||
assert.equal(state.reads, 1);
|
||||
abort.abort();
|
||||
await result;
|
||||
assert.equal(state.cancelled, 1);
|
||||
assert.equal(response.body.locked, false);
|
||||
});
|
||||
|
||||
test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness({ getUserMedia: () => gate.promise });
|
||||
t.after(() => h.c.stop());
|
||||
const stream = h.makeStream();
|
||||
const start = h.c.start();
|
||||
gate.resolve(stream);
|
||||
queueMicrotask(() => h.c.stop());
|
||||
await start;
|
||||
await tick();
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.requests.length, 0);
|
||||
assert.equal(h.sockets.length, 0);
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
h.c.getUserMedia = async () => h.makeStream();
|
||||
await h.c.start();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.tracks[1].readyState, 'live');
|
||||
assert.equal(h.captures[1].closed, false);
|
||||
});
|
||||
|
||||
test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness();
|
||||
t.after(() => h.c.stop());
|
||||
const createAudio = h.c.createAudio;
|
||||
let first = true;
|
||||
h.c.createAudio = (options) => {
|
||||
const audio = createAudio(options);
|
||||
if (first) {
|
||||
first = false;
|
||||
audio.initialize = () => gate.promise;
|
||||
audio.connectMicrophone = () =>
|
||||
assert.fail('retired initialization must not connect');
|
||||
}
|
||||
return audio;
|
||||
};
|
||||
const oldStart = h.c.start();
|
||||
await tick();
|
||||
h.c.stop();
|
||||
await h.c.start();
|
||||
gate.resolve();
|
||||
await oldStart;
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
assert.equal(h.captures[1].closed, false);
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.tracks[1].readyState, 'live');
|
||||
});
|
||||
@@ -0,0 +1,380 @@
|
||||
import { msg } from '../i18n/marker.js';
|
||||
const MAX_AUDIO_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_QUEUED_SECONDS = 60;
|
||||
const MAX_QUEUED_SOURCES = 256;
|
||||
const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz.
|
||||
const PTT_ENERGY_FLOOR = 0.002;
|
||||
const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4;
|
||||
|
||||
/** Admit manual input only after sustained energy; not an utterance detector. */
|
||||
function pttCapture(captureId) {
|
||||
return {
|
||||
captureId,
|
||||
admitted: false,
|
||||
buffers: [],
|
||||
bytes: 0,
|
||||
samples: 0,
|
||||
energy: 0,
|
||||
aboveFloor: 0,
|
||||
voicedWindows: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function hasPttEnergy(state, pcm) {
|
||||
for (let index = 0; index + 1 < pcm.byteLength; index += 2) {
|
||||
const sample = pcm.getInt16(index, true) / 32768;
|
||||
state.energy += sample * sample;
|
||||
if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++;
|
||||
if (++state.samples !== PTT_WINDOW_SAMPLES) continue;
|
||||
// Requiring occupancy as well as RMS rejects isolated high-peak key clicks.
|
||||
const voiced =
|
||||
state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 &&
|
||||
state.aboveFloor >= PTT_WINDOW_SAMPLES / 8;
|
||||
state.voicedWindows = voiced ? state.voicedWindows + 1 : 0;
|
||||
state.samples = state.energy = state.aboveFloor = 0;
|
||||
if (state.voicedWindows >= 2) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function pcmBytesToBase64(buffer) {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = '';
|
||||
for (let offset = 0; offset < bytes.length; offset += 8192) {
|
||||
binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') {
|
||||
if (
|
||||
typeof data !== 'string' ||
|
||||
data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3)
|
||||
)
|
||||
throw new Error(msg('Gemini returned an oversized audio chunk'));
|
||||
const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType);
|
||||
if (!match)
|
||||
throw new Error(msg('Gemini returned an unsupported audio format'));
|
||||
const sampleRate = Number(match[1] || 24000);
|
||||
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000)
|
||||
throw new Error(msg('Gemini returned an invalid audio sample rate'));
|
||||
const binary = atob(data);
|
||||
if (binary.length > MAX_AUDIO_BYTES)
|
||||
throw new Error(msg('Gemini returned an oversized audio chunk'));
|
||||
if (!binary.length || binary.length % 2)
|
||||
throw new Error(msg('Gemini returned incomplete PCM audio'));
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let index = 0; index < binary.length; index++)
|
||||
bytes[index] = binary.charCodeAt(index);
|
||||
const view = new DataView(bytes.buffer);
|
||||
const samples = new Float32Array(bytes.length / 2);
|
||||
for (let index = 0; index < samples.length; index++)
|
||||
samples[index] = view.getInt16(index * 2, true) / 32768;
|
||||
return { samples, sampleRate };
|
||||
}
|
||||
|
||||
/** Own capture and playback independently from the Live protocol. */
|
||||
export function createGeminiAudio({
|
||||
pushToTalk = false,
|
||||
onAudioStart,
|
||||
onAudio,
|
||||
onAudioEnd,
|
||||
onAudioDiscard,
|
||||
onDrain,
|
||||
onError,
|
||||
AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext,
|
||||
AudioWorkletNodeClass = globalThis.AudioWorkletNode,
|
||||
} = {}) {
|
||||
if (!AudioContextClass || !AudioWorkletNodeClass)
|
||||
throw new Error(
|
||||
msg('This browser does not support Gemini microphone audio'),
|
||||
);
|
||||
let captureContext;
|
||||
let playbackContext;
|
||||
let output;
|
||||
let meter;
|
||||
let resumed;
|
||||
const closeContext = (context) => {
|
||||
try {
|
||||
return Promise.resolve(context?.close());
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Capture at the device rate; the worklet explicitly resamples to 16 kHz.
|
||||
captureContext = new AudioContextClass();
|
||||
playbackContext = new AudioContextClass({ sampleRate: 24000 });
|
||||
// Resume both synchronously within the mic gesture, before any awaits.
|
||||
const resume = (context) => {
|
||||
try {
|
||||
return Promise.resolve(context.resume());
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
};
|
||||
resumed = Promise.all([resume(captureContext), resume(playbackContext)]);
|
||||
// Permission may remain pending before initialize observes this failure.
|
||||
void resumed.catch(() => {});
|
||||
output = playbackContext.createGain();
|
||||
meter = playbackContext.createMediaStreamDestination();
|
||||
output.connect(playbackContext.destination);
|
||||
output.connect(meter);
|
||||
} catch (error) {
|
||||
output?.disconnect();
|
||||
meter?.disconnect();
|
||||
meter?.stream.getTracks?.().forEach((track) => track.stop());
|
||||
void Promise.allSettled([
|
||||
closeContext(captureContext),
|
||||
closeContext(playbackContext),
|
||||
]);
|
||||
throw error;
|
||||
}
|
||||
let closed = false;
|
||||
let closeResult;
|
||||
let capture = null;
|
||||
let microphone = null;
|
||||
let mute = null;
|
||||
let nextTime = 0;
|
||||
let inputChunks = 0;
|
||||
let inputPeak = 0;
|
||||
let nonSilentInputChunks = 0;
|
||||
let outputChunks = 0;
|
||||
let outputSamples = 0;
|
||||
let completedOutputSamples = 0;
|
||||
let discardedOutputSamples = 0;
|
||||
let pendingPtt = null;
|
||||
let latestPttCaptureId = 0;
|
||||
let suppressedPttCaptures = 0;
|
||||
let admittedPttCaptures = 0;
|
||||
const sources = new Map();
|
||||
|
||||
const emitAudio = (buffer, metadata) =>
|
||||
onAudio?.(
|
||||
{ data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
|
||||
function clearPlayback() {
|
||||
const oldSources = [...sources];
|
||||
sources.clear();
|
||||
nextTime = playbackContext.currentTime;
|
||||
for (const [source, sampleCount] of oldSources) {
|
||||
discardedOutputSamples += sampleCount;
|
||||
source.onended = null;
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
/* Already ended. */
|
||||
}
|
||||
source.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
outputStream: meter.stream,
|
||||
get pending() {
|
||||
return sources.size > 0;
|
||||
},
|
||||
async initialize(signal) {
|
||||
try {
|
||||
await Promise.all([
|
||||
resumed,
|
||||
captureContext.audioWorklet.addModule(
|
||||
new URL('./geminiCaptureWorklet.js', import.meta.url),
|
||||
),
|
||||
]);
|
||||
signal?.throwIfAborted();
|
||||
if (closed) throw new DOMException('Voice stopped', 'AbortError');
|
||||
} catch (error) {
|
||||
await this.close();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
connectMicrophone(stream, enabled, metadata) {
|
||||
if (closed) throw new DOMException('Voice stopped', 'AbortError');
|
||||
capture = new AudioWorkletNodeClass(
|
||||
captureContext,
|
||||
'gev-gemini-capture',
|
||||
{
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [1],
|
||||
},
|
||||
);
|
||||
capture.port.onmessage = ({ data }) => {
|
||||
if (closed) return;
|
||||
// Capture ownership is transport metadata, never part of Live audio.
|
||||
if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0)
|
||||
return;
|
||||
const metadata = { captureId: data.captureId };
|
||||
try {
|
||||
if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) {
|
||||
inputChunks++;
|
||||
const pcm = new DataView(data.buffer);
|
||||
let peak = 0;
|
||||
for (let index = 0; index + 1 < pcm.byteLength; index += 2)
|
||||
peak = Math.max(
|
||||
peak,
|
||||
Math.abs(pcm.getInt16(index, true)) / 32768,
|
||||
);
|
||||
inputPeak = Math.max(inputPeak, peak);
|
||||
if (peak >= 0.01) nonSilentInputChunks++;
|
||||
if (!pushToTalk) emitAudio(data.buffer, metadata);
|
||||
else {
|
||||
const state = pendingPtt;
|
||||
if (state?.captureId !== data.captureId) return;
|
||||
if (state.admitted) emitAudio(data.buffer, metadata);
|
||||
else {
|
||||
state.buffers.push(data.buffer);
|
||||
state.bytes += data.buffer.byteLength;
|
||||
if (hasPttEnergy(state, pcm)) {
|
||||
state.admitted = true;
|
||||
admittedPttCaptures++;
|
||||
// Context and activityStart are gated together with PCM.
|
||||
onAudioStart?.(metadata);
|
||||
for (const buffer of state.buffers) {
|
||||
if (closed || pendingPtt !== state) break;
|
||||
emitAudio(buffer, metadata);
|
||||
}
|
||||
state.buffers = [];
|
||||
state.bytes = 0;
|
||||
} else {
|
||||
while (state.bytes > PTT_PRE_ROLL_BYTES) {
|
||||
state.bytes -= state.buffers.shift().byteLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (data?.type === 'start') {
|
||||
if (!pushToTalk) onAudioStart?.(metadata);
|
||||
else if (data.captureId > latestPttCaptureId) {
|
||||
latestPttCaptureId = data.captureId;
|
||||
pendingPtt = pttCapture(data.captureId);
|
||||
}
|
||||
} else if (data?.type === 'end') {
|
||||
if (!pushToTalk) onAudioEnd?.(metadata);
|
||||
else if (pendingPtt?.captureId === data.captureId) {
|
||||
const admitted = pendingPtt.admitted;
|
||||
pendingPtt = null;
|
||||
if (admitted) onAudioEnd?.(metadata);
|
||||
else {
|
||||
suppressedPttCaptures++;
|
||||
onAudioDiscard?.(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
onError?.(error);
|
||||
}
|
||||
};
|
||||
capture.onprocessorerror = () => {
|
||||
if (!closed)
|
||||
onError?.(new Error(msg('Microphone audio processing failed')));
|
||||
};
|
||||
microphone = captureContext.createMediaStreamSource(stream);
|
||||
mute = captureContext.createGain();
|
||||
mute.gain.value = 0;
|
||||
capture.port.postMessage({
|
||||
enabled: Boolean(enabled),
|
||||
captureId: metadata?.captureId,
|
||||
});
|
||||
microphone.connect(capture);
|
||||
capture.connect(mute);
|
||||
mute.connect(captureContext.destination);
|
||||
},
|
||||
setMicrophoneEnabled(enabled, metadata) {
|
||||
if (closed) return;
|
||||
capture?.port.postMessage({
|
||||
enabled: Boolean(enabled),
|
||||
captureId: metadata?.captureId,
|
||||
});
|
||||
},
|
||||
play(data, mimeType) {
|
||||
if (closed) return;
|
||||
// Tiny chunks can exhaust source nodes long before the duration limit.
|
||||
if (sources.size >= MAX_QUEUED_SOURCES)
|
||||
throw new Error(
|
||||
msg('Gemini audio playback fell behind. Reconnect to continue.'),
|
||||
);
|
||||
const { samples, sampleRate } = decodeGeminiPcm(data, mimeType);
|
||||
const start = Math.max(playbackContext.currentTime + 0.02, nextTime);
|
||||
if (
|
||||
start + samples.length / sampleRate - playbackContext.currentTime >
|
||||
MAX_QUEUED_SECONDS
|
||||
)
|
||||
throw new Error(
|
||||
msg('Gemini audio playback fell behind. Reconnect to continue.'),
|
||||
);
|
||||
const buffer = playbackContext.createBuffer(
|
||||
1,
|
||||
samples.length,
|
||||
sampleRate,
|
||||
);
|
||||
buffer.copyToChannel(samples, 0);
|
||||
const source = playbackContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(output);
|
||||
sources.set(source, samples.length);
|
||||
source.onended = () => {
|
||||
if (!sources.delete(source)) return;
|
||||
completedOutputSamples += samples.length;
|
||||
source.disconnect();
|
||||
if (!closed && !sources.size) onDrain?.();
|
||||
};
|
||||
nextTime = start + buffer.duration;
|
||||
outputChunks++;
|
||||
outputSamples += samples.length;
|
||||
source.start(start);
|
||||
},
|
||||
clearPlayback,
|
||||
diagnostics() {
|
||||
return {
|
||||
inputChunks,
|
||||
inputPeak,
|
||||
nonSilentInputChunks,
|
||||
outputChunks,
|
||||
outputSamples,
|
||||
completedOutputSamples,
|
||||
discardedOutputSamples,
|
||||
queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime),
|
||||
contextState:
|
||||
captureContext.state === playbackContext.state
|
||||
? captureContext.state
|
||||
: 'suspended',
|
||||
captureContextState: captureContext.state,
|
||||
playbackContextState: playbackContext.state,
|
||||
captureSampleRate: captureContext.sampleRate,
|
||||
playbackSampleRate: playbackContext.sampleRate,
|
||||
inputSampleRate: 16000,
|
||||
suppressedPttCaptures,
|
||||
admittedPttCaptures,
|
||||
pttBufferedBytes: pendingPtt?.bytes || 0,
|
||||
};
|
||||
},
|
||||
close() {
|
||||
if (closed) return closeResult;
|
||||
closed = true;
|
||||
pendingPtt = null;
|
||||
clearPlayback();
|
||||
if (capture) {
|
||||
capture.port.onmessage = null;
|
||||
capture.onprocessorerror = null;
|
||||
capture.port.close();
|
||||
capture.disconnect();
|
||||
}
|
||||
microphone?.disconnect();
|
||||
mute?.disconnect();
|
||||
output.disconnect();
|
||||
meter.disconnect();
|
||||
meter.stream.getTracks?.().forEach((track) => track.stop());
|
||||
closeResult = Promise.allSettled([
|
||||
closeContext(captureContext),
|
||||
closeContext(playbackContext),
|
||||
]).then((results) =>
|
||||
results.every((result) => result.status === 'fulfilled'),
|
||||
);
|
||||
return closeResult;
|
||||
},
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,791 @@
|
||||
// Native automatic activity, reply ownership and expiry lifecycle regressions.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
const vad = (ws, type) => ws.message({ voiceActivity: { type } });
|
||||
const onset = (ws) => {
|
||||
vad(ws, 'ACTIVITY_START');
|
||||
vad(ws, 'ACTIVITY_END');
|
||||
};
|
||||
const split = (ws) => {
|
||||
onset(ws);
|
||||
onset(ws);
|
||||
};
|
||||
const audio = {
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const done = (ws) => ws.message({ serverContent: { turnComplete: true } });
|
||||
const tools = (ws, id) =>
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'out', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
function close(h) {
|
||||
h.c.stop();
|
||||
assert.equal(h.timers.size, 0, 'owned timers released');
|
||||
}
|
||||
test('N1 VAD-only split admits the first real answer and completes after playback', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } });
|
||||
done(ws);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
split(h.sockets[0]);
|
||||
h.advance(5001);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
assert.equal(h.c.retiredBoundaryWait, null);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N3 server interruption before output still fences until ordered turnComplete', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'retired-before-boundary');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
ws.message({
|
||||
serverContent: { ...audio.serverContent, turnComplete: true },
|
||||
});
|
||||
assert.equal(
|
||||
h.captures[0].played.length,
|
||||
0,
|
||||
'coalesced retired content is discarded',
|
||||
);
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N4 generationComplete is server evidence and next onset retires it', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
assert.ok(h.c.retiredBoundaryWait);
|
||||
ws.message(audio);
|
||||
tools(ws, 'old-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
done(ws);
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
assert.ok(h.c.goAway);
|
||||
assert.equal(ws.closed, undefined);
|
||||
h.advance(5001);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'pending actual PCM drains before expiry',
|
||||
);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
assert.equal(ws.closed, true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N6 first server tool after VAD-only split executes exactly once', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
tools(ws, 'new-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'new-generation').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(responses(ws).at(-1).response.ok, true);
|
||||
tools(ws, 'new-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'new-generation').length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-old');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
assert.equal(h.calls.length, 0);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
done(ws);
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 2);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C2 first server tool in flight is aborted and never answered after new onset', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'old-running');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
onset(ws);
|
||||
assert.equal(entered[0].signal.aborted, true);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C5 typed request before first server output still retires on next onset', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
assert.equal(h.c.sendTextCommand('Zoom in'), true);
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-typed');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
const oldOwner = h.c.automaticActivityOwner;
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
const successor = h.c.automaticActivityOwner;
|
||||
assert.notEqual(successor, oldOwner);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticActivityOwner, successor);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
const owner = h.c.automaticActivityOwner;
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, owner);
|
||||
done(ws);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), false, 'no invented successor wait');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N9 coalesced old completion and new VAD preserves the new owner', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
vad(ws, 'ACTIVITY_END');
|
||||
ws.message({
|
||||
voiceActivity: { type: 'ACTIVITY_START' },
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
assert.ok(h.c.automaticActivityOwner);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N10 stop, restart and destroy release automatic owner identities and timers', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
onset(h.sockets[0]);
|
||||
const first = h.c.automaticActivityOwner;
|
||||
h.c.stop();
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
assert.equal(h.c.automaticActivitySequence, 0);
|
||||
await h.c.start();
|
||||
onset(h.sockets[1]);
|
||||
assert.notEqual(h.c.automaticActivityOwner, first);
|
||||
assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch);
|
||||
h.c.stop({ removeUi: true });
|
||||
assert.equal(h.c.disposed, true);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
assert.equal(h.timers.size, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom in'), true);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out'), true);
|
||||
assert.equal(h.c.deferredInput.kind, 'user-text');
|
||||
const before = ws.sent.filter((x) => x.realtimeInput?.audio).length;
|
||||
const metadata = { captureId: h.c.captureOwner.captureId };
|
||||
h.captures[0].callbacks.onAudio(
|
||||
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before);
|
||||
assert.equal(
|
||||
h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
|
||||
assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, {
|
||||
nonPttOwner: 1,
|
||||
closedActivity: 0,
|
||||
});
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(
|
||||
ws.sent.filter((x) => x.realtimeInput?.audio).length,
|
||||
before,
|
||||
'no wrong-owner PCM replay after boundaries',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N12 goAway before old completion still waits for the newer automatic owner', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
const successor = h.c.automaticActivityOwner;
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticActivityOwner, successor);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'old boundary during goAway cannot end successor wait',
|
||||
);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const interruptedBeforeExpiry of [false, true])
|
||||
test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
if (interruptedBeforeExpiry)
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'no successor finishes at its actual boundary',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'expiring-old-tool');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(entered[0].signal.aborted, true);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'expiring-old-tool').length,
|
||||
0,
|
||||
);
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
const turn = h.c.turn,
|
||||
sent = ws.sent.length;
|
||||
assert.equal(h.c.bargeIn(), false);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out'), false);
|
||||
h.c.interrupt('push-to-talk');
|
||||
assert.equal(h.c.turn, turn);
|
||||
assert.equal(ws.sent.length, sent);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => !t.enabled),
|
||||
true,
|
||||
);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.isActive(), false);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => t.readyState === 'ended'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
|
||||
test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
a = h.captures[0];
|
||||
ws.message(audio);
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
a.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
a.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(
|
||||
h.c.replyPending,
|
||||
true,
|
||||
'old boundary cannot clear submitted successor',
|
||||
);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out second request'), true);
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
const played = a.played.length;
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-ptt-predecessor');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(a.played.length, played);
|
||||
assert.equal(
|
||||
h.calls.length,
|
||||
0,
|
||||
'no old tool under newer local typed input',
|
||||
);
|
||||
done(ws);
|
||||
tools(ws, 'current-typed');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
assert.equal(h.c.currentTypedInput().text, 'Zoom out second request');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const expiryBeforeBoundary of [false, true])
|
||||
test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
|
||||
const typed = h.c.currentTypedInput();
|
||||
assert.ok(typed);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.replyPending, true);
|
||||
if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true, 'submitted text must finish');
|
||||
tools(ws, 'valid-successor');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
assert.equal(h.c.currentTypedInput(), typed);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const repeat of [false, true])
|
||||
test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out A'), true);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out B'), true);
|
||||
assert.equal(h.c.deferredInput.kind, 'user-text');
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
onset(ws);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.deferredInput, null);
|
||||
if (repeat) {
|
||||
ws.message({ goAway: { timeLeft: '20s' } });
|
||||
ws.message({ goAway: { timeLeft: '10s' } });
|
||||
}
|
||||
assert.equal(h.c.goAway.supersededSuccessor, true);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
tools(ws, 'superseded-A');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority');
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'ordered boundary closes superseded forwarded predecessor',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
onset(h.sockets[0]);
|
||||
h.sockets[0].message({ goAway: { timeLeft: '30s' } });
|
||||
h.advance(29999);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
h.advance(1);
|
||||
assert.equal(h.c.isActive(), false);
|
||||
assert.equal(h.timers.size, 0);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => t.readyState === 'ended'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({
|
||||
voiceActivity: { type: 'ACTIVITY_START' },
|
||||
serverContent: { turnComplete: true },
|
||||
});
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
assert.equal(h.c.replyPending, true);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
assert.equal(h.c.replyPending, false);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.retiredBoundaryWait, null);
|
||||
assert.equal(h.timers.size, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
|
||||
test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
a = h.captures[0];
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
a.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
a.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
const event = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(event.status, 'completed');
|
||||
assert.equal(event.spoken, false);
|
||||
assert.equal(a.played.length, 0);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'pending-terminal-tool');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
h.c.finishPlayback();
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up');
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
h.captures[0].drain();
|
||||
const event = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(event.status, 'completed');
|
||||
assert.equal(event.spoken, true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
done(ws);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C7 reentrant silent completion observer keeps new input pending', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
const emit = h.c.emit.bind(h.c);
|
||||
h.c.emit = (event) => {
|
||||
emit(event);
|
||||
if (event.type === 'completion')
|
||||
h.c.sendTextCommand('Zoom out reentrant');
|
||||
};
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
assert.equal(h.c.modelComplete, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
const OUTPUT_RATE = 16000;
|
||||
const FILTER_TAPS = 64;
|
||||
const FILTER_PHASES = 128;
|
||||
|
||||
// A causal, windowed-sinc low-pass filter avoids folding device frequencies
|
||||
// above 8 kHz into speech. Precompute phases; process allocates no filter data.
|
||||
function resamplingFilters(inputRate) {
|
||||
const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate);
|
||||
const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES);
|
||||
for (let phase = 0; phase < FILTER_PHASES; phase++) {
|
||||
let total = 0;
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++) {
|
||||
const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES;
|
||||
const sinc =
|
||||
distance === 0
|
||||
? 2 * cutoff
|
||||
: Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance);
|
||||
const window =
|
||||
0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1));
|
||||
const weight = sinc * window;
|
||||
filters[phase * FILTER_TAPS + tap] = weight;
|
||||
total += weight;
|
||||
}
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++)
|
||||
filters[phase * FILTER_TAPS + tap] /= total;
|
||||
}
|
||||
return filters;
|
||||
}
|
||||
|
||||
/** PCM microphone capture. Only explicitly enabled frames leave this processor. */
|
||||
class GeminiCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.enabled = false;
|
||||
this.captureId = null;
|
||||
this.samples = new Int16Array(1024);
|
||||
this.length = 0;
|
||||
this.inputRate = sampleRate;
|
||||
this.filters =
|
||||
this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate);
|
||||
this.history = new Float32Array(FILTER_TAPS);
|
||||
this.resetResampler();
|
||||
this.port.onmessage = ({ data }) => {
|
||||
if (data?.enabled === true) {
|
||||
if (
|
||||
this.enabled ||
|
||||
!Number.isSafeInteger(data.captureId) ||
|
||||
data.captureId <= 0
|
||||
)
|
||||
return;
|
||||
this.captureId = data.captureId;
|
||||
this.resetResampler();
|
||||
this.enabled = true;
|
||||
this.port.postMessage({ type: 'start', captureId: this.captureId });
|
||||
} else if (
|
||||
data?.enabled === false &&
|
||||
this.enabled &&
|
||||
data.captureId === this.captureId
|
||||
) {
|
||||
this.flush();
|
||||
this.port.postMessage({ type: 'end', captureId: this.captureId });
|
||||
this.enabled = false;
|
||||
this.captureId = null;
|
||||
this.length = 0;
|
||||
// Do not drain filter tails into a later physical microphone capture.
|
||||
this.resetResampler();
|
||||
}
|
||||
};
|
||||
}
|
||||
resetResampler() {
|
||||
this.phase = 0;
|
||||
this.historyIndex = 0;
|
||||
this.history.fill(0);
|
||||
}
|
||||
appendSample(sample) {
|
||||
const value = Math.max(-1, Math.min(1, sample));
|
||||
this.samples[this.length++] = Math.round(
|
||||
value * (value < 0 ? 32768 : 32767),
|
||||
);
|
||||
if (this.length === this.samples.length) this.flush();
|
||||
}
|
||||
flush() {
|
||||
if (!this.length) return;
|
||||
const buffer = new ArrayBuffer(this.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
for (let i = 0; i < this.length; i++)
|
||||
view.setInt16(i * 2, this.samples[i], true);
|
||||
this.port.postMessage(
|
||||
{ type: 'audio', buffer, captureId: this.captureId },
|
||||
[buffer],
|
||||
);
|
||||
this.length = 0;
|
||||
}
|
||||
process(inputs) {
|
||||
if (!this.enabled) return true;
|
||||
const channel = inputs[0]?.[0];
|
||||
if (!channel) return true;
|
||||
for (const sample of channel) {
|
||||
if (!this.filters) {
|
||||
this.appendSample(sample);
|
||||
continue;
|
||||
}
|
||||
this.history[this.historyIndex] = sample;
|
||||
this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS;
|
||||
this.phase += OUTPUT_RATE;
|
||||
while (this.phase >= this.inputRate) {
|
||||
this.phase -= this.inputRate;
|
||||
const phase = Math.min(
|
||||
FILTER_PHASES - 1,
|
||||
Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE),
|
||||
);
|
||||
let value = 0;
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++) {
|
||||
const index =
|
||||
(this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS;
|
||||
value +=
|
||||
this.history[index] * this.filters[phase * FILTER_TAPS + tap];
|
||||
}
|
||||
this.appendSample(value);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('gev-gemini-capture', GeminiCaptureProcessor);
|
||||
@@ -0,0 +1,285 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const OLD_PCM = 'BAAFAA==';
|
||||
const TEXT = 'Zoom in for the owned typed successor';
|
||||
|
||||
function exposeCapture(h) {
|
||||
const createAudio = h.controller.createAudio;
|
||||
h.controller.createAudio = (callbacks) => {
|
||||
const audio = createAudio(callbacks);
|
||||
audio.captureCallbacks = callbacks;
|
||||
return audio;
|
||||
};
|
||||
}
|
||||
|
||||
async function startManual(h) {
|
||||
await h.session.start({ pushToTalk: true });
|
||||
assert.equal(h.session.state, 'listening');
|
||||
assert.equal(h.controller.input.pushToTalkMode, true);
|
||||
assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false);
|
||||
return h.sockets.at(-1);
|
||||
}
|
||||
|
||||
// Explicit fake worklet callbacks retain the native physical-capture path;
|
||||
// this helper does not populate controller queues or ownership fields itself.
|
||||
function speakAndRelease(h, data) {
|
||||
const c = h.controller;
|
||||
const audio = h.audios.at(-1);
|
||||
c.input.pushToTalkKeyHeld = true;
|
||||
c.input.setMicrophoneEnabled(true);
|
||||
const metadata = audio.metadata;
|
||||
assert.ok(Number.isSafeInteger(metadata?.captureId));
|
||||
audio.captureCallbacks.onAudioStart(metadata);
|
||||
audio.captureCallbacks.onAudio(
|
||||
{ data, mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
c.input.pushToTalkKeyHeld = false;
|
||||
c.input.setMicrophoneEnabled(false);
|
||||
audio.captureCallbacks.onAudioEnd(metadata);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function prepareDeferredPtt(h) {
|
||||
exposeCapture(h);
|
||||
const socket = await startManual(h);
|
||||
speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply.
|
||||
speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A.
|
||||
const wireBeforeC = socket.sent.length;
|
||||
const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary.
|
||||
const owner = h.controller.deferredInput;
|
||||
assert.equal(owner?.kind, 'push-to-talk');
|
||||
assert.equal(owner.captureId, metadata.captureId);
|
||||
assert.equal(
|
||||
socket.sent.length,
|
||||
wireBeforeC,
|
||||
'C must still be wholly deferred',
|
||||
);
|
||||
assert.deepEqual(
|
||||
owner.messages.map((message) =>
|
||||
message.realtimeInput?.activityStart
|
||||
? 'start'
|
||||
: message.realtimeInput?.audio
|
||||
? 'audio'
|
||||
: message.realtimeInput?.activityEnd
|
||||
? 'end'
|
||||
: 'other',
|
||||
),
|
||||
['start', 'audio', 'end'],
|
||||
);
|
||||
return { socket, owner, metadata };
|
||||
}
|
||||
|
||||
async function assertHealthySuccessor(h, socket, callId) {
|
||||
const beforeHeight = h.state.height;
|
||||
await h.tool(callId, socket);
|
||||
assertSuccessfulZoom(h, socket, callId, beforeHeight);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
h.complete(socket); // Native post-tool response boundary, with no output PCM.
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.session.state, 'listening');
|
||||
}
|
||||
|
||||
for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) {
|
||||
test(
|
||||
`F04-5 deferred PTT flush loses its owner on ${mode}`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const { socket: oldSocket, owner } = await prepareDeferredPtt(h);
|
||||
const oldAudio = h.audios.at(-1);
|
||||
let observed = 0;
|
||||
let sentAtObserverReturn;
|
||||
let restart;
|
||||
let stateAtObserverReturn;
|
||||
const unsubscribe = h.session.subscribe((event) => {
|
||||
if (
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'user' ||
|
||||
event.text !== '' ||
|
||||
observed
|
||||
)
|
||||
return;
|
||||
observed++;
|
||||
if (mode === 'typed replacement') {
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
} else {
|
||||
h.session.stop();
|
||||
if (mode === 'Stop+restart')
|
||||
restart = h.session.start({ pushToTalk: true });
|
||||
}
|
||||
sentAtObserverReturn = oldSocket.sent.length;
|
||||
stateAtObserverReturn = {
|
||||
epoch: h.controller.epoch,
|
||||
turn: h.controller.turn,
|
||||
replyPending: h.controller.replyPending,
|
||||
forwardedActivityOpen: h.controller.forwardedActivityOpen,
|
||||
};
|
||||
});
|
||||
t.after(unsubscribe);
|
||||
|
||||
h.complete(oldSocket); // A's boundary starts flushing C. C's start emits.
|
||||
assert.equal(
|
||||
observed,
|
||||
1,
|
||||
'must enter the actual flush/onset observer boundary',
|
||||
);
|
||||
assert.ok(
|
||||
stateAtObserverReturn,
|
||||
'observer must complete its replacement action',
|
||||
);
|
||||
assert.deepEqual(
|
||||
oldSocket.sent.slice(sentAtObserverReturn),
|
||||
[],
|
||||
'retired C must send neither queued PCM nor an extra end after the observer returns',
|
||||
);
|
||||
assert.deepEqual(
|
||||
{
|
||||
epoch: h.controller.epoch,
|
||||
turn: h.controller.turn,
|
||||
replyPending: h.controller.replyPending,
|
||||
forwardedActivityOpen: h.controller.forwardedActivityOpen,
|
||||
},
|
||||
stateAtObserverReturn,
|
||||
'old flush must not write successor state',
|
||||
);
|
||||
assert.equal(h.controller.deferredInput, null);
|
||||
assert.equal(
|
||||
h.controller.typedIntent?.text ?? null,
|
||||
mode === 'typed replacement' ? TEXT : null,
|
||||
);
|
||||
|
||||
let successorSocket;
|
||||
if (mode === 'typed replacement') {
|
||||
assert.equal(h.controller.epoch, owner.epoch);
|
||||
assert.notEqual(h.controller.turn, owner.turn);
|
||||
assert.equal(
|
||||
oldSocket.sent.some(
|
||||
(message) => message.realtimeInput?.audio?.data === OLD_PCM,
|
||||
),
|
||||
false,
|
||||
);
|
||||
const typedIndex = oldSocket.sent.findIndex((message) =>
|
||||
message.clientContent?.turns?.some((turn) =>
|
||||
turn.parts?.some((part) => part.text === TEXT),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
typedIndex >= 0,
|
||||
'owned replacement must actually reach transport',
|
||||
);
|
||||
// The replacement legitimately closes the already-forwarded C start once.
|
||||
assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd);
|
||||
h.complete(oldSocket); // B's remaining retired boundary, deliberately separate.
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
successorSocket = oldSocket;
|
||||
} else {
|
||||
assert.equal(oldAudio.closed, true);
|
||||
assert.equal(oldSocket.closed, true);
|
||||
if (!restart) restart = h.session.start({ pushToTalk: true });
|
||||
await restart;
|
||||
successorSocket = h.sockets.at(-1);
|
||||
assert.notEqual(successorSocket, oldSocket);
|
||||
assert.equal(
|
||||
successorSocket.sent.some((message) => message.realtimeInput?.audio),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
successorSocket.sent.some(
|
||||
(message) => message.realtimeInput?.activityEnd,
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
}
|
||||
await assertHealthySuccessor(
|
||||
h,
|
||||
successorSocket,
|
||||
`healthy-after-deferred-${mode}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const oldSocket = await h.start();
|
||||
assert.equal(h.session.sendText('A'), true);
|
||||
assert.equal(h.session.sendText('B'), true);
|
||||
assert.equal(h.session.sendText('C deferred text'), true);
|
||||
assert.equal(h.controller.deferredInput?.kind, 'user-text');
|
||||
let restart;
|
||||
let observed = 0;
|
||||
const unsubscribe = h.session.subscribe((event) => {
|
||||
if (
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'user' ||
|
||||
event.text !== 'C deferred text'
|
||||
)
|
||||
return;
|
||||
observed++;
|
||||
h.session.stop();
|
||||
restart = h.session.start();
|
||||
});
|
||||
t.after(unsubscribe);
|
||||
h.complete(oldSocket);
|
||||
assert.equal(observed, 1);
|
||||
assert.equal(
|
||||
h.controller.replyPending,
|
||||
false,
|
||||
'old onSent wrapper must not resurrect a stopped reply',
|
||||
);
|
||||
await restart;
|
||||
assert.equal(h.controller.replyPending, false);
|
||||
const socket = h.sockets.at(-1);
|
||||
assert.equal(
|
||||
socket.sent.length,
|
||||
1,
|
||||
'replacement connection should contain setup only',
|
||||
);
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text');
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const { socket } = await prepareDeferredPtt(h);
|
||||
const before = socket.sent.length;
|
||||
h.complete(socket);
|
||||
assert.deepEqual(
|
||||
socket.sent
|
||||
.slice(before)
|
||||
.map((message) =>
|
||||
message.realtimeInput?.activityStart
|
||||
? 'start'
|
||||
: message.realtimeInput?.audio?.data === OLD_PCM
|
||||
? 'audio'
|
||||
: message.realtimeInput?.activityEnd
|
||||
? 'end'
|
||||
: 'other',
|
||||
),
|
||||
['start', 'audio', 'end'],
|
||||
);
|
||||
assert.equal(h.controller.deferredInput, null);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.retiredReplyPending, true);
|
||||
h.complete(socket); // B's boundary; C remains the owned submitted input.
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred');
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,287 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
turnOfEventLoop,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const resultFor = (h, id) =>
|
||||
h.events.filter(
|
||||
(event) => event.type === 'action-result' && event.callId === id,
|
||||
);
|
||||
const callFor = (h, id) =>
|
||||
h.events.filter(
|
||||
(event) => event.type === 'action-call' && event.callId === id,
|
||||
);
|
||||
|
||||
test(
|
||||
'native tool reaches actual common session and camera runner without a caption',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws); // Native activity is input framing, not transcript admission.
|
||||
await h.tool('ordinary', ws);
|
||||
assertSuccessfulZoom(h, ws, 'ordinary', 1000);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(h.runnerCalls[0].options.callId, 'ordinary');
|
||||
assert.equal(h.runnerCalls[0].options.signal.aborted, false);
|
||||
assert.equal(h.runnerCalls[0].options.isCurrent(), true);
|
||||
h.complete(ws);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
},
|
||||
);
|
||||
|
||||
for (const cancellation of ['stop', 'native interruption']) {
|
||||
test(
|
||||
`${cancellation} discards a stale completed action result; successor changes camera once`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const oldSocket = await h.start();
|
||||
h.activity(oldSocket);
|
||||
const gate = h.holdResult('old');
|
||||
const oldDispatch = h.tool('old', oldSocket);
|
||||
const committedResult = await gate.ready;
|
||||
assert.equal(committedResult.ok, true);
|
||||
assert.equal(
|
||||
h.state.height,
|
||||
750,
|
||||
'the legitimate first action committed before cancellation',
|
||||
);
|
||||
assert.equal(callFor(h, 'old').length, 1);
|
||||
assert.deepEqual(resultFor(h, 'old'), []);
|
||||
assert.deepEqual(h.outputs(oldSocket), []);
|
||||
|
||||
if (cancellation === 'stop') h.session.stop();
|
||||
else
|
||||
oldSocket.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
assert.equal(h.runnerCalls[0].options.signal.aborted, true);
|
||||
assert.equal(h.runnerCalls[0].options.isCurrent(), false);
|
||||
await oldDispatch; // Cancellation must settle before the held result returns.
|
||||
gate.release();
|
||||
await turnOfEventLoop();
|
||||
assert.equal(
|
||||
h.state.height,
|
||||
750,
|
||||
'settling a cancelled result adds no camera mutation',
|
||||
);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.deepEqual(resultFor(h, 'old'), []);
|
||||
assert.deepEqual(h.outputs(oldSocket), []);
|
||||
if (cancellation === 'native interruption')
|
||||
assert.ok(h.events.some((event) => event.type === 'interruption'));
|
||||
|
||||
const successorSocket =
|
||||
cancellation === 'stop' ? await h.start() : oldSocket;
|
||||
h.activity(successorSocket);
|
||||
await h.tool('successor', successorSocket);
|
||||
assertSuccessfulZoom(h, successorSocket, 'successor', 750);
|
||||
assert.equal(h.state.zooms.length, 2);
|
||||
assert.equal(h.runnerCalls.length, 2);
|
||||
assert.equal(
|
||||
h.outputs(successorSocket).some((output) => output.id === 'old'),
|
||||
false,
|
||||
);
|
||||
h.complete(successorSocket);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const phase of ['common connecting', 'adapter connecting']) {
|
||||
test(
|
||||
`Stop from ${phase} observer prevents acquisition and permits fresh Start`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
let stopped = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
const matchingDetail =
|
||||
phase === 'common connecting'
|
||||
? event.detail === undefined
|
||||
: typeof event.detail === 'string' && event.detail.length > 0;
|
||||
if (
|
||||
!stopped &&
|
||||
event.type === 'state' &&
|
||||
event.state === 'connecting' &&
|
||||
matchingDetail
|
||||
) {
|
||||
stopped = true;
|
||||
h.session.stop();
|
||||
}
|
||||
});
|
||||
await h.session.start();
|
||||
remove();
|
||||
assert.equal(
|
||||
stopped,
|
||||
true,
|
||||
'the chosen production observer boundary was exercised',
|
||||
);
|
||||
assert.deepEqual(h.acquisitions, {
|
||||
microphone: 0,
|
||||
audio: 0,
|
||||
token: 0,
|
||||
socket: 0,
|
||||
});
|
||||
assert.equal(h.session.state, 'idle');
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
await h.tool('fresh-after-start-cancel', ws);
|
||||
assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000);
|
||||
h.complete(ws);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'Stop from common action-call observer prevents runner and leaves no unhandled rejection',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (event.type === 'action-call' && event.callId === 'cancel-at-call')
|
||||
h.session.stop();
|
||||
});
|
||||
h.activity(ws);
|
||||
await h.tool('cancel-at-call', ws);
|
||||
// Do not install a swallowing process rejection listener. node:test records
|
||||
// an unhandled rejection as a failure; these checkpoints let it surface here.
|
||||
await turnOfEventLoop();
|
||||
await turnOfEventLoop();
|
||||
remove();
|
||||
assert.equal(callFor(h, 'cancel-at-call').length, 1);
|
||||
assert.deepEqual(resultFor(h, 'cancel-at-call'), []);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.outputs(ws), []);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
const successor = await h.start();
|
||||
h.activity(successor);
|
||||
await h.tool('after-call-cancel', successor);
|
||||
assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000);
|
||||
h.complete(successor);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'Stop from executing observer cannot recreate narration or dispatch the runner',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
let stopped = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (!stopped && event.type === 'state' && event.state === 'executing') {
|
||||
stopped = true;
|
||||
h.session.stop();
|
||||
}
|
||||
});
|
||||
h.activity(ws);
|
||||
await h.tool('cancel-at-executing', ws);
|
||||
remove();
|
||||
assert.equal(stopped, true);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
assert.deepEqual(callFor(h, 'cancel-at-executing'), []);
|
||||
assert.deepEqual(resultFor(h, 'cancel-at-executing'), []);
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.outputs(ws), []);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
assert.equal(ws.closed, true);
|
||||
assert.ok(h.audios.every((audio) => audio.closed));
|
||||
assert.ok(h.tracks.every((track) => track.readyState === 'ended'));
|
||||
const before = {
|
||||
events: h.events.length,
|
||||
wire: ws.sent.length,
|
||||
earcon: h.state.earconStarts,
|
||||
};
|
||||
h.clock.advance(60_000);
|
||||
assert.deepEqual(
|
||||
{
|
||||
events: h.events.length,
|
||||
wire: ws.sent.length,
|
||||
earcon: h.state.earconStarts,
|
||||
},
|
||||
before,
|
||||
);
|
||||
const successor = await h.start();
|
||||
h.activity(successor);
|
||||
await h.tool('after-executing-cancel', successor);
|
||||
assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000);
|
||||
h.complete(successor);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'native idle observer restart remains connecting and public Stop cancels its pending permission',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
await h.start();
|
||||
let grant;
|
||||
const permission = new Promise((resolve) => {
|
||||
grant = resolve;
|
||||
});
|
||||
h.controller.getUserMedia = () => permission;
|
||||
const states = [];
|
||||
let restarted = false;
|
||||
let startup;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (event.type === 'state') states.push(event.state);
|
||||
if (!restarted && event.type === 'state' && event.state === 'idle') {
|
||||
restarted = true;
|
||||
startup = h.session.start();
|
||||
}
|
||||
});
|
||||
h.session.stop();
|
||||
remove();
|
||||
assert.equal(restarted, true);
|
||||
assert.equal(states.filter((state) => state === 'idle').length, 1);
|
||||
assert.equal(h.session.state, 'connecting');
|
||||
assert.equal(h.controller.status, 'connecting');
|
||||
assert.equal(
|
||||
h.session.isActive(),
|
||||
true,
|
||||
'the common button/facade must offer Stop',
|
||||
);
|
||||
h.session.stop();
|
||||
assert.equal(h.session.state, 'idle');
|
||||
assert.equal(h.controller.status, 'idle');
|
||||
const lateTrack = {
|
||||
readyState: 'live',
|
||||
stopCalls: 0,
|
||||
stop() {
|
||||
this.stopCalls++;
|
||||
this.readyState = 'ended';
|
||||
},
|
||||
};
|
||||
h.tracks.push(lateTrack);
|
||||
grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] });
|
||||
await startup;
|
||||
await turnOfEventLoop();
|
||||
assert.equal(lateTrack.stopCalls, 1);
|
||||
assert.equal(
|
||||
h.requests.length,
|
||||
1,
|
||||
'cancelled restart must not mint another token',
|
||||
);
|
||||
assert.equal(
|
||||
h.sockets.length,
|
||||
1,
|
||||
'cancelled restart must not open another socket',
|
||||
);
|
||||
assert.equal(h.session.state, 'idle');
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,85 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { harness } from '../testSupport/geminiNativeHarness.mjs';
|
||||
|
||||
const caption = (socket, text) =>
|
||||
socket.message({ serverContent: { inputTranscription: { text } } });
|
||||
const userEvents = (events) =>
|
||||
events.filter(
|
||||
(event) => event.type === 'transcript' && event.role === 'user',
|
||||
);
|
||||
|
||||
test('native caption events preserve received advisory segments without granting tool authority', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
const start = h.events.length;
|
||||
for (const text of ['little. ', 'Zoom in a ']) caption(ws, text);
|
||||
const received = userEvents(h.events.slice(start));
|
||||
assert.deepEqual(
|
||||
received.map((event) => event.text),
|
||||
['little. ', 'Zoom in a '],
|
||||
);
|
||||
assert.equal(
|
||||
received.every(
|
||||
(event) =>
|
||||
event.delta === true &&
|
||||
event.final === false &&
|
||||
event.attribution === 'unverified' &&
|
||||
event.responseId === null,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.currentTypedInput(), null);
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'ordinary-native',
|
||||
name: 'get_current_view_state',
|
||||
args: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(
|
||||
userEvents(h.events.slice(start)).some((event) => event.final),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('native old socket captions cannot enter a restarted controller event stream', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const old = h.sockets[0];
|
||||
caption(old, 'old partial');
|
||||
h.c.stop();
|
||||
await h.c.start();
|
||||
const start = h.events.length;
|
||||
const current = h.sockets[1];
|
||||
caption(current, 'Zoom ');
|
||||
caption(old, 'stale old socket');
|
||||
caption(current, 'in a little.');
|
||||
current.message({ serverContent: { turnComplete: true } });
|
||||
assert.deepEqual(
|
||||
userEvents(h.events.slice(start)).map((event) => event.text),
|
||||
['Zoom ', 'in a little.'],
|
||||
);
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,667 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { withToolCatalog } from './gevRealtime.js';
|
||||
|
||||
const audioContent = (data) => ({
|
||||
modelTurn: {
|
||||
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
|
||||
},
|
||||
});
|
||||
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
|
||||
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
const onset = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
const caption = (ws, text) =>
|
||||
ws.message({ serverContent: { inputTranscription: { text } } });
|
||||
|
||||
for (const [id, name, args] of [
|
||||
['H01', 'get_entity_context', {}],
|
||||
['H02', 'control_radio', { action: 'status' }],
|
||||
]) {
|
||||
test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
|
||||
const ready = deferred(),
|
||||
query = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return args.action === 'play'
|
||||
? Promise.resolve({
|
||||
ok: true,
|
||||
action: 'control_radio',
|
||||
radioAction: 'play',
|
||||
radioPlaybackRequested: true,
|
||||
})
|
||||
: query.promise;
|
||||
},
|
||||
});
|
||||
h.c.radio.radioLayer.playForVoice = () => ready.promise;
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Play radio');
|
||||
const ws = h.sockets[0];
|
||||
request(ws, 'owner', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
ws.message({
|
||||
serverContent: { ...audioContent('confirmation'), turnComplete: true },
|
||||
});
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
request(ws, 'query', name, args);
|
||||
await tick();
|
||||
assert.equal(
|
||||
executions.length,
|
||||
2,
|
||||
'current native query may start after provider completion',
|
||||
);
|
||||
const epoch = h.c.radio.radioHandoffEpoch;
|
||||
const pending = h.c.radio.pendingRadioPlaybackResult;
|
||||
const attempt = h.c.radio.radioHandoffAttemptId;
|
||||
const ownerCurrent = executions[0].options.isCurrent();
|
||||
ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(executions[1].options.signal.aborted, true);
|
||||
assert.equal(executions[1].options.isCurrent(), false);
|
||||
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
||||
assert.equal(executions[0].options.signal.aborted, false);
|
||||
assert.equal(executions[0].options.isCurrent(), ownerCurrent);
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
|
||||
assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
assert.equal(h.state.stopped, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['owner'],
|
||||
);
|
||||
assert.equal(responses(ws)[0].response.ok, true);
|
||||
query.resolve({ ok: true });
|
||||
ready.resolve(true);
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'idle');
|
||||
assert.equal(h.state.ducked, false);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['owner'],
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, prior] of [
|
||||
['H03', 'playing'],
|
||||
['H04', 'before-first-output'],
|
||||
]) {
|
||||
test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
if (prior === 'playing')
|
||||
ws.message({ serverContent: audioContent('old') });
|
||||
else {
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
audio.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
audio.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
}
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
const turn = h.c.turn,
|
||||
played = audio.played.length,
|
||||
firstEvent = h.events.length;
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('late-old'),
|
||||
outputTranscription: { text: 'retired output' },
|
||||
inputTranscription: { text: 'new request' },
|
||||
},
|
||||
});
|
||||
request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(audio.played.length, played);
|
||||
assert.equal(audio.pending, false);
|
||||
assert.equal(h.calls.length, 0);
|
||||
assert.equal(
|
||||
h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events
|
||||
.slice(firstEvent)
|
||||
.some((e) => e.role === 'user' && e.text === 'new request'),
|
||||
true,
|
||||
);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.turn, turn);
|
||||
ws.message({ serverContent: audioContent('after-interrupted') });
|
||||
assert.equal(
|
||||
audio.played.length,
|
||||
played,
|
||||
'interrupted alone cannot release the fence',
|
||||
);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('boundary-old'),
|
||||
outputTranscription: { text: 'still retired' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
audio.played.length,
|
||||
played,
|
||||
'coalesced old boundary output is discarded',
|
||||
);
|
||||
assert.equal(
|
||||
h.events.slice(firstEvent).some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
caption(ws, 'unrelated advisory words');
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'retired-call',
|
||||
name: 'fly_to_location',
|
||||
args: { query: 'replayed old target' },
|
||||
},
|
||||
{
|
||||
id: 'new-call',
|
||||
name: 'fly_to_location',
|
||||
args: { query: 'new target' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(
|
||||
h.calls.map((c) => c.args.query),
|
||||
['new target'],
|
||||
);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('new'),
|
||||
outputTranscription: { text: 'new reply' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(audio.played.at(-1).data, 'new');
|
||||
audio.drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion').status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
h.c.sendTextCommand('Show the current view');
|
||||
const response = h.c.currentTypedInput();
|
||||
assert.equal(response.text, 'Show the current view');
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
|
||||
.text,
|
||||
response.text,
|
||||
);
|
||||
request(ws, 'first');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('confirmation'),
|
||||
outputTranscription: { text: 'Here is the current view' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.role === 'assistant').responseId,
|
||||
response.responseId,
|
||||
);
|
||||
audio.drain();
|
||||
const completion = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(completion.responseId, response.responseId);
|
||||
assert.equal(completion.spoken, true);
|
||||
assert.equal(completion.audioDrained, true);
|
||||
assert.equal(h.c.currentTypedInput(), response);
|
||||
assert.equal(response.text, 'Show the current view');
|
||||
request(ws, 'same-turn');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls[1].options.userTurn, undefined);
|
||||
assert.equal(h.c.currentTypedInput(), response);
|
||||
h.c.sendTextCommand('New request');
|
||||
assert.notEqual(h.c.currentTypedInput(), response);
|
||||
assert.equal(h.c.currentTypedInput().text, 'New request');
|
||||
assert.equal(h.calls[0].options.isCurrent(), false);
|
||||
assert.equal(h.calls[1].options.isCurrent(), false);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
|
||||
const gate = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return options.callId === 'old'
|
||||
? gate.promise
|
||||
: Promise.resolve({ ok: true });
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'old');
|
||||
await tick();
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
await oldWork;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(executions[0].options.signal.aborted, true);
|
||||
assert.equal(executions[0].options.isCurrent(), false);
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
request(ws, 'fresh');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(executions.at(-1).options.callId, 'fresh');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
gate.resolve({ ok: true });
|
||||
await tick();
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['fresh'],
|
||||
);
|
||||
ws.message({
|
||||
serverContent: { ...audioContent('fresh reply'), turnComplete: true },
|
||||
});
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion').status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
|
||||
const gate = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return gate.promise;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
h.c.input.bargeIn();
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
audio.callbacks.onAudio(
|
||||
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
|
||||
audio.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
caption(ws, 'Show the current view');
|
||||
assert.equal(h.c.currentTypedInput(), null);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
|
||||
.text,
|
||||
'Show the current view',
|
||||
);
|
||||
request(ws, 'held');
|
||||
await tick();
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(executions.length, 1);
|
||||
assert.equal(executions[0].options.userTurn, undefined);
|
||||
assert.equal(executions[0].options.isCurrent(), true);
|
||||
h.c.sendTextCommand('new intent');
|
||||
await oldWork;
|
||||
assert.equal(executions[0].options.signal.aborted, true);
|
||||
assert.equal(executions[0].options.isCurrent(), false);
|
||||
gate.resolve({ ok: true });
|
||||
await tick();
|
||||
assert.equal(
|
||||
responses(ws).some((r) => r.id === 'held'),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
for (const [id, restart] of [
|
||||
['H11', false],
|
||||
['H12', true],
|
||||
]) {
|
||||
test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
|
||||
const gate = deferred(),
|
||||
mutations = [],
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: async (name, args, options) => {
|
||||
executions.push(options);
|
||||
await gate.promise;
|
||||
if (options.isCurrent()) mutations.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'old');
|
||||
await tick();
|
||||
const pending = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
h.c.stop();
|
||||
const boundary = h.events.length;
|
||||
const successor = restart ? h.c.start() : null;
|
||||
await pending;
|
||||
assert.equal(
|
||||
h.events
|
||||
.slice(boundary)
|
||||
.some((e) => e.type === 'state' && e.state === 'listening'),
|
||||
false,
|
||||
);
|
||||
assert.equal(executions[0].signal.aborted, true);
|
||||
assert.equal(executions[0].isCurrent(), false);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
if (successor) await successor;
|
||||
const laterBoundary = h.events.length;
|
||||
gate.resolve();
|
||||
caption(ws, 'late old speech');
|
||||
await tick();
|
||||
assert.deepEqual(mutations, []);
|
||||
assert.equal(
|
||||
h.events.slice(laterBoundary).some((e) => e.type === 'state'),
|
||||
false,
|
||||
);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, boundary] of [
|
||||
['H14', 'interrupted'],
|
||||
['H15', 'natural'],
|
||||
]) {
|
||||
test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
|
||||
const mutations = [],
|
||||
observed = [];
|
||||
const guarded = withToolCatalog(
|
||||
async (name, args) => {
|
||||
mutations.push({ name, args });
|
||||
return { ok: true };
|
||||
},
|
||||
async () => null,
|
||||
);
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
observed.push(options);
|
||||
return guarded(name, args, options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message({ serverContent: audioContent('old reply') });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...(boundary === 'interrupted' ? { interrupted: true } : {}),
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
caption(ws, 'zoom in');
|
||||
request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
|
||||
await tick();
|
||||
assert.equal(
|
||||
mutations.length,
|
||||
1,
|
||||
'native action executes without a transcript wait',
|
||||
);
|
||||
assert.equal(observed[0].userTurn, undefined);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
responses(ws).some((r) => r.response.voiceIntentUnverified),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, exit] of [
|
||||
['H18', 'stop'],
|
||||
['H19', 'replacement'],
|
||||
]) {
|
||||
test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
|
||||
const mutations = [],
|
||||
observed = [];
|
||||
const guarded = withToolCatalog(
|
||||
async (name) => {
|
||||
mutations.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
async () => null,
|
||||
);
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
observed.push(name);
|
||||
return guarded(name, args, options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
if (exit === 'stop') h.c.stop();
|
||||
else h.c.sendTextCommand('Show the whole globe');
|
||||
caption(ws, 'zoom in');
|
||||
await oldWork;
|
||||
assert.deepEqual(mutations, []);
|
||||
assert.deepEqual(observed, []);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
if (exit === 'replacement') {
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
request(ws, 'fresh-camera', 'zoom_to_globe');
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(mutations, ['zoom_to_globe']);
|
||||
assert.deepEqual(observed, ['zoom_to_globe']);
|
||||
assert.equal(responses(ws).at(-1).response.ok, true);
|
||||
}
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const settlement of ['queued', 'running']) {
|
||||
test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
|
||||
const result = deferred();
|
||||
let executions = 0;
|
||||
const h = harness({
|
||||
runAction: () => {
|
||||
executions++;
|
||||
return result.promise;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
request(ws, 'cancelled-query');
|
||||
if (settlement === 'running') await tick();
|
||||
ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(executions, settlement === 'queued' ? 0 : 1);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.equal(
|
||||
h.events.filter((event) => event.type === 'completion').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(h.c.pendingCompletionStatus, null);
|
||||
result.resolve({ ok: true });
|
||||
await tick();
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.filter((event) => event.type === 'completion').length,
|
||||
1,
|
||||
);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
} finally {
|
||||
result.resolve({ ok: true });
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const order of ['before', 'after']) {
|
||||
test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
ws.bufferedAmount = 1e9;
|
||||
request(ws, 'queued-output');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.controlQueue.length, 1);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
if (order === 'after')
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
|
||||
if (order === 'before')
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.c.controlQueue.length, 0);
|
||||
assert.equal(h.c.controlTimer, null);
|
||||
assert.equal(h.c.pendingCompletionStatus, null);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
ws.bufferedAmount = 0;
|
||||
h.advance(1000);
|
||||
h.captures[0].drain();
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const successor of ['input', 'stop']) {
|
||||
test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
|
||||
let h;
|
||||
let successorTurn;
|
||||
h = harness({
|
||||
emit: (event) => {
|
||||
h.events.push(event);
|
||||
if (event.type !== 'completion') return;
|
||||
h.sockets[0].bufferedAmount = 0;
|
||||
if (successor === 'stop') h.c.stop();
|
||||
else {
|
||||
h.c.sendTextCommand('Show the whole globe');
|
||||
successorTurn = h.c.turn;
|
||||
}
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
ws.bufferedAmount = 1e9;
|
||||
request(ws, 'queued-output');
|
||||
await h.c.toolQueue;
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
const callCount = h.calls.length;
|
||||
ws.message({
|
||||
toolCallCancellation: { ids: ['queued-output'] },
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Retired packet caption' },
|
||||
turnComplete: true,
|
||||
},
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{ id: 'retired-tool', name: 'get_current_view_state', args: {} },
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(h.calls.length, callCount);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.ok(
|
||||
!h.events.some((e) =>
|
||||
JSON.stringify(e).includes('Retired packet caption'),
|
||||
),
|
||||
);
|
||||
if (successor === 'input') {
|
||||
assert.equal(h.c.turn, successorTurn);
|
||||
assert.equal(h.c.modelComplete, false);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
} else assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,734 @@
|
||||
// Native ownership replacements for historical Radio regression cases.
|
||||
// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { DataLayerManager } from '../data/manager.js';
|
||||
|
||||
const viewerFixture = () => ({
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
});
|
||||
const request = (ws, id, name, args = {}) =>
|
||||
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []);
|
||||
const responseFor = (ws, id) =>
|
||||
responses(ws).findLast((response) => response.id === id)?.response;
|
||||
const speechTurn = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
function drainRadioReply(h) {
|
||||
h.sockets.at(-1).message({
|
||||
serverContent: {
|
||||
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
h.captures.at(-1).drain();
|
||||
}
|
||||
|
||||
function radioSiblingFixture() {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const lookupEntered = deferred();
|
||||
const lookupReleased = deferred();
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: { id: 'existing-station', name: 'Existing Station' },
|
||||
audioState: 'stopped',
|
||||
};
|
||||
const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() };
|
||||
const radio = {
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked() {},
|
||||
pause({ origin } = {}) {
|
||||
if (origin === 'voice') trace.pauses++;
|
||||
state.audioState = 'paused';
|
||||
return true;
|
||||
},
|
||||
stopPlayback() {
|
||||
trace.stops++;
|
||||
state.audioState = 'stopped';
|
||||
return true;
|
||||
},
|
||||
selectRequestedStation() {
|
||||
trace.selections++;
|
||||
state.selected = { id: 'late-station', name: 'Late Station' };
|
||||
return state.selected;
|
||||
},
|
||||
subscribePlaybackControls: () => () => {},
|
||||
};
|
||||
manager.register(
|
||||
Object.assign(radio, {
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
}),
|
||||
);
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: manager,
|
||||
placeSearch: {
|
||||
async geocode(_query, { signal }) {
|
||||
trace.lookupSignal = signal;
|
||||
lookupEntered.resolve();
|
||||
// Deliberately ignore abort: production currentness must still prevent
|
||||
// this late service completion from committing a station selection.
|
||||
await lookupReleased.promise;
|
||||
return { place: { lat: 10, lng: 20, label: 'Deferred Place' } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const h = harness({
|
||||
dataManager: manager,
|
||||
radioLayer: radio,
|
||||
runAction(name, args, options) {
|
||||
trace.options.set(options.callId, options);
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
return {
|
||||
h,
|
||||
manager,
|
||||
runner,
|
||||
radio,
|
||||
trace,
|
||||
state,
|
||||
lookupEntered,
|
||||
lookupReleased,
|
||||
};
|
||||
}
|
||||
|
||||
for (const action of ['select', 'play']) {
|
||||
for (const stronger of ['stop', 'pause', 'disable', 'off']) {
|
||||
test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace, state } = f;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
const turn = h.c.turn;
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action,
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'query-sibling', 'control_radio', { action: 'status' });
|
||||
request(
|
||||
ws,
|
||||
'stronger-radio',
|
||||
stronger === 'off' ? 'set_layer_visibility' : 'control_radio',
|
||||
stronger === 'off'
|
||||
? { layerId: 'radio', enabled: false }
|
||||
: { action: stronger },
|
||||
);
|
||||
await tick();
|
||||
assert.equal(
|
||||
responseFor(ws, 'stronger-radio')?.ok,
|
||||
true,
|
||||
'stronger control must finish before lookup release',
|
||||
);
|
||||
assert.equal(trace.lookupSignal.aborted, true);
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
h.c.turn,
|
||||
turn,
|
||||
'sibling control cannot invent a new turn',
|
||||
);
|
||||
assert.equal(trace.selections, 0);
|
||||
assert.equal(state.selected.id, 'existing-station');
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, false);
|
||||
assert.match(responseFor(ws, 'slow-selection').error, /cancelled/);
|
||||
assert.equal(
|
||||
responseFor(ws, 'slow-selection').enabled,
|
||||
!['disable', 'off'].includes(stronger),
|
||||
);
|
||||
assert.equal(responseFor(ws, 'query-sibling').ok, true);
|
||||
assert.equal(trace.options.get('query-sibling').signal.aborted, false);
|
||||
assert.equal(
|
||||
responseFor(ws, 'stronger-radio').enabled,
|
||||
!['disable', 'off'].includes(stronger),
|
||||
);
|
||||
assert.equal(trace.stops, stronger === 'stop' ? 1 : 0);
|
||||
assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0);
|
||||
f.lookupReleased.resolve();
|
||||
await tick();
|
||||
assert.equal(trace.selections, 0, 'late lookup cannot commit');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
for (const id of ['slow-selection', 'query-sibling', 'stronger-radio'])
|
||||
assert.equal(responses(ws).filter((r) => r.id === id).length, 1);
|
||||
} finally {
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const guard of ['malformed', 'replay', 'invalid-arguments']) {
|
||||
test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace } = f;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
if (guard === 'replay') {
|
||||
request(ws, 'stronger-radio', 'control_radio', { action: 'stop' });
|
||||
await h.c.toolQueue;
|
||||
}
|
||||
speechTurn(ws);
|
||||
if (guard === 'replay')
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
const stops = trace.stops;
|
||||
const handoffEpoch = h.c.radio.radioHandoffEpoch;
|
||||
request(
|
||||
ws,
|
||||
guard === 'malformed' ? '\n' : 'stronger-radio',
|
||||
'control_radio',
|
||||
{
|
||||
action: 'stop',
|
||||
...(guard === 'invalid-arguments'
|
||||
? { latitude: 91, longitude: 0 }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
await tick();
|
||||
assert.equal(trace.lookupSignal.aborted, false);
|
||||
assert.equal(trace.stops, stops);
|
||||
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
if (guard === 'invalid-arguments') {
|
||||
assert.equal(responseFor(ws, 'stronger-radio').ok, false);
|
||||
assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/);
|
||||
}
|
||||
f.lookupReleased.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
trace.selections,
|
||||
1,
|
||||
'older eligible work retains authority',
|
||||
);
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
} finally {
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test('native failed Stop releases its reservation and lets the older selection finish', async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'failed-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
|
||||
assert.equal(
|
||||
trace.lookupSignal.aborted,
|
||||
false,
|
||||
'provisional Stop cannot revoke work',
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 2);
|
||||
stopReleased.resolve(false);
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'failed-stop').ok, false);
|
||||
assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(trace.lookupSignal.aborted, false);
|
||||
f.lookupReleased.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(trace.selections, 1);
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('native pending Stop preserves selection completed before its own successful settlement', async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
f.lookupReleased.resolve();
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
stopReleased.resolve(true);
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'pending-stop').ok, true);
|
||||
assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station');
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
for (const restart of [false, true]) {
|
||||
test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
const oldDrain = h.c.toolQueue;
|
||||
const stopOptions = f.trace.options.get('pending-stop');
|
||||
assert.equal(h.c.pendingTools, 2);
|
||||
h.c.stop();
|
||||
assert.equal(stopOptions.signal.aborted, true);
|
||||
assert.equal(f.trace.lookupSignal.aborted, true);
|
||||
if (restart) {
|
||||
await h.c.start();
|
||||
request(h.sockets[1], 'new-query', 'control_radio', {
|
||||
action: 'status',
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(h.sockets[1], 'new-query').ok, true);
|
||||
}
|
||||
stopReleased.resolve(true);
|
||||
f.lookupReleased.resolve();
|
||||
await oldDrain;
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'pending-stop'), undefined);
|
||||
assert.equal(responseFor(ws, 'slow-selection'), undefined);
|
||||
assert.equal(f.trace.selections, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(h.c.isActive(), restart);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const phase of ['prepared', 'preflight']) {
|
||||
test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const ready = deferred();
|
||||
const stopped = [];
|
||||
let preflight;
|
||||
f.radio.playForVoice = () => ready.promise;
|
||||
f.radio.stopPlayback = (options) => {
|
||||
stopped.push(options);
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'prepared-play', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
responseFor(ws, 'prepared-play').radioPlaybackRequested,
|
||||
true,
|
||||
);
|
||||
if (phase === 'preflight')
|
||||
preflight = h.c.radio.startPendingRadioHandoff();
|
||||
const attemptId = h.c.radio.radioHandoffAttemptId;
|
||||
request(ws, 'stronger-stop', 'control_radio', { action: 'stop' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'stronger-stop').ok, true);
|
||||
assert.equal(f.state.selected.id, 'existing-station');
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup');
|
||||
assert.deepEqual(
|
||||
cleanups,
|
||||
phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [],
|
||||
);
|
||||
ready.resolve(true);
|
||||
await preflight;
|
||||
await tick();
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'late old readiness cannot stop voice',
|
||||
);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
ready.resolve(false);
|
||||
h.c.stop();
|
||||
await preflight;
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const phase of ['prepared', 'preflight', 'settling']) {
|
||||
test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const prepared = deferred();
|
||||
const settleResult = deferred();
|
||||
const playbackReady = deferred();
|
||||
let playCalls = 0;
|
||||
let stopCalls = 0;
|
||||
let selections = 0;
|
||||
let activeOptions;
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: null,
|
||||
audioState: 'stopped',
|
||||
voiceDucked: false,
|
||||
};
|
||||
const radio = {
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked(value) {
|
||||
state.voiceDucked = value;
|
||||
},
|
||||
pause() {
|
||||
state.audioState = 'paused';
|
||||
},
|
||||
subscribePlaybackControls() {
|
||||
return () => {};
|
||||
},
|
||||
selectRequestedStation(_criteria, options) {
|
||||
assert.equal(options.autoplay, false);
|
||||
selections++;
|
||||
state.selected = { id: 'station-1', name: 'Station One' };
|
||||
return state.selected;
|
||||
},
|
||||
playForVoice() {
|
||||
playCalls++;
|
||||
return playbackReady.promise;
|
||||
},
|
||||
stopPlayback() {
|
||||
stopCalls++;
|
||||
state.audioState = 'stopped';
|
||||
},
|
||||
};
|
||||
manager.register({
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
...radio,
|
||||
});
|
||||
const runner = createGevActionRunner({ viewer, dataManager: manager });
|
||||
let h;
|
||||
let preflight;
|
||||
const session = createVoiceSession({
|
||||
runner: async (name, args, options) => {
|
||||
activeOptions = options;
|
||||
const result = await runner(name, args, options);
|
||||
prepared.resolve(result);
|
||||
if (phase === 'settling') await settleResult.promise;
|
||||
return result;
|
||||
},
|
||||
createAdapter: (shared) => {
|
||||
h = harness({ ...shared, dataManager: manager, radioLayer: radio });
|
||||
return {
|
||||
start: (settings) => h.c.start(settings),
|
||||
stop: (settings) => h.c.stop(settings),
|
||||
sendText: (text) => h.c.sendTextCommand(text),
|
||||
sendMapEvent: (event) => h.c.notifyMapEvent(event),
|
||||
};
|
||||
},
|
||||
});
|
||||
try {
|
||||
await session.start();
|
||||
const ws = h.sockets[0];
|
||||
assert.equal(session.sendText('Select Station One on Radio'), true);
|
||||
request(ws, 'prepared-radio', 'control_radio', {
|
||||
action: 'select',
|
||||
stationQuery: 'Station One',
|
||||
});
|
||||
const actual = await prepared.promise;
|
||||
assert.equal(actual.ok, true);
|
||||
assert.equal(actual.radioPlaybackRequested, true);
|
||||
assert.equal(actual.stationId, 'station-1');
|
||||
assert.equal(manager.isEnabled('radio'), true);
|
||||
if (phase !== 'settling') {
|
||||
await h.c.toolQueue;
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
}
|
||||
if (phase === 'preflight') {
|
||||
preflight = h.c.radio.startPendingRadioHandoff();
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
assert.equal(playCalls, 1);
|
||||
}
|
||||
// Native call ownership revokes only this deferred playback. No caption
|
||||
// parsing or caption-driven provider policy participates in cancellation.
|
||||
ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } });
|
||||
if (phase === 'settling')
|
||||
assert.equal(activeOptions.signal.aborted, true);
|
||||
settleResult.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
assert.equal(manager.isEnabled('radio'), true);
|
||||
assert.equal(state.selected.id, 'station-1');
|
||||
assert.equal(selections, 1);
|
||||
const modelResult = responseFor(ws, 'prepared-radio');
|
||||
if (phase === 'settling')
|
||||
assert.equal(
|
||||
modelResult,
|
||||
undefined,
|
||||
'cancelled active call never sends a late result',
|
||||
);
|
||||
else {
|
||||
assert.equal(modelResult.ok, true);
|
||||
assert.equal(modelResult.stationId, 'station-1');
|
||||
}
|
||||
const stopsAfterCancellation = stopCalls;
|
||||
assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0);
|
||||
drainRadioReply(h);
|
||||
playbackReady.resolve(true);
|
||||
await preflight;
|
||||
await tick();
|
||||
assert.equal(
|
||||
playCalls,
|
||||
phase === 'preflight' ? 1 : 0,
|
||||
'drain cannot revive cancelled playback',
|
||||
);
|
||||
assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice');
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(session.sendText('Play radio'), true);
|
||||
request(ws, 'fresh-radio', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
drainRadioReply(h);
|
||||
await tick();
|
||||
assert.equal(playCalls, phase === 'preflight' ? 2 : 1);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'fresh typed Radio handoff alone ends voice',
|
||||
);
|
||||
} finally {
|
||||
settleResult.resolve();
|
||||
playbackReady.resolve(false);
|
||||
session.destroy();
|
||||
await preflight;
|
||||
runner.dispose();
|
||||
await manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
// A stronger Radio control skips the ordered queue. An older Radio call still
|
||||
// queued behind a slow sibling must not start after that control succeeded,
|
||||
// while a failed control or a later call in the same turn keeps its order.
|
||||
for (const [label, ids, stopOk, expectPending] of [
|
||||
[
|
||||
'older queued select is revoked by a later successful stop',
|
||||
['slow', 'sel', 'stop'],
|
||||
true,
|
||||
false,
|
||||
],
|
||||
[
|
||||
'a failed stop leaves the older queued select current',
|
||||
['slow', 'sel', 'stop'],
|
||||
false,
|
||||
true,
|
||||
],
|
||||
[
|
||||
'a select after the stop in the same turn still runs',
|
||||
['slow', 'stop', 'sel'],
|
||||
true,
|
||||
true,
|
||||
],
|
||||
]) {
|
||||
test(`stronger Radio ordering: ${label}`, async () => {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: { id: 'existing', name: 'Existing' },
|
||||
audioState: 'stopped',
|
||||
};
|
||||
const radio = {
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked() {},
|
||||
pause() {
|
||||
state.audioState = 'paused';
|
||||
return true;
|
||||
},
|
||||
stopPlayback() {
|
||||
state.audioState = 'stopped';
|
||||
return stopOk;
|
||||
},
|
||||
selectRequestedStation() {
|
||||
state.selected = { id: 'late', name: 'Late' };
|
||||
return state.selected;
|
||||
},
|
||||
async playForVoice() {
|
||||
return true;
|
||||
},
|
||||
subscribePlaybackControls: () => () => {},
|
||||
};
|
||||
manager.register(radio);
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: manager,
|
||||
placeSearch: {
|
||||
async geocode() {
|
||||
return { place: { lat: 10, lng: 20, label: 'Place' } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const slow = deferred();
|
||||
const h = harness({
|
||||
dataManager: manager,
|
||||
radioLayer: radio,
|
||||
async runAction(name, args, options) {
|
||||
if (name === 'slow_sibling') {
|
||||
await slow.promise;
|
||||
return { ok: true, action: 'slow_sibling' };
|
||||
}
|
||||
if (name === 'control_radio' && args.action === 'stop' && !stopOk)
|
||||
return {
|
||||
ok: false,
|
||||
action: 'control_radio',
|
||||
error: 'Radio stop failed',
|
||||
};
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
const calls = {
|
||||
slow: { id: 'slow', name: 'slow_sibling', args: {} },
|
||||
sel: {
|
||||
id: 'sel',
|
||||
name: 'control_radio',
|
||||
args: { action: 'select', locationQuery: 'Place' },
|
||||
},
|
||||
stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } },
|
||||
};
|
||||
try {
|
||||
await manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } });
|
||||
await tick();
|
||||
assert.equal(
|
||||
responseFor(ws, 'stop')?.ok,
|
||||
stopOk,
|
||||
'stop settles without waiting',
|
||||
);
|
||||
slow.resolve();
|
||||
await h.c.toolQueue;
|
||||
await tick();
|
||||
assert.equal(
|
||||
Boolean(h.c.radio.pendingRadioPlaybackResult),
|
||||
expectPending,
|
||||
);
|
||||
assert.equal(responseFor(ws, 'sel')?.ok, expectPending);
|
||||
} finally {
|
||||
slow.resolve();
|
||||
h.c.stop();
|
||||
runner.dispose();
|
||||
await manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
|
||||
const tool = (ws, id) =>
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const caption = (ws, text) =>
|
||||
ws.message({ serverContent: { inputTranscription: { text } } });
|
||||
const onset = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
const complete = (ws) => ws.message({ serverContent: { turnComplete: true } });
|
||||
const oldAudio = {
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
data: 'AAAA',
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const outputs = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
|
||||
// Socket/media are deterministic. Mutations use the production action runner.
|
||||
function fixture({ holdFirst = false } = {}) {
|
||||
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: 1000 },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(n) {
|
||||
position.x -= n;
|
||||
state.height -= n;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
zoomOut(n) {
|
||||
position.x += n;
|
||||
state.height += n;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
},
|
||||
scene: {
|
||||
canvas: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
},
|
||||
requestRender() {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: { layers: new Map() },
|
||||
styleManager: {
|
||||
setVoiceProvider: (p) => state.providerWrites.push(p),
|
||||
setPanelCollapsed: (...args) => state.panels.push(args),
|
||||
},
|
||||
});
|
||||
const entered = [];
|
||||
const gate = deferred();
|
||||
const h = harness({
|
||||
runAction: async (name, args, options) => {
|
||||
entered.push({ name, args, options });
|
||||
if (holdFirst && entered.length === 1) await gate.promise;
|
||||
if (!options.isCurrent())
|
||||
return { ok: false, action: name, changed: false };
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
return {
|
||||
...h,
|
||||
state,
|
||||
entered,
|
||||
gate,
|
||||
close() {
|
||||
gate.resolve();
|
||||
h.c.stop();
|
||||
runner.dispose();
|
||||
assert.equal(h.timers.size, 0);
|
||||
assert.deepEqual(state.providerWrites, []);
|
||||
assert.deepEqual(state.panels, []);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
for (const [id, order] of [
|
||||
['H20', 'before-boundary'],
|
||||
['H21', 'after-boundary'],
|
||||
]) {
|
||||
test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(oldAudio);
|
||||
onset(ws);
|
||||
caption(ws, 'zoom in');
|
||||
if (order === 'before-boundary') {
|
||||
tool(ws, 'ambiguous-before');
|
||||
await tick();
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.equal(h.entered.length, 0);
|
||||
assert.equal(outputs(ws).length, 0);
|
||||
}
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
if (order === 'after-boundary') {
|
||||
tool(ws, 'native-after');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.entered.length, 1);
|
||||
assert.equal(outputs(ws).at(-1).response.ok, true);
|
||||
complete(ws);
|
||||
}
|
||||
onset(ws);
|
||||
caption(ws, 'zoom in');
|
||||
tool(ws, 'fresh-clean-onset');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5);
|
||||
assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2);
|
||||
assert.equal(h.entered.at(-1).options.userTurn, undefined);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
outputs(ws).some((o) => o.id === 'ambiguous-before'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok,
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
caption(ws, 'switch to');
|
||||
tool(ws, 'ordinary-before-caption-resolution');
|
||||
await tick();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
h.advance(1499);
|
||||
caption(ws, 'Austin');
|
||||
h.advance(5000);
|
||||
tool(ws, 'released-sibling');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 562.5);
|
||||
assert.equal(h.entered.length, 2);
|
||||
assert.equal(
|
||||
h.entered.every((c) => c.options.userTurn === undefined),
|
||||
true,
|
||||
);
|
||||
assert.equal(outputs(ws).length, 2);
|
||||
assert.equal(
|
||||
outputs(ws).every((o) => o.response.ok),
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
for (const [id, exit] of [
|
||||
['H32', 'stop'],
|
||||
['H33', 'restart'],
|
||||
['H34', 'new-input'],
|
||||
]) {
|
||||
test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => {
|
||||
const h = fixture({ holdFirst: true });
|
||||
try {
|
||||
await h.c.start();
|
||||
let ws = h.sockets[0];
|
||||
onset(ws);
|
||||
caption(ws, 'switch to');
|
||||
tool(ws, 'old-owned-action');
|
||||
await tick();
|
||||
assert.equal(h.entered.length, 1);
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
assert.equal(h.state.height, 1000);
|
||||
const oldWork = h.c.toolQueue;
|
||||
const oldMessage = ws.onmessage;
|
||||
const oldOwner = h.entered[0].options;
|
||||
if (exit === 'new-input') h.c.sendTextCommand('zoom out');
|
||||
else h.c.stop();
|
||||
assert.equal(oldOwner.signal.aborted, true);
|
||||
assert.equal(oldOwner.isCurrent(), false);
|
||||
if (exit === 'restart') {
|
||||
await h.c.start();
|
||||
ws = h.sockets.at(-1);
|
||||
}
|
||||
if (exit !== 'new-input')
|
||||
oldMessage({
|
||||
data: JSON.stringify({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'late-old-call',
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in' },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
});
|
||||
caption(ws, 'Gemini');
|
||||
h.advance(1500);
|
||||
h.gate.resolve();
|
||||
await oldWork;
|
||||
await tick();
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
outputs(ws).some(
|
||||
(o) => o.id === 'old-owned-action' || o.id === 'late-old-call',
|
||||
),
|
||||
false,
|
||||
);
|
||||
if (exit === 'stop') {
|
||||
assert.equal(h.c.status, 'idle');
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} else {
|
||||
if (exit === 'new-input') {
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
} else h.c.sendTextCommand('zoom out');
|
||||
tool(ws, 'fresh-after-retirement');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.entered.length, 2);
|
||||
assert.equal(
|
||||
outputs(ws).find((o) => o.id === 'fresh-after-retirement').response
|
||||
.ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
}
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
turnOfEventLoop,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const OLD_TEXT = 'Old packet caption.';
|
||||
const OLD_PCM = 'AAABAA==';
|
||||
const POST_TOOL_PCM = 'AgADAA==';
|
||||
const OLD_CALL = 'old-coalesced-call';
|
||||
|
||||
function coalescedPacket(callId = OLD_CALL) {
|
||||
return {
|
||||
serverContent: {
|
||||
outputTranscription: { text: OLD_TEXT },
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
turnComplete: true,
|
||||
},
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: callId,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Local test observations wrap only injected device fakes and the actual
|
||||
// scheduler methods, delegating unchanged. No frozen fixture file is modified.
|
||||
function observePacketEffects(h) {
|
||||
const played = [];
|
||||
const narration = { audible: 0, toolStarted: 0, toolFinished: 0 };
|
||||
const createAudio = h.controller.createAudio;
|
||||
h.controller.createAudio = (callbacks) => {
|
||||
const audio = createAudio(callbacks);
|
||||
const play = audio.play.bind(audio);
|
||||
audio.play = (data, mimeType) => {
|
||||
played.push({ audio, data, mimeType });
|
||||
return play(data, mimeType);
|
||||
};
|
||||
return audio;
|
||||
};
|
||||
for (const method of Object.keys(narration)) {
|
||||
const invoke = h.controller.narration[method].bind(h.controller.narration);
|
||||
h.controller.narration[method] = (...args) => {
|
||||
narration[method]++;
|
||||
return invoke(...args);
|
||||
};
|
||||
}
|
||||
return { played, narration };
|
||||
}
|
||||
|
||||
function packetState(h) {
|
||||
const c = h.controller;
|
||||
return {
|
||||
epoch: c.epoch,
|
||||
inputGeneration: c.inputGeneration,
|
||||
turn: c.turn,
|
||||
completedTurns: c.completedTurns,
|
||||
pendingTools: c.pendingTools,
|
||||
modelComplete: c.modelComplete,
|
||||
pendingCompletionStatus: c.pendingCompletionStatus,
|
||||
postToolOutput: c.postToolOutput,
|
||||
modelSpoke: c.modelSpoke,
|
||||
assistant: c.transcripts.assistant,
|
||||
replyPending: c.replyPending,
|
||||
retiredReplyPending: c.retiredReplyPending,
|
||||
typedText: c.typedIntent?.text ?? null,
|
||||
typedResponseId: c.typedIntent?.responseId ?? null,
|
||||
narrationTools: c.narration.state().tools,
|
||||
narrationTimers: c.narration.state().pendingTimers,
|
||||
ownedTimers: h.clock.timers.size,
|
||||
};
|
||||
}
|
||||
|
||||
const completions = (h) =>
|
||||
h.events.filter((event) => event.type === 'completion');
|
||||
const toolEvents = (h, callId) =>
|
||||
h.events.filter(
|
||||
(event) =>
|
||||
['action-call', 'action-result', 'progress'].includes(event.type) &&
|
||||
event.callId === callId,
|
||||
);
|
||||
|
||||
function assertOldRemainderAbsent(h, observed) {
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.state.zooms, []);
|
||||
assert.deepEqual(h.runnerCalls, []);
|
||||
assert.deepEqual(toolEvents(h, OLD_CALL), []);
|
||||
assert.deepEqual(observed.played, []);
|
||||
assert.deepEqual(observed.narration, {
|
||||
audible: 0,
|
||||
toolStarted: 0,
|
||||
toolFinished: 0,
|
||||
});
|
||||
assert.deepEqual(completions(h), []);
|
||||
assert.equal(h.controller.completedTurns, 0);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
assert.equal(
|
||||
h.events.filter(
|
||||
(event) =>
|
||||
event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.final &&
|
||||
event.text === OLD_TEXT,
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
for (const socket of h.sockets)
|
||||
assert.equal(
|
||||
h.outputs(socket).some((output) => output.id === OLD_CALL),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
for (const mode of ['Stop', 'Stop then Start', 'typed successor']) {
|
||||
test(
|
||||
`assistant transcript observer ${mode} fences the rest of a coalesced native packet`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const observed = observePacketEffects(h);
|
||||
const oldSocket = await h.start();
|
||||
const oldAudio = h.audios[0];
|
||||
h.activity(oldSocket);
|
||||
const before = packetState(h);
|
||||
let reached = false;
|
||||
let restart;
|
||||
let typedAccepted;
|
||||
let afterObserver;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (
|
||||
reached ||
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'assistant' ||
|
||||
event.final ||
|
||||
event.text !== OLD_TEXT
|
||||
)
|
||||
return;
|
||||
reached = true;
|
||||
if (mode === 'typed successor') {
|
||||
typedAccepted = h.session.sendText('Zoom in for the typed successor');
|
||||
} else {
|
||||
h.session.stop();
|
||||
if (mode === 'Stop then Start') restart = h.session.start();
|
||||
}
|
||||
afterObserver = packetState(h);
|
||||
});
|
||||
oldSocket.message(coalescedPacket());
|
||||
remove();
|
||||
assert.equal(
|
||||
reached,
|
||||
true,
|
||||
'the actual common transcript observer ran synchronously',
|
||||
);
|
||||
// A retired reply may consume its own received terminal, but cannot
|
||||
// complete or overwrite the successor. Stop/restart permits no old writes.
|
||||
const afterPacketExpected =
|
||||
mode === 'typed successor'
|
||||
? {
|
||||
...afterObserver,
|
||||
retiredReplyPending: false,
|
||||
ownedTimers: afterObserver.ownedTimers - 1,
|
||||
}
|
||||
: afterObserver;
|
||||
assert.deepEqual(packetState(h), afterPacketExpected);
|
||||
await h.controller.toolQueue;
|
||||
await turnOfEventLoop();
|
||||
if (restart) await restart;
|
||||
assertOldRemainderAbsent(h, observed);
|
||||
|
||||
let successorSocket;
|
||||
if (mode === 'typed successor') {
|
||||
assert.equal(typedAccepted, true);
|
||||
assert.equal(
|
||||
h.controller.epoch,
|
||||
before.epoch,
|
||||
'typed replacement shares the socket epoch',
|
||||
);
|
||||
assert.ok(h.controller.inputGeneration > before.inputGeneration);
|
||||
assert.ok(h.controller.turn > before.turn);
|
||||
assert.deepEqual(packetState(h), afterPacketExpected);
|
||||
assert.equal(
|
||||
h.controller.typedIntent.text,
|
||||
'Zoom in for the typed successor',
|
||||
);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(oldAudio.closed, false);
|
||||
const typedMessages = oldSocket.sent.filter((message) =>
|
||||
message.clientContent?.turns?.some((turn) =>
|
||||
turn.parts?.some(
|
||||
(part) => part.text === 'Zoom in for the typed successor',
|
||||
),
|
||||
),
|
||||
);
|
||||
assert.equal(typedMessages.length, 1);
|
||||
// The received packet already carried the old terminal. No extra
|
||||
// provider boundary may be invented before testing the actual successor.
|
||||
assert.equal(h.controller.retiredBoundaryWait, null);
|
||||
assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.completedTurns, 0);
|
||||
assert.deepEqual(completions(h), []);
|
||||
successorSocket = oldSocket;
|
||||
} else {
|
||||
assert.equal(oldSocket.closed, true);
|
||||
assert.equal(oldAudio.closed, true);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
if (mode === 'Stop') {
|
||||
assert.equal(h.session.state, 'idle');
|
||||
successorSocket = await h.start();
|
||||
} else {
|
||||
assert.equal(h.session.state, 'listening');
|
||||
assert.equal(h.sockets.length, 2);
|
||||
assert.equal(h.audios.length, 2);
|
||||
assert.notEqual(h.audios[1], oldAudio);
|
||||
assert.equal(h.audios[1].closed, false);
|
||||
assert.equal(h.audios[1].pending, false);
|
||||
successorSocket = h.sockets[1];
|
||||
}
|
||||
h.activity(successorSocket);
|
||||
}
|
||||
await h.tool('owned-after-packet-cancel', successorSocket);
|
||||
assertSuccessfulZoom(
|
||||
h,
|
||||
successorSocket,
|
||||
'owned-after-packet-cancel',
|
||||
1000,
|
||||
);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(
|
||||
h.outputs(successorSocket).some((output) => output.id === OLD_CALL),
|
||||
false,
|
||||
);
|
||||
h.complete(successorSocket);
|
||||
assert.equal(completions(h).length, 1);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const observed = observePacketEffects(h);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
ws.message(coalescedPacket('ordinary-coalesced'));
|
||||
assert.equal(
|
||||
h.controller.completedTurns,
|
||||
1,
|
||||
'first generation terminal is counted once',
|
||||
);
|
||||
assert.equal(h.controller.pendingTools, 1);
|
||||
assert.deepEqual(
|
||||
observed.played.map(({ data }) => data),
|
||||
[OLD_PCM],
|
||||
);
|
||||
assert.equal(observed.played[0].audio, h.audios[0]);
|
||||
assert.deepEqual(completions(h), [], 'tool and playback are still pending');
|
||||
await h.controller.toolQueue;
|
||||
assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
assert.equal(observed.narration.toolStarted, 1);
|
||||
assert.equal(observed.narration.toolFinished, 1);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
h.audios[0].drain();
|
||||
assert.equal(
|
||||
h.controller.modelComplete,
|
||||
false,
|
||||
'function response waits for the post-tool generation',
|
||||
);
|
||||
assert.equal(h.controller.completedTurns, 1);
|
||||
assert.deepEqual(
|
||||
completions(h),
|
||||
[],
|
||||
'draining first-generation PCM cannot finish the tool reply',
|
||||
);
|
||||
|
||||
// The accepted contract has two generations: the function response prompts
|
||||
// a fresh model reply. This boundary is intentional, not a stale-packet fix.
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'The requested zoom is complete.' },
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
data: POST_TOOL_PCM,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(h.controller.completedTurns, 2);
|
||||
assert.deepEqual(
|
||||
observed.played.map(({ data }) => data),
|
||||
[OLD_PCM, POST_TOOL_PCM],
|
||||
);
|
||||
assert.deepEqual(
|
||||
completions(h),
|
||||
[],
|
||||
'post-tool playback must drain before completion',
|
||||
);
|
||||
h.audios[0].drain();
|
||||
assert.equal(completions(h).length, 1);
|
||||
assert.equal(completions(h)[0].status, 'completed');
|
||||
assert.equal(completions(h)[0].spoken, true);
|
||||
assert.equal(completions(h)[0].audioDrained, true);
|
||||
assert.equal(h.controller.pendingCompletionStatus, null);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
assert.equal(observed.narration.audible, 2);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.equal(h.outputs(ws).length, 1);
|
||||
h.audios[0].drain();
|
||||
assert.equal(
|
||||
completions(h).length,
|
||||
1,
|
||||
'a duplicate device drain cannot complete twice',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Real native message-type fields are separate packets here; no extra old
|
||||
// completion is inserted before the accepted successor.
|
||||
const BOUNDARY_OLD = 'Prior reply caption.';
|
||||
const BOUNDARY_NEXT = 'Zoom in for the typed successor';
|
||||
const boundaryZoomCall = (id) => ({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
function boundarySnapshot(h) {
|
||||
const c = h.controller;
|
||||
return {
|
||||
retiredReplyPending: c.retiredReplyPending,
|
||||
retiredBoundaryWait: Boolean(c.retiredBoundaryWait),
|
||||
replyPending: c.replyPending,
|
||||
typedText: c.typedIntent?.text ?? null,
|
||||
completions: h.events.filter((event) => event.type === 'completion').length,
|
||||
};
|
||||
}
|
||||
|
||||
for (const mode of [
|
||||
'partial assistant transcript',
|
||||
'input transcript',
|
||||
'interruption event',
|
||||
'listening event',
|
||||
'final assistant transcript control',
|
||||
]) {
|
||||
test(
|
||||
`received native terminal preserves typed successor after ${mode}`,
|
||||
{ concurrency: false, timeout: 5000 },
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
let oldDispatch, gate;
|
||||
const interruptionMode =
|
||||
mode === 'interruption event' || mode === 'listening event';
|
||||
if (interruptionMode) {
|
||||
h.session.sendText('Prior zoom request');
|
||||
gate = h.holdResult('already-committed-old-tool');
|
||||
oldDispatch = h.tool('already-committed-old-tool', ws);
|
||||
await gate.ready;
|
||||
} else if (mode === 'input transcript') {
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Previously emitted output' },
|
||||
},
|
||||
});
|
||||
}
|
||||
let entered = 0,
|
||||
accepted;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (entered) return;
|
||||
const matches =
|
||||
mode === 'interruption event'
|
||||
? event.type === 'interruption' && event.reason === 'user-speech'
|
||||
: mode === 'listening event'
|
||||
? event.type === 'state' && event.state === 'listening'
|
||||
: mode === 'input transcript'
|
||||
? event.type === 'transcript' &&
|
||||
event.role === 'user' &&
|
||||
event.text === 'Old input caption'
|
||||
: event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.text === BOUNDARY_OLD &&
|
||||
event.final ===
|
||||
(mode === 'final assistant transcript control');
|
||||
if (!matches) return;
|
||||
entered++;
|
||||
accepted = h.session.sendText(BOUNDARY_NEXT);
|
||||
});
|
||||
if (interruptionMode)
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
else
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...(mode === 'input transcript'
|
||||
? { inputTranscription: { text: 'Old input caption' } }
|
||||
: {}),
|
||||
outputTranscription: { text: BOUNDARY_OLD },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
remove();
|
||||
if (oldDispatch) {
|
||||
await oldDispatch;
|
||||
gate.release();
|
||||
}
|
||||
await turnOfEventLoop();
|
||||
const afterPacket = boundarySnapshot(h);
|
||||
const beforeSuccessorHeight = h.state.height;
|
||||
// No extra synthetic old boundary. This is the next reply to the accepted text.
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Handling your new zoom.' },
|
||||
},
|
||||
});
|
||||
ws.message(boundaryZoomCall('typed-successor-call'));
|
||||
await h.controller.toolQueue;
|
||||
await turnOfEventLoop();
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
const afterSuccessor = boundarySnapshot(h);
|
||||
assert.equal(entered, 1);
|
||||
assert.equal(accepted, true);
|
||||
assert.equal(
|
||||
afterPacket.retiredReplyPending,
|
||||
false,
|
||||
'received old terminal must settle its retirement',
|
||||
);
|
||||
assert.equal(afterPacket.retiredBoundaryWait, false);
|
||||
assert.equal(afterPacket.typedText, BOUNDARY_NEXT);
|
||||
assert.equal(
|
||||
h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call')
|
||||
.length,
|
||||
1,
|
||||
'accepted typed successor must reach the actual runner',
|
||||
);
|
||||
assert.equal(h.state.height, beforeSuccessorHeight * 0.75);
|
||||
assert.equal(afterSuccessor.completions, 1);
|
||||
assert.equal(afterSuccessor.replyPending, false);
|
||||
assert.equal(
|
||||
h.controller.cancelledCalls.has('typed-successor-call'),
|
||||
false,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const terminalAlreadyApplied of [false, true]) {
|
||||
test(
|
||||
`typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
let entered = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (
|
||||
entered ||
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'assistant' ||
|
||||
event.text !== 'Boundary control' ||
|
||||
event.final !== terminalAlreadyApplied
|
||||
)
|
||||
return;
|
||||
entered = true;
|
||||
assert.equal(h.session.sendText('First typed replacement'), true);
|
||||
if (terminalAlreadyApplied)
|
||||
assert.equal(h.session.sendText('Second typed replacement'), true);
|
||||
});
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Boundary control' },
|
||||
...(terminalAlreadyApplied ? { turnComplete: true } : {}),
|
||||
},
|
||||
});
|
||||
remove();
|
||||
assert.equal(entered, true);
|
||||
// With no old terminal received, or when that terminal was consumed
|
||||
// before the observer, a real predecessor boundary is still outstanding.
|
||||
assert.equal(h.controller.retiredReplyPending, true);
|
||||
assert.ok(h.controller.retiredBoundaryWait);
|
||||
assert.equal(completions(h).length, 0);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.retiredBoundaryWait, null);
|
||||
await h.tool('after-real-pending-boundary', ws);
|
||||
assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000);
|
||||
h.complete(ws);
|
||||
assert.equal(completions(h).length, 1);
|
||||
},
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -61,10 +61,8 @@ export function initGevVoiceCommands(options) {
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
commands.session?.signal?.addEventListener(
|
||||
'abort',
|
||||
() => pointer?.destroy(),
|
||||
{ once: true },
|
||||
);
|
||||
const lifetime = commands.signal ?? commands.session?.signal;
|
||||
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
|
||||
if (lifetime?.aborted) pointer?.destroy();
|
||||
return commands;
|
||||
}
|
||||
@@ -182,6 +182,63 @@ export function pointerChip(snapshot) {
|
||||
};
|
||||
}
|
||||
|
||||
/** Dedupe key of the clearing pointer_context item. */
|
||||
export const POINTER_NONE_KEY = 'none';
|
||||
|
||||
/**
|
||||
* The voice-card event for a turn's pointer: a gesture reveals its target at
|
||||
* once; open-mic and typed turns reveal it only through announcePointer.
|
||||
* @param {object|null} snapshot
|
||||
* @param {boolean} gesture
|
||||
* @returns {{announced: object|null, pointer: object|null}}
|
||||
*/
|
||||
export function pointerTurnEvent(snapshot, gesture) {
|
||||
const chip = gesture ? pointerChip(snapshot) : null;
|
||||
return chip
|
||||
? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } }
|
||||
: { announced: null, pointer: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* The pointer_context item a turn should send, or null when there is nothing
|
||||
* new. Without a gesture the model learns only what kind of thing is pointed
|
||||
* at, never which. Losing a sent pointer produces one clearing item.
|
||||
* @param {object|null} snapshot
|
||||
* @param {{at: string, gesture: boolean, sentKey: string|null}} turn
|
||||
* @returns {{item: object, key: string}|null}
|
||||
*/
|
||||
export function pointerTurnContext(snapshot, { at, gesture, sentKey }) {
|
||||
const chip = pointerChip(snapshot);
|
||||
const item = !chip
|
||||
? sentKey && sentKey !== POINTER_NONE_KEY
|
||||
? { type: 'pointer_context', at, target: 'none' }
|
||||
: null
|
||||
: gesture
|
||||
? pointerContextItem(snapshot)
|
||||
: {
|
||||
type: 'pointer_context',
|
||||
at,
|
||||
pointing: snapshot.entity?.kind || 'ground',
|
||||
};
|
||||
if (!item) return null;
|
||||
const { at: _at, ...content } = item;
|
||||
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
|
||||
return key === sentKey ? null : { item, key };
|
||||
}
|
||||
|
||||
/**
|
||||
* The voice-card event revealing a pointer a tool resolved, once per snapshot.
|
||||
* @param {object|null} snapshot
|
||||
* @param {object|null} announced Snapshot already shown.
|
||||
* @returns {object|null}
|
||||
*/
|
||||
export function pointerAnnouncement(snapshot, announced) {
|
||||
const chip = pointerChip(snapshot);
|
||||
return chip && snapshot !== announced
|
||||
? { ...chip, screenPx: snapshot.screenPx }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track the pointer over one element (the globe canvas).
|
||||
* @param {object} options
|
||||
@@ -330,7 +387,14 @@ export function createPointerTracker({
|
||||
* (open mic only when it moved recently: `recentMs`). The result becomes
|
||||
* the turn's referent (null when stale).
|
||||
*/
|
||||
beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) {
|
||||
beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) {
|
||||
// Delayed admitted audio keeps its capture's pointer without replacing a
|
||||
// newer physical hold. Undefined preserves existing caller behavior.
|
||||
if (capturedSnapshot !== undefined) {
|
||||
const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot;
|
||||
active = snapshot?.fresh ? snapshot : null;
|
||||
return snapshot;
|
||||
}
|
||||
const reuse =
|
||||
at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS;
|
||||
const snapshot = reuse ? held.snapshot : capture(at, { recentMs });
|
||||
|
||||
@@ -6,8 +6,13 @@ import {
|
||||
POINTER_HOLD_MS,
|
||||
buildPointerSnapshot,
|
||||
createPointerTracker,
|
||||
POINTER_NONE_KEY,
|
||||
formatPointerContext,
|
||||
pointerAnnouncement,
|
||||
pointerChip,
|
||||
pointerContextItem,
|
||||
pointerTurnContext,
|
||||
pointerTurnEvent,
|
||||
} from './pointerContext.js';
|
||||
|
||||
function harness({ pick } = {}) {
|
||||
@@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => {
|
||||
assert.equal(h.tracker.capture('keydown').fresh, false);
|
||||
});
|
||||
|
||||
test('delayed speech uses its released capture without replacing a newer physical hold', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const capturedA = h.tracker.hold();
|
||||
h.tracker.releaseHold();
|
||||
h.move(200, 100);
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
|
||||
assert.equal(h.tracker.activeSnapshot(), capturedA);
|
||||
const heldB = h.tracker.hold();
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
|
||||
assert.equal(h.tracker.activeSnapshot(), capturedA);
|
||||
assert.equal(h.tracker.isHeld(), true);
|
||||
assert.equal(h.tracker.beginTurn('speech_start'), heldB);
|
||||
h.tracker.destroy();
|
||||
});
|
||||
|
||||
test('explicit stale captures clear the referent while a silent hold leaves it alone', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const current = h.tracker.beginTurn('text');
|
||||
h.move(200, 100);
|
||||
h.tracker.hold();
|
||||
h.tracker.releaseHold();
|
||||
assert.equal(h.tracker.activeSnapshot(), current);
|
||||
for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) {
|
||||
h.tracker.beginTurn('text');
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot });
|
||||
assert.equal(h.tracker.activeSnapshot(), null);
|
||||
}
|
||||
h.tracker.destroy();
|
||||
});
|
||||
|
||||
test('a capture arriving after destruction cannot restore an active referent', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const capturedSnapshot = h.tracker.hold();
|
||||
assert.equal(capturedSnapshot.fresh, true);
|
||||
h.tracker.destroy();
|
||||
const result = h.tracker.beginTurn('speech_start', { capturedSnapshot });
|
||||
assert.equal(result.fresh, false);
|
||||
assert.equal(h.tracker.activeSnapshot(), null);
|
||||
assert.equal(h.tracker.isHeld(), false);
|
||||
assert.equal(h.tracker.beginTurn('speech_start').fresh, false);
|
||||
});
|
||||
|
||||
test('format: the model item is compact inert JSON; labels are bounded data', () => {
|
||||
const snapshot = buildPointerSnapshot({
|
||||
at: 'speech_start',
|
||||
@@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => {
|
||||
assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' });
|
||||
assert.equal(pointerChip(null), null);
|
||||
});
|
||||
|
||||
test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => {
|
||||
const snapshot = {
|
||||
fresh: true,
|
||||
at: 'keydown',
|
||||
entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' },
|
||||
lat: 10,
|
||||
lon: 20,
|
||||
screen: { x: 5, y: 6 },
|
||||
screenPx: { x: 5, y: 6 },
|
||||
};
|
||||
const shown = pointerTurnEvent(snapshot, true);
|
||||
assert.equal(shown.announced, snapshot);
|
||||
assert.deepEqual(shown.pointer, {
|
||||
kind: 'this',
|
||||
label: 'UAL1',
|
||||
screenPx: { x: 5, y: 6 },
|
||||
});
|
||||
assert.deepEqual(pointerTurnEvent(snapshot, false), {
|
||||
announced: null,
|
||||
pointer: null,
|
||||
});
|
||||
|
||||
const gesture = pointerTurnContext(snapshot, {
|
||||
at: 'keydown',
|
||||
gesture: true,
|
||||
sentKey: null,
|
||||
});
|
||||
assert.deepEqual(gesture.item, pointerContextItem(snapshot));
|
||||
assert.equal(
|
||||
pointerTurnContext(snapshot, {
|
||||
at: 'keydown',
|
||||
gesture: true,
|
||||
sentKey: gesture.key,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
assert.deepEqual(
|
||||
pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null })
|
||||
.item,
|
||||
{ type: 'pointer_context', at: 'text', pointing: 'aircraft' },
|
||||
);
|
||||
|
||||
const cleared = pointerTurnContext(null, {
|
||||
at: 'text',
|
||||
gesture: false,
|
||||
sentKey: gesture.key,
|
||||
});
|
||||
assert.deepEqual(cleared, {
|
||||
item: { type: 'pointer_context', at: 'text', target: 'none' },
|
||||
key: POINTER_NONE_KEY,
|
||||
});
|
||||
for (const sentKey of [null, POINTER_NONE_KEY])
|
||||
assert.equal(
|
||||
pointerTurnContext(null, { at: 'text', gesture: false, sentKey }),
|
||||
null,
|
||||
);
|
||||
|
||||
assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer);
|
||||
assert.equal(pointerAnnouncement(snapshot, snapshot), null);
|
||||
assert.equal(pointerAnnouncement(null, null), null);
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
export const VOICE_PROVIDERS = Object.freeze({
|
||||
openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name
|
||||
gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name
|
||||
});
|
||||
|
||||
/** Optional providers require an explicit host opt-in and an available factory. */
|
||||
export function supportedVoiceProviders(
|
||||
requested = ['openai'],
|
||||
factories = {},
|
||||
) {
|
||||
if (!Array.isArray(requested))
|
||||
throw new TypeError('supportedProviders must be an array');
|
||||
const selected = [...new Set(requested)].filter(
|
||||
(name) =>
|
||||
Object.hasOwn(VOICE_PROVIDERS, name) &&
|
||||
typeof factories[name] === 'function',
|
||||
);
|
||||
if (!selected.length)
|
||||
throw new Error('No supported voice provider is configured');
|
||||
return Object.freeze(selected);
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { supportedVoiceProviders } from './providerCapabilities.js';
|
||||
|
||||
const STORAGE_KEY = 'gev.voice.provider';
|
||||
|
||||
export function readVoiceProvider(storage, supported = ['openai', 'gemini']) {
|
||||
let saved;
|
||||
try {
|
||||
saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY);
|
||||
} catch {
|
||||
/* Storage is optional. */
|
||||
}
|
||||
return supported.includes(saved)
|
||||
? saved
|
||||
: supported.includes('openai')
|
||||
? 'openai'
|
||||
: supported[0];
|
||||
}
|
||||
|
||||
/** Keep one public controller identity while replacing each provider's lifetime. */
|
||||
export function createProviderCommands(options, { bind, factories, storage }) {
|
||||
const supportedProviders = supportedVoiceProviders(
|
||||
options.supportedProviders ?? Object.keys(factories),
|
||||
factories,
|
||||
);
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
let provider = readVoiceProvider(storage, supportedProviders);
|
||||
let current;
|
||||
let disposed = false;
|
||||
const lifetime = new AbortController();
|
||||
let mountGeneration = 0;
|
||||
let voiceEnabled = false;
|
||||
const stored = () => {
|
||||
try {
|
||||
return storage ?? globalThis.localStorage;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const stop = (settings = {}) => {
|
||||
if (disposed) return;
|
||||
voiceEnabled = false;
|
||||
if (settings.removeUi) {
|
||||
disposed = true;
|
||||
options.signal?.removeEventListener('abort', destroy);
|
||||
}
|
||||
try {
|
||||
current?.session.stop(settings);
|
||||
} finally {
|
||||
if (settings.removeUi) {
|
||||
lifetime.abort();
|
||||
options.runner.dispose?.();
|
||||
}
|
||||
}
|
||||
};
|
||||
const destroy = () => stop({ removeUi: true });
|
||||
const selectProvider = (next) => {
|
||||
if (disposed || !supportedProviders.includes(next) || next === provider)
|
||||
return false;
|
||||
const resume = voiceEnabled && current.session.isActive();
|
||||
// Carry the input mode, never a held key, microphone or conversation.
|
||||
const pushToTalk = Boolean(
|
||||
current.session.adapter.getStartOptions?.().pushToTalk,
|
||||
);
|
||||
const generation = ++mountGeneration;
|
||||
voiceEnabled = false;
|
||||
current.session.destroy();
|
||||
// Disposal suppresses the retired adapter's idle event. Clear the shared
|
||||
// conversation explicitly before the new mount can use these owners.
|
||||
if (disposed || generation !== mountGeneration) return false;
|
||||
options.pointer?.clear();
|
||||
options.referents?.clear();
|
||||
options.runner.resetConversation?.();
|
||||
if (disposed || generation !== mountGeneration) return false;
|
||||
provider = next;
|
||||
try {
|
||||
stored()?.setItem(STORAGE_KEY, provider);
|
||||
} catch {
|
||||
/* Storage is optional. */
|
||||
}
|
||||
mount();
|
||||
if (resume && !disposed && provider === next)
|
||||
void current.session.start({ pushToTalk });
|
||||
globalThis.document?.getElementById('gev-voice-provider')?.focus();
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleSessionEvent = (event) => {
|
||||
if (event?.type === 'state' && event.state === 'connecting')
|
||||
voiceEnabled = true;
|
||||
if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) {
|
||||
voiceEnabled = false;
|
||||
}
|
||||
// The host still observes the mounted provider's neutral session events.
|
||||
options.onSessionEvent?.(event);
|
||||
};
|
||||
|
||||
const sendTextCommand = (text) => {
|
||||
if (disposed) return false;
|
||||
return current.session.sendText(text);
|
||||
};
|
||||
|
||||
const api = {
|
||||
// The shared action surface is independent of an active voice connection.
|
||||
runner: options.runner,
|
||||
supportedProviders,
|
||||
signal: lifetime.signal,
|
||||
get provider() {
|
||||
return provider;
|
||||
},
|
||||
get session() {
|
||||
return current?.session;
|
||||
},
|
||||
get status() {
|
||||
return current?.status ?? current?.session.state ?? 'idle';
|
||||
},
|
||||
get state() {
|
||||
return current?.session.state ?? 'idle';
|
||||
},
|
||||
get disposed() {
|
||||
return disposed;
|
||||
},
|
||||
start: (settings) => !disposed && current.session.start(settings),
|
||||
stop,
|
||||
destroy,
|
||||
isActive: () => !disposed && current.session.isActive(),
|
||||
sendTextCommand,
|
||||
sendText: sendTextCommand,
|
||||
notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event),
|
||||
};
|
||||
// Existing diagnostic and controller accessors continue to reflect the active provider.
|
||||
const facade = new Proxy(api, {
|
||||
get(target, name, receiver) {
|
||||
if (Reflect.has(target, name)) return Reflect.get(target, name, receiver);
|
||||
const value = current?.[name];
|
||||
return typeof value === 'function' ? value.bind(current) : value;
|
||||
},
|
||||
});
|
||||
function mount() {
|
||||
voiceEnabled = false;
|
||||
const generation = ++mountGeneration;
|
||||
current = bind({
|
||||
...options,
|
||||
signal: lifetime.signal,
|
||||
resetExisting: false,
|
||||
disposeRunner: false,
|
||||
provider,
|
||||
supportedProviders,
|
||||
onProviderChange: selectProvider,
|
||||
onSessionEvent: (event) => {
|
||||
if (!disposed && generation === mountGeneration)
|
||||
handleSessionEvent(event);
|
||||
},
|
||||
createSession: factories[provider],
|
||||
});
|
||||
window.__gevVoiceCommands = facade;
|
||||
}
|
||||
mount();
|
||||
options.signal?.addEventListener('abort', destroy, { once: true });
|
||||
if (options.signal?.aborted) destroy();
|
||||
return facade;
|
||||
}
|
||||
@@ -0,0 +1,888 @@
|
||||
// Imports intentionally target the candidate, never the immutable reference.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
createProviderCommands,
|
||||
readVoiceProvider,
|
||||
} from './providerCommands.js';
|
||||
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
|
||||
import { createVoiceCommands as createCommands } from './commands.js';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { describeUnsavedWork, t as translate } from '../i18n/index.js';
|
||||
|
||||
const PROVIDER_KEY = 'gev.voice.provider';
|
||||
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((yes, no) => {
|
||||
resolve = yes;
|
||||
reject = no;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
// Listener counts observe the actual binder's registrations/removals.
|
||||
class Element extends EventTarget {
|
||||
listeners = new Map();
|
||||
attributes = new Map();
|
||||
addEventListener(type, handler, options) {
|
||||
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
|
||||
this.listeners.get(type).add(handler);
|
||||
super.addEventListener(type, handler, options);
|
||||
}
|
||||
removeEventListener(type, handler, options) {
|
||||
this.listeners.get(type)?.delete(handler);
|
||||
super.removeEventListener(type, handler, options);
|
||||
}
|
||||
listenerCount(type) {
|
||||
return this.listeners.get(type)?.size || 0;
|
||||
}
|
||||
setAttribute(name, value) {
|
||||
this.attributes.set(name, String(value));
|
||||
}
|
||||
focus() {
|
||||
this.focusCount = (this.focusCount || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (const custom of [false, true]) {
|
||||
test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => {
|
||||
const prior = globalThis.window;
|
||||
globalThis.window = new EventTarget();
|
||||
let controls;
|
||||
t.after(() => {
|
||||
controls?.session.destroy();
|
||||
globalThis.window = prior;
|
||||
});
|
||||
const ui = {
|
||||
root: { dataset: {}, remove() {} },
|
||||
button: new Element(),
|
||||
status: {},
|
||||
detail: {},
|
||||
providerField: {},
|
||||
};
|
||||
let defaultCreated = 0;
|
||||
let customCreated = 0;
|
||||
const options = {
|
||||
runner: async () => ({ ok: true }),
|
||||
createControl: () => ui,
|
||||
createController: ({ onSessionEvent }) => {
|
||||
defaultCreated++;
|
||||
return {
|
||||
ui,
|
||||
start({ pushToTalk }) {
|
||||
this.pushToTalkMode = pushToTalk;
|
||||
onSessionEvent({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
syncCostUi() {},
|
||||
bindPushToTalkShortcut() {},
|
||||
sendTextCommand: (text) => `default:${text}`,
|
||||
};
|
||||
},
|
||||
};
|
||||
if (custom) {
|
||||
options.supportedProviders = ['gemini'];
|
||||
options.createSession = ({ emit }) => {
|
||||
customCreated++;
|
||||
return {
|
||||
capabilities: { pushToTalk: false },
|
||||
start() {
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
sendText: (text) => `custom:${text}`,
|
||||
sendMapEvent: () => true,
|
||||
};
|
||||
};
|
||||
}
|
||||
controls = createCommands(options);
|
||||
await controls.session.start({ pushToTalk: true });
|
||||
assert.equal(defaultCreated, custom ? 0 : 1);
|
||||
assert.equal(customCreated, custom ? 1 : 0);
|
||||
assert.equal(
|
||||
controls.session.sendText('test'),
|
||||
`${custom ? 'custom' : 'default'}:test`,
|
||||
);
|
||||
assert.equal(ui.providerField.hidden, true);
|
||||
assert.equal(ui.root.dataset.providerChoice, 'false');
|
||||
if (!custom)
|
||||
assert.deepEqual(controls.session.adapter.getStartOptions(), {
|
||||
pushToTalk: true,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Real facade -> real binder -> real common session and real action runner.
|
||||
* Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count
|
||||
* calls to real pointer/referent/conversation cleanup; they do not replace it.
|
||||
*/
|
||||
function compositionFixture(
|
||||
t,
|
||||
{
|
||||
supportedProviders = ['openai', 'gemini'],
|
||||
factoryProviders = ['openai', 'gemini'],
|
||||
savedProvider = null,
|
||||
storageDenied = false,
|
||||
gateStarts = () => false,
|
||||
onSessionEvent = () => {},
|
||||
} = {},
|
||||
) {
|
||||
const original = new Map(
|
||||
['window', 'document'].map((key) => [
|
||||
key,
|
||||
Object.getOwnPropertyDescriptor(globalThis, key),
|
||||
]),
|
||||
);
|
||||
const lifetime = new AbortController();
|
||||
const views = [];
|
||||
const instances = [];
|
||||
const mounts = [];
|
||||
const events = [];
|
||||
const order = [];
|
||||
const annotations = new Set();
|
||||
const storageWrites = [];
|
||||
const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 };
|
||||
const state = { height: 1000, zooms: 0, renders: 0 };
|
||||
let facade;
|
||||
let pointer;
|
||||
globalThis.window = new EventTarget();
|
||||
globalThis.document = {
|
||||
getElementById: (id) =>
|
||||
id === 'gev-voice-provider' ? views.at(-1)?.providerSelect : null,
|
||||
};
|
||||
t.after(async () => {
|
||||
try {
|
||||
facade?.destroy();
|
||||
pointer?.destroy();
|
||||
for (const instance of instances)
|
||||
for (const gate of instance.gates) gate.resolve();
|
||||
await Promise.allSettled(instances.flatMap((instance) => instance.jobs));
|
||||
} finally {
|
||||
for (const [key, descriptor] of original) {
|
||||
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
|
||||
else delete globalThis[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const canvas = new EventTarget();
|
||||
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: state.height },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(metres) {
|
||||
position.x -= metres;
|
||||
state.height -= metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
zoomOut(metres) {
|
||||
position.x += metres;
|
||||
state.height += metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
},
|
||||
scene: {
|
||||
canvas,
|
||||
requestRender() {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
const dataManager = { layers: new Map() };
|
||||
pointer = createPointerTracker({ element: null, now: () => 1000 });
|
||||
const referents = createReferentRegistry();
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager,
|
||||
styleManager: {},
|
||||
deixis: { pointer, referents },
|
||||
});
|
||||
for (const [object, method, counter] of [
|
||||
[pointer, 'clear', 'pointer'],
|
||||
[referents, 'clear', 'referents'],
|
||||
[runner, 'resetConversation', 'conversation'],
|
||||
[runner, 'dispose', 'dispose'],
|
||||
]) {
|
||||
const originalMethod = object[method];
|
||||
object[method] = (...args) => {
|
||||
counters[counter]++;
|
||||
order.push(counter);
|
||||
return originalMethod.apply(object, args);
|
||||
};
|
||||
}
|
||||
|
||||
const factories = Object.fromEntries(
|
||||
factoryProviders.map((provider) => [
|
||||
provider,
|
||||
(hooks) => {
|
||||
const id = instances.length;
|
||||
let generation = 0;
|
||||
const instance = {
|
||||
id,
|
||||
provider,
|
||||
hooks,
|
||||
starts: [],
|
||||
stops: [],
|
||||
jobs: [],
|
||||
gates: [],
|
||||
acquired: 0,
|
||||
microphoneEnabled: false,
|
||||
bound: 0,
|
||||
input: { pushToTalk: false, held: false },
|
||||
texts: [],
|
||||
mapEvents: [],
|
||||
};
|
||||
instances.push(instance);
|
||||
order.push(`factory:${id}`);
|
||||
const adapter = {
|
||||
identity: `${provider}:${id}`,
|
||||
capabilities: {
|
||||
pushToTalk: true,
|
||||
costControls: provider === 'openai',
|
||||
},
|
||||
controller: {
|
||||
providerDiagnostic() {
|
||||
return this.session.adapter.identity;
|
||||
},
|
||||
},
|
||||
start(settings = {}) {
|
||||
const attempt = ++generation;
|
||||
instance.starts.push({ ...settings });
|
||||
instance.input.pushToTalk = Boolean(settings.pushToTalk);
|
||||
hooks.emit({
|
||||
type: 'state',
|
||||
state: 'connecting',
|
||||
detail: `adapter:${provider}`,
|
||||
});
|
||||
const job = (async () => {
|
||||
if (attempt !== generation || hooks.signal.aborted) return;
|
||||
if (gateStarts(instance)) {
|
||||
const gate = deferred();
|
||||
instance.gates.push(gate);
|
||||
await gate.promise;
|
||||
}
|
||||
// Adapters own cancellation of their asynchronous acquisition.
|
||||
if (attempt !== generation || hooks.signal.aborted) return;
|
||||
instance.acquired++;
|
||||
instance.microphoneEnabled =
|
||||
!instance.input.pushToTalk || instance.input.held;
|
||||
hooks.emit({
|
||||
type: 'state',
|
||||
state: 'listening',
|
||||
detail: provider,
|
||||
});
|
||||
})();
|
||||
instance.jobs.push(job);
|
||||
return job;
|
||||
},
|
||||
stop(settings = {}) {
|
||||
generation++;
|
||||
instance.stops.push({ ...settings });
|
||||
order.push(`stop:${id}`);
|
||||
instance.input.pushToTalk = instance.input.held = false;
|
||||
instance.microphoneEnabled = false;
|
||||
},
|
||||
getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }),
|
||||
bindControls() {
|
||||
instance.bound++;
|
||||
},
|
||||
sendText(text) {
|
||||
instance.texts.push(text);
|
||||
return `${provider}:${text}`;
|
||||
},
|
||||
sendMapEvent(event) {
|
||||
instance.mapEvents.push(event);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
instance.adapter = adapter;
|
||||
return adapter;
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
function createControl() {
|
||||
const id = views.length;
|
||||
order.push(`control:${id}`);
|
||||
const providerSelect = new Element();
|
||||
providerSelect.ownerDocument = { createElement: () => ({}) };
|
||||
providerSelect.replaceChildren = (...options) => {
|
||||
providerSelect.options = options;
|
||||
};
|
||||
const view = {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
removeCount: 0,
|
||||
remove() {
|
||||
this.removeCount++;
|
||||
order.push(`remove:${id}`);
|
||||
},
|
||||
},
|
||||
button: new Element(),
|
||||
providerSelect,
|
||||
providerField: {},
|
||||
status: {},
|
||||
detail: {},
|
||||
errorDetail: {},
|
||||
tierButton: {},
|
||||
costValue: {},
|
||||
helpDetail: {},
|
||||
card: {},
|
||||
};
|
||||
views.push(view);
|
||||
return view;
|
||||
}
|
||||
function passiveView() {
|
||||
return {
|
||||
events: [],
|
||||
outlines: [],
|
||||
destroyed: 0,
|
||||
handle(event) {
|
||||
this.events.push(event);
|
||||
},
|
||||
outline(event) {
|
||||
this.outlines.push(event);
|
||||
},
|
||||
destroy() {
|
||||
this.destroyed++;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
facade = createProviderCommands(
|
||||
{
|
||||
runner,
|
||||
pointer,
|
||||
referents,
|
||||
dataManager,
|
||||
signal: lifetime.signal,
|
||||
supportedProviders,
|
||||
createControl,
|
||||
createCard() {
|
||||
const card = passiveView();
|
||||
views.at(-1).cardView = card;
|
||||
return card;
|
||||
},
|
||||
createReticle() {
|
||||
const reticle = passiveView();
|
||||
views.at(-1).reticle = reticle;
|
||||
return reticle;
|
||||
},
|
||||
annotations: {
|
||||
onOutlineEvent(callback) {
|
||||
annotations.add(callback);
|
||||
return () => annotations.delete(callback);
|
||||
},
|
||||
},
|
||||
onSessionEvent(event) {
|
||||
events.push(event);
|
||||
onSessionEvent(event, facade);
|
||||
},
|
||||
},
|
||||
{
|
||||
factories,
|
||||
storage: {
|
||||
getItem() {
|
||||
if (storageDenied) throw Error('Storage denied');
|
||||
return savedProvider;
|
||||
},
|
||||
setItem(key, value) {
|
||||
if (storageDenied) throw Error('Storage denied');
|
||||
storageWrites.push([key, value]);
|
||||
savedProvider = value;
|
||||
},
|
||||
},
|
||||
bind(options) {
|
||||
// Keep the real callback so stale-event suppression tests exercise the
|
||||
// facade's mount generation as well as the common session's disposal.
|
||||
const mount = { options, callback: options.onSessionEvent };
|
||||
mounts.push(mount);
|
||||
mount.controls = bindVoiceCommands(options);
|
||||
return mount.controls;
|
||||
},
|
||||
},
|
||||
);
|
||||
return {
|
||||
facade,
|
||||
runner,
|
||||
pointer,
|
||||
referents,
|
||||
lifetime,
|
||||
instances,
|
||||
views,
|
||||
mounts,
|
||||
events,
|
||||
order,
|
||||
annotations,
|
||||
counters,
|
||||
storageWrites,
|
||||
state,
|
||||
select(provider) {
|
||||
const select = views.at(-1).providerSelect;
|
||||
select.value = provider;
|
||||
select.dispatchEvent(new Event('change'));
|
||||
},
|
||||
seedConversation() {
|
||||
pointer.beginTurn('text', {
|
||||
capturedSnapshot: { fresh: true, at: 'text' },
|
||||
});
|
||||
referents.recordResult('frame_overhead', {
|
||||
ok: true,
|
||||
display: { title: 'One aircraft' },
|
||||
referents: [
|
||||
{ n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' },
|
||||
],
|
||||
});
|
||||
assert.ok(pointer.activeSnapshot());
|
||||
assert.equal(referents.list().length, 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('provider preference uses only the supplied host capability list', () => {
|
||||
const stored = (value) => ({ getItem: () => value });
|
||||
assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai');
|
||||
assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini');
|
||||
assert.equal(
|
||||
readVoiceProvider(stored('unknown'), ['gemini', 'openai']),
|
||||
'openai',
|
||||
);
|
||||
assert.equal(
|
||||
readVoiceProvider(stored('gemini'), ['openai', 'gemini']),
|
||||
'gemini',
|
||||
);
|
||||
assert.equal(
|
||||
readVoiceProvider(
|
||||
{
|
||||
getItem() {
|
||||
throw Error('denied');
|
||||
},
|
||||
},
|
||||
['gemini'],
|
||||
),
|
||||
'gemini',
|
||||
);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{
|
||||
title: 'OpenAI-only host ignores saved Gemini',
|
||||
requested: ['openai'],
|
||||
saved: 'gemini',
|
||||
expected: ['openai'],
|
||||
provider: 'openai',
|
||||
},
|
||||
{
|
||||
title: 'Gemini-only host falls back from saved OpenAI',
|
||||
requested: ['gemini'],
|
||||
saved: 'openai',
|
||||
expected: ['gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
{
|
||||
title: 'dual host restores Gemini',
|
||||
requested: ['openai', 'gemini'],
|
||||
saved: 'gemini',
|
||||
expected: ['openai', 'gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
{
|
||||
title: 'unknown, duplicate and unavailable factories are excluded',
|
||||
requested: ['unknown', 'gemini', 'openai', 'gemini'],
|
||||
factories: ['gemini'],
|
||||
saved: 'openai',
|
||||
expected: ['gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
]) {
|
||||
test(scenario.title, (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
supportedProviders: scenario.requested,
|
||||
factoryProviders: scenario.factories,
|
||||
savedProvider: scenario.saved,
|
||||
});
|
||||
assert.deepEqual(f.facade.supportedProviders, scenario.expected);
|
||||
assert.equal(Object.isFrozen(f.facade.supportedProviders), true);
|
||||
assert.equal(f.facade.provider, scenario.provider);
|
||||
assert.deepEqual(
|
||||
f.views[0].providerSelect.options.map((option) => option.value),
|
||||
scenario.expected,
|
||||
);
|
||||
assert.equal(
|
||||
f.views[0].providerField.hidden,
|
||||
scenario.expected.length === 1,
|
||||
);
|
||||
assert.equal(
|
||||
f.views[0].root.dataset.providerChoice,
|
||||
String(scenario.expected.length > 1),
|
||||
);
|
||||
assert.equal(f.instances.length, 1);
|
||||
assert.equal(
|
||||
f.instances[0].starts.length,
|
||||
0,
|
||||
'discovery must not start a provider',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
});
|
||||
}
|
||||
|
||||
test('invalid or empty host support fails before mounting a control', () => {
|
||||
let mounts = 0;
|
||||
const dependencies = {
|
||||
bind() {
|
||||
mounts++;
|
||||
},
|
||||
factories: { openai() {} },
|
||||
};
|
||||
assert.throws(
|
||||
() =>
|
||||
createProviderCommands({ supportedProviders: 'gemini' }, dependencies),
|
||||
/must be an array/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createProviderCommands(
|
||||
{ supportedProviders: ['gemini', 'unknown'] },
|
||||
dependencies,
|
||||
),
|
||||
/No supported voice provider/,
|
||||
);
|
||||
assert.equal(mounts, 0);
|
||||
});
|
||||
|
||||
test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => {
|
||||
const f = compositionFixture(t);
|
||||
const firstSession = f.facade.session;
|
||||
const outerSignal = f.facade.signal;
|
||||
const first = f.instances[0];
|
||||
// PTT controls can activate the adapter directly; the facade must learn
|
||||
// activity from its common events, not only from facade.start().
|
||||
first.input.held = true;
|
||||
await first.adapter.start({ pushToTalk: true });
|
||||
assert.equal(first.microphoneEnabled, true);
|
||||
assert.equal(f.facade.isActive(), true);
|
||||
f.seedConversation();
|
||||
const before = { ...f.counters };
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const second = f.instances[1];
|
||||
assert.equal(window.__gevVoiceCommands, f.facade);
|
||||
assert.equal(f.facade.runner, f.runner);
|
||||
assert.equal(f.facade.signal, outerSignal);
|
||||
assert.equal(outerSignal.aborted, false);
|
||||
assert.equal(firstSession.disposed, true);
|
||||
assert.equal(first.hooks.signal.aborted, true);
|
||||
assert.deepEqual(first.stops, [{ removeUi: true }]);
|
||||
assert.equal(first.input.held, false);
|
||||
assert.equal(first.microphoneEnabled, false);
|
||||
assert.deepEqual(second.starts, [{ pushToTalk: true }]);
|
||||
assert.equal(second.input.held, false);
|
||||
assert.equal(
|
||||
second.microphoneEnabled,
|
||||
false,
|
||||
'a held key must never migrate to the new adapter',
|
||||
);
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1'));
|
||||
assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1'));
|
||||
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1'));
|
||||
assert.equal(f.views[0].button.listenerCount('click'), 0);
|
||||
assert.equal(f.views[0].providerSelect.listenerCount('change'), 0);
|
||||
assert.equal(f.views[1].providerSelect.listenerCount('change'), 1);
|
||||
assert.equal(f.views[1].providerSelect.focusCount, 1);
|
||||
assert.equal(f.views[1].tierButton.hidden, true);
|
||||
assert.equal(f.views[1].costValue.hidden, true);
|
||||
assert.equal(f.facade.providerDiagnostic(), 'gemini:1');
|
||||
assert.equal(f.pointer.activeSnapshot(), null);
|
||||
assert.deepEqual(f.referents.list(), []);
|
||||
for (const key of ['pointer', 'referents', 'conversation'])
|
||||
assert.equal(
|
||||
f.counters[key],
|
||||
before[key] + 1,
|
||||
`one explicit replacement reset: ${key}`,
|
||||
);
|
||||
assert.equal(
|
||||
f.counters.dispose,
|
||||
0,
|
||||
'provider replacement keeps the page runner',
|
||||
);
|
||||
assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]);
|
||||
assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello');
|
||||
assert.deepEqual(first.texts, []);
|
||||
assert.deepEqual(second.texts, ['hello']);
|
||||
const result = await f.facade.runner('adjust_camera_zoom', {
|
||||
direction: 'in',
|
||||
amount: 'little',
|
||||
});
|
||||
assert.equal(
|
||||
result.ok,
|
||||
true,
|
||||
'the actual shared action runner remains usable after switching',
|
||||
);
|
||||
assert.equal(f.state.zooms, 1);
|
||||
});
|
||||
|
||||
for (const initialState of ['idle', 'error']) {
|
||||
test(`${initialState} selection never autostarts; manual start still works`, async (t) => {
|
||||
const f = compositionFixture(t, { storageDenied: true });
|
||||
if (initialState === 'error') {
|
||||
await f.facade.start({ pushToTalk: true });
|
||||
f.instances[0].adapter.stop({ preserveStatus: true });
|
||||
f.instances[0].hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Fixture failure',
|
||||
});
|
||||
}
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(
|
||||
f.facade.provider,
|
||||
'gemini',
|
||||
'denied storage cannot block a supported selection',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
assert.deepEqual(f.instances[1].starts, []);
|
||||
f.select('gemini');
|
||||
f.select('unknown');
|
||||
assert.equal(
|
||||
f.instances.length,
|
||||
2,
|
||||
'same and unsupported selections cannot remount',
|
||||
);
|
||||
assert.equal(f.facade.provider, 'gemini');
|
||||
await f.facade.start({ pushToTalk: false });
|
||||
assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]);
|
||||
assert.equal(f.instances[1].microphoneEnabled, true);
|
||||
});
|
||||
}
|
||||
|
||||
for (const outcome of ['resolve', 'reject']) {
|
||||
test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
gateStarts: (instance) => instance.id === 0,
|
||||
});
|
||||
const oldStart = f.facade.start({ pushToTalk: false });
|
||||
const old = f.instances[0];
|
||||
assert.equal(f.facade.state, 'connecting');
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
const current = f.instances[1];
|
||||
const eventCount = f.events.length;
|
||||
const currentStops = current.stops.length;
|
||||
const currentCardCount = f.views[1].cardView.events.length;
|
||||
old.gates[0][outcome](new Error('Late retired startup failure'));
|
||||
await oldStart;
|
||||
for (const state of ['idle', 'error', 'connecting']) {
|
||||
old.hooks.emit({ type: 'state', state, detail: 'retired adapter' });
|
||||
f.mounts[0].callback({
|
||||
type: 'state',
|
||||
state,
|
||||
detail: 'retired forward callback',
|
||||
});
|
||||
}
|
||||
assert.equal(f.events.length, eventCount);
|
||||
assert.equal(f.views[1].cardView.events.length, currentCardCount);
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
assert.equal(old.acquired, 0);
|
||||
assert.equal(current.stops.length, currentStops);
|
||||
assert.equal(current.starts.length, 1);
|
||||
f.facade.stop();
|
||||
f.mounts[0].callback({ type: 'state', state: 'connecting' });
|
||||
f.select('openai');
|
||||
assert.deepEqual(
|
||||
f.instances[2].starts,
|
||||
[],
|
||||
'a retired connecting callback cannot revive voice intent',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
});
|
||||
}
|
||||
|
||||
test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
gateStarts: (instance) => instance.id === 1,
|
||||
});
|
||||
await f.facade.start();
|
||||
f.select('gemini');
|
||||
const replacement = f.instances[1];
|
||||
assert.equal(f.facade.state, 'connecting');
|
||||
f.facade.stop();
|
||||
replacement.gates[0].resolve();
|
||||
await Promise.all(replacement.jobs);
|
||||
assert.equal(replacement.acquired, 0);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
f.select('openai');
|
||||
assert.deepEqual(f.instances[2].starts, []);
|
||||
await f.facade.start();
|
||||
assert.equal(
|
||||
f.facade.state,
|
||||
'listening',
|
||||
'a later explicit start remains healthy',
|
||||
);
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => {
|
||||
const f = compositionFixture(t);
|
||||
if (emitsIdle) {
|
||||
const instance = f.instances[0];
|
||||
const originalStop = instance.adapter.stop;
|
||||
instance.adapter.stop = function (settings = {}) {
|
||||
originalStop.call(this, settings);
|
||||
if (!settings.preserveStatus && !settings.removeUi)
|
||||
instance.hooks.emit({ type: 'state', state: 'idle' });
|
||||
};
|
||||
}
|
||||
for (const end of ['stop', 'error']) {
|
||||
await f.facade.start();
|
||||
f.seedConversation();
|
||||
const before = { ...f.counters };
|
||||
const generation = f.referents.generation();
|
||||
if (end === 'stop') f.facade.stop();
|
||||
else {
|
||||
f.instances[0].adapter.stop({ preserveStatus: true });
|
||||
f.instances[0].hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Fixture failure',
|
||||
});
|
||||
}
|
||||
assert.equal(f.pointer.activeSnapshot(), null);
|
||||
assert.deepEqual(f.referents.list(), []);
|
||||
assert.equal(f.referents.generation(), generation + 1);
|
||||
for (const key of ['pointer', 'referents', 'conversation'])
|
||||
assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('outer destruction disposes the runner once and removes each provider binding', async (t) => {
|
||||
const priorUnsaved = describeUnsavedWork();
|
||||
const f = compositionFixture(t);
|
||||
await f.facade.start();
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(f.annotations.size, 1);
|
||||
assert.ok(
|
||||
describeUnsavedWork().includes(translate('the active voice session')),
|
||||
);
|
||||
const oldCardEvents = f.views[0].cardView.events.length;
|
||||
for (const callback of f.annotations) callback({ id: 'fixture-outline' });
|
||||
assert.equal(f.views[0].cardView.outlines.length, 0);
|
||||
assert.equal(f.views[1].cardView.outlines.length, 1);
|
||||
assert.equal(f.instances[1].mapEvents.length, 1);
|
||||
f.facade.destroy();
|
||||
f.facade.destroy();
|
||||
f.facade.stop({ removeUi: true });
|
||||
f.lifetime.abort();
|
||||
assert.equal(f.counters.dispose, 1);
|
||||
assert.equal(f.facade.signal.aborted, true);
|
||||
assert.equal(f.facade.disposed, true);
|
||||
assert.equal(f.annotations.size, 0);
|
||||
assert.deepEqual(describeUnsavedWork(), priorUnsaved);
|
||||
const finalEvents = f.events.length;
|
||||
const starts = f.instances.map((instance) => instance.starts.length);
|
||||
for (const [index, view] of f.views.entries()) {
|
||||
assert.equal(view.root.removeCount, 1);
|
||||
assert.equal(view.cardView.destroyed, 1);
|
||||
assert.equal(view.reticle.destroyed, 1);
|
||||
assert.equal(view.button.listenerCount('click'), 0);
|
||||
assert.equal(view.providerSelect.listenerCount('change'), 0);
|
||||
assert.equal(f.instances[index].hooks.signal.aborted, true);
|
||||
view.button.dispatchEvent(new Event('click'));
|
||||
view.providerSelect.value = 'openai';
|
||||
view.providerSelect.dispatchEvent(new Event('change'));
|
||||
f.instances[index].hooks.emit({ type: 'state', state: 'connecting' });
|
||||
f.mounts[index].callback({ type: 'state', state: 'connecting' });
|
||||
}
|
||||
assert.equal(f.events.length, finalEvents);
|
||||
assert.equal(f.views[0].cardView.events.length, oldCardEvents);
|
||||
assert.deepEqual(
|
||||
f.instances.map((instance) => instance.starts.length),
|
||||
starts,
|
||||
);
|
||||
assert.equal(f.instances.length, 2);
|
||||
assert.equal(f.facade.start(), false);
|
||||
assert.equal(f.facade.sendText('after destroy'), false);
|
||||
assert.equal(f.facade.notifyMapEvent({}), false);
|
||||
const result = await f.facade.runner('adjust_camera_zoom', {
|
||||
direction: 'in',
|
||||
amount: 'little',
|
||||
});
|
||||
assert.equal(result.cancelled, true);
|
||||
assert.equal(
|
||||
f.state.zooms,
|
||||
0,
|
||||
'the actual runner cannot move the camera after outer disposal',
|
||||
);
|
||||
});
|
||||
|
||||
for (const action of ['stop', 'destroy']) {
|
||||
for (const phase of ['common', 'adapter']) {
|
||||
test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => {
|
||||
let intercepted = 0;
|
||||
const f = compositionFixture(t, {
|
||||
onSessionEvent(event, facade) {
|
||||
if (
|
||||
facade?.provider !== 'gemini' ||
|
||||
event.type !== 'state' ||
|
||||
event.state !== 'connecting'
|
||||
)
|
||||
return;
|
||||
const matching =
|
||||
phase === 'common'
|
||||
? event.detail === undefined
|
||||
: event.detail === 'adapter:gemini';
|
||||
if (matching && intercepted++ === 0) facade[action]();
|
||||
},
|
||||
});
|
||||
await f.facade.start({ pushToTalk: true });
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const replacement = f.instances[1];
|
||||
assert.equal(intercepted, 1);
|
||||
assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1);
|
||||
assert.equal(replacement.acquired, 0);
|
||||
assert.equal(replacement.microphoneEnabled, false);
|
||||
assert.equal(f.facade.isActive(), false);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
assert.equal(
|
||||
f.events.filter(
|
||||
(event) => event.state === 'listening' && event.detail === 'gemini',
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
assert.equal(f.facade.disposed, action === 'destroy');
|
||||
assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0);
|
||||
if (action === 'stop') {
|
||||
f.select('openai');
|
||||
assert.deepEqual(
|
||||
f.instances[2].starts,
|
||||
[],
|
||||
'Stop cleared the intent to resume',
|
||||
);
|
||||
await f.facade.start({ pushToTalk: false });
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
} else {
|
||||
f.select('openai');
|
||||
assert.equal(f.instances.length, 2);
|
||||
assert.equal(f.facade.start(), false);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js';
|
||||
import { createNarrationScheduler } from './narration.js';
|
||||
import { createEarcon } from './earcon.js';
|
||||
import {
|
||||
POINTER_NONE_KEY,
|
||||
POINTER_OPEN_MIC_RECENT_MS,
|
||||
pointerChip,
|
||||
pointerContextItem,
|
||||
pointerAnnouncement,
|
||||
pointerTurnContext,
|
||||
pointerTurnEvent,
|
||||
} from './pointerContext.js';
|
||||
|
||||
/** Dedupe key of the clearing pointer_context item. */
|
||||
const POINTER_NONE_KEY = 'none';
|
||||
|
||||
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
|
||||
import { postDebugLog } from './realtimeDiagnostics.js';
|
||||
|
||||
@@ -287,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
} catch {
|
||||
snapshot = null;
|
||||
}
|
||||
const chip = pointerChip(snapshot);
|
||||
this.pointerAnnounced = gesture && chip ? snapshot : null;
|
||||
this.emitSessionEvent({
|
||||
type: 'pointer',
|
||||
pointer:
|
||||
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
|
||||
const event = pointerTurnEvent(snapshot, gesture);
|
||||
this.pointerAnnounced = event.announced;
|
||||
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
|
||||
const context = pointerTurnContext(snapshot, {
|
||||
at,
|
||||
gesture,
|
||||
sentKey: this.pointerSentKey,
|
||||
});
|
||||
const item = !chip
|
||||
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
|
||||
? { type: 'pointer_context', at, target: 'none' }
|
||||
: null
|
||||
: gesture
|
||||
? pointerContextItem(snapshot)
|
||||
: // What kind of thing, never which: no label or coordinates.
|
||||
{
|
||||
type: 'pointer_context',
|
||||
at,
|
||||
pointing: snapshot.entity?.kind || 'ground',
|
||||
};
|
||||
if (!item) return snapshot;
|
||||
const { at: _at, ...content } = item;
|
||||
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
|
||||
if (key === this.pointerSentKey) return snapshot;
|
||||
if (this._turns.notifyMapEvent(item)) {
|
||||
this.pointerSentKey = key;
|
||||
if (!context) return snapshot;
|
||||
if (this._turns.notifyMapEvent(context.item)) {
|
||||
this.pointerSentKey = context.key;
|
||||
const chip = context.key !== POINTER_NONE_KEY;
|
||||
this.debugLog('pointer.context', {
|
||||
at,
|
||||
target: chip ? snapshot.target : 'none',
|
||||
@@ -327,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
*/
|
||||
announcePointer() {
|
||||
const snapshot = this.pointer?.activeSnapshot?.() || null;
|
||||
const chip = pointerChip(snapshot);
|
||||
if (!chip || snapshot === this.pointerAnnounced) return false;
|
||||
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
|
||||
if (!pointer) return false;
|
||||
this.pointerAnnounced = snapshot;
|
||||
this.emitSessionEvent({
|
||||
type: 'pointer',
|
||||
pointer: { ...chip, screenPx: snapshot.screenPx },
|
||||
});
|
||||
this.emitSessionEvent({ type: 'pointer', pointer });
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -247,7 +247,9 @@ export class RealtimeRadio {
|
||||
handoffChannel?.readyState === 'open',
|
||||
},
|
||||
);
|
||||
const stillCurrent = handoffEpoch === this.radioHandoffEpoch;
|
||||
const stillCurrent =
|
||||
handoffEpoch === this.radioHandoffEpoch &&
|
||||
this.radioHandoffAttemptId === handoffAttemptId;
|
||||
if (this.radioHandoffAttemptId === handoffAttemptId) {
|
||||
this.radioHandoffInFlight = false;
|
||||
this.radioHandoffAttemptId = null;
|
||||
@@ -266,6 +268,25 @@ export class RealtimeRadio {
|
||||
}
|
||||
}
|
||||
|
||||
/** Revoke one call's prepared/preflight result, preserving other owners. */
|
||||
cancelPlaybackResult(result) {
|
||||
if (!result) return;
|
||||
if (this.pendingRadioPlaybackResult === result) {
|
||||
this.pendingRadioPlaybackResult = null;
|
||||
this.radioHandoffDeferredByReservation = false;
|
||||
}
|
||||
if (this.radioHandoffInFlightResult !== result) return;
|
||||
const attemptId = this.radioHandoffAttemptId;
|
||||
const shouldStopPlayback = this.radioHandoffInFlight;
|
||||
this.radioHandoffInFlightResult = null;
|
||||
this.radioHandoffInFlight = false;
|
||||
this.radioHandoffAttemptId = null;
|
||||
// Keep the group epoch and sibling reservations; release ownership before
|
||||
// Stop can synchronously notify playback observers.
|
||||
if (shouldStopPlayback)
|
||||
this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId });
|
||||
}
|
||||
|
||||
/** Invalidate delayed Radio work inside the requested authority scope. */
|
||||
abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) {
|
||||
for (const [controller, metadata] of this.activeRadioToolControllers) {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { GevRealtimeController } from './realtimeController.js';
|
||||
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve));
|
||||
const playback = (id) => ({
|
||||
ok: true,
|
||||
action: 'control_radio',
|
||||
radioAction: 'play',
|
||||
radioPlaybackRequested: true,
|
||||
id,
|
||||
});
|
||||
|
||||
function fixture(t) {
|
||||
const attempts = [],
|
||||
stopped = [],
|
||||
corrections = [],
|
||||
states = [];
|
||||
const controller = new GevRealtimeController({
|
||||
ui: {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
querySelectorAll: () => [],
|
||||
remove() {},
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
errorDetail: {},
|
||||
},
|
||||
runner: async () => ({ ok: false }),
|
||||
debugSink: null,
|
||||
onSessionEvent: (event) => states.push(event),
|
||||
radioLayer: {
|
||||
setVoiceDucked() {},
|
||||
playForVoice: ({ attemptId }) =>
|
||||
new Promise((resolve, reject) =>
|
||||
attempts.push({ attemptId, resolve, reject }),
|
||||
),
|
||||
stopPlayback: (options) => stopped.push(options),
|
||||
},
|
||||
});
|
||||
controller.queueResponseCreate = (message) => corrections.push(message);
|
||||
controller.dc = { readyState: 'open', send() {}, close() {} };
|
||||
t.after(() => controller.stop({ removeUi: true }));
|
||||
return { controller, attempts, stopped, corrections, states };
|
||||
}
|
||||
|
||||
test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
const a = playback('A');
|
||||
c.pendingRadioPlaybackResult = a;
|
||||
const pending = c.startPendingRadioHandoff();
|
||||
const reservation = c.reserveRadioToolHandoff();
|
||||
assert.equal(c.pendingRadioPlaybackResult, a);
|
||||
assert.equal(c.radioHandoffInFlight, false);
|
||||
h.attempts[0].reject(new Error('obsolete frozen preflight'));
|
||||
await pending;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
assert.equal(
|
||||
h.states.some((event) => event.detail === 'Radio did not start'),
|
||||
false,
|
||||
);
|
||||
c.settleRadioToolHandoffReservation(reservation, { commit: false });
|
||||
await tick();
|
||||
assert.equal(h.attempts.length, 2);
|
||||
assert.equal(c.radioHandoffInFlightResult, a);
|
||||
assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId);
|
||||
h.attempts[1].reject(new Error('current preflight failed'));
|
||||
await tick();
|
||||
assert.equal(h.corrections.length, 1);
|
||||
assert.equal(c.status, 'listening');
|
||||
});
|
||||
|
||||
test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
c.pendingRadioPlaybackResult = playback('A');
|
||||
const old = c.startPendingRadioHandoff();
|
||||
c.cancelRadioHandoff();
|
||||
h.attempts[0].reject(new Error('cancelled preflight'));
|
||||
await old;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
c.pendingRadioPlaybackResult = playback('B');
|
||||
const current = c.startPendingRadioHandoff();
|
||||
h.attempts[1].resolve(false);
|
||||
await current;
|
||||
assert.equal(h.corrections.length, 1);
|
||||
});
|
||||
|
||||
test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
const a = playback('A'),
|
||||
b = playback('B');
|
||||
c.pendingRadioPlaybackResult = a;
|
||||
const old = c.startPendingRadioHandoff();
|
||||
const siblingEpoch = c.radioHandoffEpoch;
|
||||
c.pendingRadioPlaybackResult = b;
|
||||
c._radio.cancelPlaybackResult(a);
|
||||
c._radio.cancelPlaybackResult(a);
|
||||
c._radio.cancelPlaybackResult(playback('unknown'));
|
||||
assert.equal(c.radioHandoffEpoch, siblingEpoch);
|
||||
assert.equal(c.pendingRadioPlaybackResult, b);
|
||||
assert.equal(h.stopped.length, 1);
|
||||
h.attempts[0].reject(new Error('late cancelled preflight'));
|
||||
await old;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
const current = c.startPendingRadioHandoff();
|
||||
const reservation = c.reserveRadioToolHandoff();
|
||||
c._radio.cancelPlaybackResult(b);
|
||||
assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true);
|
||||
h.attempts[1].resolve(false);
|
||||
await current;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
c.settleRadioToolHandoffReservation(reservation, { commit: false });
|
||||
await tick();
|
||||
assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed');
|
||||
});
|
||||
@@ -16,6 +16,7 @@ export function createRealtimeSession({
|
||||
controller,
|
||||
capabilities: { costControls: true, pushToTalk: true },
|
||||
start: (settings) => controller.start(settings),
|
||||
getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }),
|
||||
stop: (settings) => controller.stop(settings),
|
||||
sendText: (text) => controller.sendTextCommand(text),
|
||||
sendMapEvent: (event) => controller.notifyMapEvent(event),
|
||||
|
||||
@@ -137,7 +137,7 @@ export class RealtimeTurns {
|
||||
content: [{ type: 'input_text', text: cleanText }],
|
||||
},
|
||||
};
|
||||
this.sendRealtimeEvent(itemEvent, 'client.user_text');
|
||||
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
|
||||
this.requestUserTextResponse();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { GevRealtimeController } from './realtimeController.js';
|
||||
|
||||
function fixture({ pointer = null, onUserSend = () => {} } = {}) {
|
||||
const sent = [];
|
||||
const executed = [];
|
||||
const rejectedTexts = new Set();
|
||||
let rejectResponse = false;
|
||||
const controller = new GevRealtimeController({
|
||||
runner: async (name) => {
|
||||
executed.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
backend: {},
|
||||
debugSink: null,
|
||||
pointer,
|
||||
ui: {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
querySelectorAll: () => [],
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
},
|
||||
});
|
||||
controller.dc = {
|
||||
readyState: 'open',
|
||||
send(body) {
|
||||
const value = JSON.parse(body);
|
||||
const text = value.item?.role === 'user' && value.item.content?.[0]?.text;
|
||||
if (value.item?.role === 'user') onUserSend();
|
||||
if (
|
||||
rejectedTexts.has(text) ||
|
||||
(rejectResponse && value.type === 'response.create')
|
||||
)
|
||||
throw new Error('Synthetic channel rejection');
|
||||
sent.push(value);
|
||||
},
|
||||
close() {
|
||||
this.readyState = 'closed';
|
||||
},
|
||||
};
|
||||
controller.status = 'listening';
|
||||
controller._turns.narration = null;
|
||||
const event = (payload) =>
|
||||
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
|
||||
const created = (id) => event({ type: 'response.created', response: { id } });
|
||||
const done = (id) =>
|
||||
event({
|
||||
type: 'response.done',
|
||||
response: { id, status: 'completed', output: [] },
|
||||
});
|
||||
const creates = () =>
|
||||
sent.filter(
|
||||
(x) =>
|
||||
x.type === 'response.create' &&
|
||||
x.response?.metadata?.gev !== 'progress',
|
||||
);
|
||||
const texts = () =>
|
||||
sent
|
||||
.filter((x) => x.item?.role === 'user')
|
||||
.map((x) => x.item.content[0].text);
|
||||
return {
|
||||
controller,
|
||||
sent,
|
||||
executed,
|
||||
rejectedTexts,
|
||||
event,
|
||||
created,
|
||||
done,
|
||||
creates,
|
||||
texts,
|
||||
rejectResponses(value) {
|
||||
rejectResponse = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('rejected idle text schedules no answer, and a successful successor answers once', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.rejectedTexts.add('rejected');
|
||||
assert.equal(f.controller.sendTextCommand('rejected'), undefined);
|
||||
assert.equal(f.creates().length, 0);
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
assert.equal(f.controller.responseCreatePending, false);
|
||||
assert.equal(f.controller.sendTextCommand('successor'), undefined);
|
||||
assert.deepEqual(f.texts(), ['successor']);
|
||||
assert.equal(f.creates().length, 1);
|
||||
await f.created('successor');
|
||||
await f.done('successor');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text behind an active response cannot queue an answer', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.queueResponseCreate('Old tool follow-up');
|
||||
assert.equal(
|
||||
f.controller.pendingResponseInstructions,
|
||||
'Old tool follow-up',
|
||||
);
|
||||
const runningTool = new AbortController();
|
||||
f.controller._turns.activeToolAbortControllers.add(runningTool);
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
assert.equal(f.controller.pendingResponseInstructions, null);
|
||||
assert.equal(f.controller._turns.isSupersededResponse('active'), true);
|
||||
assert.equal(runningTool.signal.aborted, true);
|
||||
const staleCall = {
|
||||
type: 'response.function_call_arguments.done',
|
||||
response_id: 'active',
|
||||
call_id: 'stale-call',
|
||||
name: 'fly_to_location',
|
||||
arguments: '{"query":"Paris"}',
|
||||
};
|
||||
await f.event(staleCall);
|
||||
await f.event(staleCall);
|
||||
assert.deepEqual(f.executed, []);
|
||||
const outputs = f.sent.filter(
|
||||
(x) => x.item?.type === 'function_call_output',
|
||||
);
|
||||
assert.equal(outputs.length, 1);
|
||||
assert.equal(outputs[0].item.call_id, 'stale-call');
|
||||
assert.equal(JSON.parse(outputs[0].item.output).superseded, true);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text still prepares narration and pointer context before sending', () => {
|
||||
const steps = [];
|
||||
const f = fixture({
|
||||
pointer: {
|
||||
beginTurn() {
|
||||
steps.push('pointer');
|
||||
return null;
|
||||
},
|
||||
clear() {},
|
||||
},
|
||||
onUserSend() {
|
||||
steps.push('send');
|
||||
},
|
||||
});
|
||||
f.controller._turns.narration = {
|
||||
cancel() {
|
||||
steps.push('cancel');
|
||||
},
|
||||
userTurnEnded() {
|
||||
steps.push('user-turn-ended');
|
||||
},
|
||||
};
|
||||
try {
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']);
|
||||
assert.equal(f.creates().length, 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('earlier accepted deferred input survives a newer failed send and answers once', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.deepEqual(f.texts(), ['accepted']);
|
||||
assert.equal(f.controller.pendingUserTextResponse, true);
|
||||
assert.equal(f.creates().length, 0);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text preserves an outstanding create without adding deferred input', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.responseCreatePending, true);
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
await f.created('accepted');
|
||||
await f.done('accepted');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
for (const acceptedFirst of [false, true]) {
|
||||
test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
assert.equal(
|
||||
f.controller._turns.speakProgressLine('Checking the map.'),
|
||||
true,
|
||||
);
|
||||
await f.event({
|
||||
type: 'response.created',
|
||||
response: { id: 'progress', metadata: { gev: 'progress' } },
|
||||
});
|
||||
if (acceptedFirst) f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.pendingUserTextResponse, acceptedFirst);
|
||||
await f.event({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'progress',
|
||||
status: 'cancelled',
|
||||
metadata: { gev: 'progress' },
|
||||
},
|
||||
});
|
||||
assert.equal(f.creates().length, acceptedFirst ? 1 : 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('accepted text with a rejected response.create releases the pending latch for a successor', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.rejectResponses(true);
|
||||
f.controller.sendTextCommand('accepted without response');
|
||||
assert.equal(f.controller.responseCreatePending, false);
|
||||
assert.equal(f.creates().length, 0);
|
||||
f.rejectResponses(false);
|
||||
f.controller.sendTextCommand('successor');
|
||||
assert.deepEqual(f.texts(), ['accepted without response', 'successor']);
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('successful deferred typed burst still retains both items and coalesces one answer', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.sendTextCommand('first');
|
||||
f.controller.sendTextCommand('second');
|
||||
assert.deepEqual(f.texts(), ['first', 'second']);
|
||||
assert.equal(f.creates().length, 0);
|
||||
await f.done('active');
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
@@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
},
|
||||
async start(options) {
|
||||
if (disposed || !['idle', 'error'].includes(state)) return;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
const attempt = ++startAttempt;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
if (disposed || attempt !== startAttempt) return;
|
||||
try {
|
||||
await adapter.start(options);
|
||||
} catch (error) {
|
||||
@@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
stop(options = {}) {
|
||||
if (options.removeUi) return dispose();
|
||||
if (disposed) return;
|
||||
startAttempt++;
|
||||
const attempt = ++startAttempt;
|
||||
cancelActions();
|
||||
try {
|
||||
adapter.stop(options);
|
||||
} finally {
|
||||
if (!options.preserveStatus)
|
||||
if (
|
||||
!options.preserveStatus &&
|
||||
!disposed &&
|
||||
attempt === startAttempt &&
|
||||
state !== 'idle'
|
||||
)
|
||||
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
|
||||
}
|
||||
},
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { createRealtimeSession } from './realtimeSession.js';
|
||||
import { realtimeSessionEvent } from './realtimeEvents.js';
|
||||
import { createVoiceCommands } from './sessionCommands.js';
|
||||
import { describeUnsavedWork } from '../i18n/index.js';
|
||||
@@ -441,3 +442,189 @@ test('a late rejection cannot settle a newer turn card', async () => {
|
||||
);
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
function startReentrancyFixture() {
|
||||
let starts = 0;
|
||||
let stops = 0;
|
||||
const settings = [];
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({ ok: true }),
|
||||
createAdapter: ({ emit }) => ({
|
||||
async start(options) {
|
||||
starts++;
|
||||
settings.push(options);
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {
|
||||
stops++;
|
||||
},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
}),
|
||||
});
|
||||
return { session, settings, starts: () => starts, stops: () => stops };
|
||||
}
|
||||
|
||||
for (const action of ['stop', 'destroy']) {
|
||||
test(`initial connecting observer ${action} prevents adapter acquisition`, async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session[action]();
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
assert.equal(
|
||||
f.starts(),
|
||||
0,
|
||||
'cancelled attempt must not call adapter.start',
|
||||
);
|
||||
assert.equal(f.stops(), 1);
|
||||
assert.equal(f.session.state, 'idle');
|
||||
if (action === 'stop') {
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 1, 'later explicit Start remains usable');
|
||||
assert.equal(f.session.state, 'listening');
|
||||
}
|
||||
f.session.destroy();
|
||||
});
|
||||
}
|
||||
|
||||
test('preserveStatus Stop still prevents adapter acquisition', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session.stop({ preserveStatus: true });
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 0);
|
||||
assert.equal(
|
||||
f.session.state,
|
||||
'connecting',
|
||||
'preserved state is not ownership',
|
||||
);
|
||||
f.session.stop();
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 1);
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('Stop and explicit reentrant successor start acquire only for the successor', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
let successor;
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session.stop();
|
||||
successor = f.session.start();
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
await successor;
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('ordinary start forwards caller settings once and remains stoppable', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const settings = { pushToTalk: true };
|
||||
await f.session.start(settings);
|
||||
assert.equal(f.settings.length, 1);
|
||||
assert.equal(f.settings[0], settings);
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.stop();
|
||||
assert.equal(f.session.state, 'idle');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
|
||||
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
|
||||
let adapter;
|
||||
const calls = [];
|
||||
const events = [];
|
||||
const failure = new Error('Controlled stop failure');
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({}),
|
||||
createAdapter: ({ emit }) =>
|
||||
(adapter = {
|
||||
async start() {
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop(settings = {}) {
|
||||
calls.push({ receiver: this, settings });
|
||||
if (settings.removeUi) return;
|
||||
if (emitsIdle && !settings.preserveStatus)
|
||||
emit({ type: 'state', state: 'idle' });
|
||||
if (outcome === 'throw') throw failure;
|
||||
return 'adapter-only return';
|
||||
},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
}),
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
session.subscribe((event) => events.push(event));
|
||||
await session.start();
|
||||
events.length = 0;
|
||||
const settings =
|
||||
outcome === 'preserveStatus' ? { preserveStatus: true } : {};
|
||||
if (outcome === 'throw')
|
||||
assert.throws(
|
||||
() => session.stop(settings),
|
||||
(error) => error === failure,
|
||||
);
|
||||
else assert.equal(session.stop(settings), undefined);
|
||||
assert.equal(calls[0].receiver, adapter);
|
||||
assert.equal(calls[0].settings, settings);
|
||||
assert.equal(
|
||||
session.state,
|
||||
outcome === 'preserveStatus' ? 'listening' : 'idle',
|
||||
);
|
||||
assert.equal(
|
||||
events.filter(
|
||||
(event) => event.type === 'state' && event.state === 'idle',
|
||||
).length,
|
||||
outcome === 'preserveStatus' ? 0 : 1,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test('actual default OpenAI adapter stop publishes one common idle', (t) => {
|
||||
const events = [];
|
||||
const ui = {
|
||||
root: {
|
||||
dataset: {},
|
||||
querySelectorAll: () => [],
|
||||
classList: { remove() {} },
|
||||
remove() {},
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
};
|
||||
const runner = async () => ({});
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
createAdapter: (hooks) =>
|
||||
createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }),
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
session.subscribe((event) => events.push(event));
|
||||
// Exercise the real stop path without acquiring microphone/WebRTC resources.
|
||||
session.adapter.controller.setStatus('listening', 'Controlled active state');
|
||||
events.length = 0;
|
||||
session.stop();
|
||||
assert.equal(
|
||||
events.filter((event) => event.type === 'state' && event.state === 'idle')
|
||||
.length,
|
||||
1,
|
||||
);
|
||||
assert.equal(session.state, 'idle');
|
||||
assert.equal(session.adapter.controller.status, 'idle');
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { createVoiceControl, voiceStatusLabel } from './control.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
import { VOICE_PROVIDERS } from './providerCapabilities.js';
|
||||
import { msg, registerUnsavedWork, t } from '../i18n/index.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
@@ -25,12 +26,18 @@ export function createVoiceCommands({
|
||||
signal,
|
||||
debugSink,
|
||||
createControl = createVoiceControl,
|
||||
resetExisting = true,
|
||||
disposeRunner = true,
|
||||
provider,
|
||||
supportedProviders,
|
||||
onProviderChange,
|
||||
onSessionEvent,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
createReticle = createDefaultReticle,
|
||||
}) {
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
const ui = createControl({ reset: true });
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
@@ -50,6 +57,34 @@ export function createVoiceCommands({
|
||||
});
|
||||
const adapter = session.adapter;
|
||||
const capabilities = adapter.capabilities || {};
|
||||
const providerHandler = () => onProviderChange?.(ui.providerSelect.value);
|
||||
if (ui.providerSelect && onProviderChange) {
|
||||
if (supportedProviders && ui.providerSelect.replaceChildren) {
|
||||
const doc = ui.providerSelect.ownerDocument;
|
||||
ui.providerSelect.replaceChildren(
|
||||
...supportedProviders.map((name) => {
|
||||
const option = doc.createElement('option');
|
||||
option.value = name;
|
||||
option.textContent = VOICE_PROVIDERS[name].label;
|
||||
return option;
|
||||
}),
|
||||
);
|
||||
}
|
||||
ui.providerSelect.value = provider;
|
||||
ui.providerSelect.title = [
|
||||
t('Changing provider reconnects voice if it is on.'),
|
||||
capabilities.advisoryLimit,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
ui.providerSelect.addEventListener('change', providerHandler);
|
||||
ui.root.dataset.provider = provider;
|
||||
}
|
||||
const hasProviderChoice = Boolean(
|
||||
onProviderChange && supportedProviders?.length > 1,
|
||||
);
|
||||
if (ui.providerField) ui.providerField.hidden = !hasProviderChoice;
|
||||
ui.root.dataset.providerChoice = String(hasProviderChoice);
|
||||
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
|
||||
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
|
||||
if (!capabilities.pushToTalk) {
|
||||
@@ -60,6 +95,7 @@ export function createVoiceCommands({
|
||||
// Retain the existing controller's inspection surface for browser tools.
|
||||
const controls = adapter.controller || session;
|
||||
controls.session = session;
|
||||
controls.ui = ui;
|
||||
const updateStatus = session.subscribe((event) => {
|
||||
if (event.type !== 'state') return;
|
||||
ui.root.dataset.status = event.state;
|
||||
@@ -101,6 +137,10 @@ export function createVoiceCommands({
|
||||
});
|
||||
controls.pointer = pointer;
|
||||
controls.referents = referents;
|
||||
// Forward last: a host observer may synchronously replace or stop this mount.
|
||||
const forwardEvents = onSessionEvent
|
||||
? session.subscribe(onSessionEvent)
|
||||
: null;
|
||||
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
|
||||
card?.outline(event);
|
||||
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
|
||||
@@ -115,34 +155,26 @@ export function createVoiceCommands({
|
||||
else void session.start({ pushToTalk: false });
|
||||
};
|
||||
ui.button.addEventListener('click', buttonHandler);
|
||||
session.signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
unregisterUnsavedWork();
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
ui.root.remove();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
if (session.disposed) {
|
||||
let cleaned = false;
|
||||
const cleanup = () => {
|
||||
if (cleaned) return;
|
||||
cleaned = true;
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
ui.providerSelect?.removeEventListener('change', providerHandler);
|
||||
unregisterUnsavedWork();
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
forwardEvents?.();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
if (disposeRunner) runner.dispose?.();
|
||||
ui.root.remove();
|
||||
} else adapter.bindControls?.();
|
||||
};
|
||||
session.signal.addEventListener('abort', cleanup, { once: true });
|
||||
if (session.disposed) cleanup();
|
||||
else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
return controls;
|
||||
}
|
||||
Reference in new issue
Block a user