Merge branch 'main' into upstream/cctv-tarktee-refresh

This commit is contained in:
Said Turksever authored and GitHub committed 2026-10-19 22:24:44 +01:00
commit ca71b27791
62 files changed
+16223 -195

No files matched your search

+115
View File
@@ -0,0 +1,115 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { EventEmitter } from 'node:events';
import { PassThrough } from 'node:stream';
import { localProviderPlugins } from '../server/providers/local.js';
import { standaloneVoiceTools } from '../server/standalone/voiceTools.js';
import {
KEY_SETUP_KEYS,
keySetupStatus,
validateKeySetupUpdates,
upsertDotenvValues,
} from './keySetupCore.mjs';
test('local hosts opt into Gemini explicitly and retain the default plugin order', (t) => {
t.mock.method(globalThis, 'fetch', () => {
throw Error('construction must not fetch');
});
const defaults = localProviderPlugins().map((plugin) => plugin.name);
assert.ok(!defaults.includes('gemini-live-proxy'));
const enabled = localProviderPlugins({ gemini: {} }).map(
(plugin) => plugin.name,
);
assert.equal(
enabled.filter((name) => name === 'gemini-live-proxy').length,
1,
);
assert.deepEqual(
enabled.filter((name) => name !== 'gemini-live-proxy'),
defaults,
);
});
for (const hook of ['configureServer', 'configurePreviewServer']) {
test(`opted-in ${hook} locks the standalone catalog into the Gemini token`, async () => {
const tools = standaloneVoiceTools();
const calls = [];
const plugin = localProviderPlugins({
gemini: {
tools,
resolveApiKey: () => 'fixture-key',
resolveModel: () => 'gemini-3.8-live',
resolveHost: () => '',
resolveAllowedHosts: () => '',
resolveRateLimit: () => '0',
fetchImpl: async (url, options) => {
calls.push({ url, options });
return Response.json({ name: 'auth_tokens/fixture' });
},
},
}).find((entry) => entry.name === 'gemini-live-proxy');
let handler;
plugin[hook]({
middlewares: {
use(route, value) {
assert.equal(route, '/api/gemini/token');
handler = value;
},
},
});
const req = new PassThrough();
req.method = 'POST';
req.headers = {
host: 'localhost:5173',
origin: 'http://localhost:5173',
'content-type': 'application/json',
};
req.socket = { remoteAddress: '127.0.0.1' };
const res = new EventEmitter();
res.setHeader = () => {};
res.end = (body) => {
res.body = JSON.parse(body);
res.writableEnded = true;
};
req.end('{}');
await handler(req, res);
assert.equal(res.statusCode, 200);
assert.equal(calls.length, 1);
const locked = JSON.parse(calls[0].options.body).bidiGenerateContentSetup
.tools[0].functionDeclarations;
assert.deepEqual(
locked.map(({ name }) => name),
tools.map(({ name }) => name),
);
assert.deepEqual(
locked.map(({ parametersJsonSchema }) => parametersJsonSchema),
tools.map(({ parameters }) => parameters),
);
assert.ok(locked.every(({ behavior }) => behavior === 'BLOCKING'));
assert.deepEqual(res.body.config.tools[0].functionDeclarations, locked);
assert.ok(!JSON.stringify(res.body).includes('fixture-key'));
});
}
test('Gemini key setup uses the existing server-only status and save contract', () => {
const entry = KEY_SETUP_KEYS.find(({ id }) => id === 'gemini');
assert.deepEqual(entry.envVars, ['GEMINI_API_KEY']);
assert.ok(!entry.clientExposed);
assert.equal(
keySetupStatus({}).keys.find(({ id }) => id === 'gemini').set,
false,
);
const env = { GEMINI_API_KEY: 'fixture-private-key' };
const status = keySetupStatus(env);
assert.equal(status.keys.find(({ id }) => id === 'gemini').set, true);
assert.ok(!JSON.stringify(status).includes(env.GEMINI_API_KEY));
assert.equal(validateKeySetupUpdates(env).ok, true);
assert.match(
upsertDotenvValues('GEMINI_API_KEY=old\n', env),
/^GEMINI_API_KEY=fixture-private-key\n$/,
);
assert.equal(
validateKeySetupUpdates({ VITE_GEMINI_API_KEY: 'fixture-private-key' }).ok,
false,
);
});
File diff suppressed because it is too large. Load diff
+52 -1
View File
@@ -2890,5 +2890,56 @@
"USB work abandoned after a stalled transfer": "Trabajo USB cancelado tras una transferencia bloqueada",
"SDR decoder did not start; receiver stopped": "El decodificador SDR no arrancó; receptor detenido",
"SDR decoder stopped responding; receiver stopped": "El decodificador SDR dejó de responder; receptor detenido",
"SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedó atrás; receptor detenido"
"SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedó atrás; receptor detenido",
"Gemini returned an oversized audio chunk": "Gemini devolvió un fragmento de audio demasiado grande",
"Gemini returned an unsupported audio format": "Gemini devolvió un formato de audio no compatible",
"Gemini returned an invalid audio sample rate": "Gemini devolvió una frecuencia de muestreo de audio no válida",
"Gemini returned incomplete PCM audio": "Gemini devolvió audio PCM incompleto",
"This browser does not support Gemini microphone audio": "Este navegador no admite audio del micrófono para Gemini",
"Microphone audio processing failed": "Falló el procesamiento del audio del micrófono",
"Gemini audio playback fell behind. Reconnect to continue.": "La reproducción de audio de Gemini se ha retrasado. Vuelve a conectar para continuar.",
"Gemini token response did not include an ephemeral credential": "La respuesta de Gemini no incluyó una credencial temporal",
"Invalid Gemini connection endpoint": "El destino de conexión de Gemini no es válido",
"Voice command context is too large": "El contexto del comando de voz es demasiado grande",
"Gemini reliable transport exceeded its queue limit. Start voice again.": "La cola de envío fiable de Gemini superó su límite. Vuelve a iniciar la voz.",
"Gemini reliable transport did not drain. Start voice again.": "La cola de envío fiable de Gemini no se vació. Vuelve a iniciar la voz.",
"Gemini token request failed (HTTP {status})": "Falló la solicitud de credencial de Gemini (HTTP {status})",
"Gemini token response did not include a valid model": "La respuesta de Gemini no incluyó un modelo válido",
"Gemini token response did not include session settings": "La respuesta de Gemini no incluyó la configuración de la sesión",
"Gemini session settings do not match the input mode": "La configuración de la sesión de Gemini no coincide con el modo de entrada",
"Gemini could not submit this input. Start voice again.": "Gemini no pudo enviar esta entrada. Vuelve a iniciar la voz.",
"Connecting to Gemini": "Conectando con Gemini",
"Gemini returned an oversized message": "Gemini devolvió un mensaje demasiado grande",
"Gemini Live connection failed. Check network access, then start voice again.": "Falló la conexión con Gemini Live. Comprueba el acceso a la red y vuelve a iniciar la voz.",
"Gemini Live connection ended. Start voice again for a new conversation.": "La conexión con Gemini Live terminó. Vuelve a iniciar la voz para una nueva conversación.",
"Gemini Live could not start. Start voice again to request a fresh session.": "Gemini Live no pudo iniciarse. Vuelve a iniciar la voz para solicitar una nueva sesión.",
"Start voice again for a fresh token and a new conversation.": "Vuelve a iniciar la voz para obtener una credencial nueva y comenzar otra conversación.",
"Start voice again to request a fresh session credential.": "Vuelve a iniciar la voz para solicitar una nueva credencial de sesión.",
"Microphone permission was denied. Allow microphone access and try again.": "Se denegó el permiso del micrófono. Permite el acceso al micrófono e inténtalo de nuevo.",
"Gemini connection timed out. Try again.": "Se agotó el tiempo de conexión con Gemini. Inténtalo de nuevo.",
"Gemini voice could not start": "No se pudo iniciar la voz de Gemini",
"Gemini did not finish the interrupted reply. Start voice again.": "Gemini no terminó la respuesta interrumpida. Vuelve a iniciar la voz.",
"Gemini session expiring; finishing the current reply": "La sesión de Gemini va a caducar; terminando la respuesta actual",
"Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini rechazó la sesión de voz. Vuelve a iniciar la voz para solicitar una nueva sesión.",
"Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "Vuelve a iniciar la voz para obtener una nueva credencial de sesión. Si se repite, comprueba el modelo de Gemini y el acceso a la API.",
"Gemini action history is full. Start voice again for a fresh session.": "El historial de acciones de Gemini está lleno. Vuelve a iniciar la voz para una nueva sesión.",
"Gemini requested too many simultaneous actions": "Gemini solicitó demasiadas acciones simultáneas",
"Voice text command is too long": "El comando de voz escrito es demasiado largo",
"Gemini voice is not connected": "La voz de Gemini no está conectada",
"Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "La entrada interrumpida de Gemini superó el límite de búfer o de espera. Vuelve a iniciar la voz.",
"Gemini Live connection is expiring. Start voice again for a new conversation.": "La conexión con Gemini Live va a caducar. Vuelve a iniciar la voz para una nueva conversación.",
"Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "Las sesiones de voz de Gemini tienen un límite de tiempo. Cuando caduque la sesión, vuelve a iniciar la voz para una nueva conversación.",
"Gemini Live voice control": "Control por voz con Gemini Live",
"Voice provider": "Proveedor de voz",
"Changing provider reconnects voice if it is on.": "Cambiar de proveedor vuelve a conectar la voz si está activada.",
"Gemini voice could not connect. Try again.": "No se pudo conectar la voz de Gemini. Inténtalo de nuevo.",
"Gemini voice requires a same-origin app request": "La voz de Gemini requiere una solicitud del mismo origen que la aplicación",
"Add a Gemini API key in Provider Settings": "Añade una clave de API de Gemini en los ajustes de proveedores",
"The Gemini Live model configuration is invalid": "La configuración del modelo de Gemini Live no es válida",
"Gemini voice is starting too often. Try again shortly.": "La voz de Gemini se está iniciando con demasiada frecuencia. Vuelve a intentarlo en breve.",
"Gemini quota is unavailable. Try again later.": "La cuota de Gemini no está disponible. Inténtalo más tarde.",
"Gemini access was refused. Check the API key and model access.": "Se denegó el acceso a Gemini. Comprueba la clave de API y el acceso al modelo.",
"Gemini Live is unavailable for the configured model.": "Gemini Live no está disponible para el modelo configurado.",
"Token request is too large": "La solicitud de credencial es demasiado grande",
"Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío"
}
+52 -1
View File
@@ -2957,5 +2957,56 @@
"USB work abandoned after a stalled transfer": "Работа с USB прервана: передача данных зависла",
"SDR decoder did not start; receiver stopped": "Декодер SDR не запустился; приёмник остановлен",
"SDR decoder stopped responding; receiver stopped": "Декодер SDR перестал отвечать; приёмник остановлен",
"SDR decoder fell behind; receiver stopped": "Декодер SDR не успевает; приёмник остановлен"
"SDR decoder fell behind; receiver stopped": "Декодер SDR не успевает; приёмник остановлен",
"Gemini returned an oversized audio chunk": "Gemini вернул слишком большой фрагмент аудио",
"Gemini returned an unsupported audio format": "Gemini вернул неподдерживаемый формат аудио",
"Gemini returned an invalid audio sample rate": "Gemini вернул недопустимую частоту дискретизации аудио",
"Gemini returned incomplete PCM audio": "Gemini вернул неполные аудиоданные PCM",
"This browser does not support Gemini microphone audio": "Этот браузер не поддерживает звук с микрофона для Gemini",
"Microphone audio processing failed": "Не удалось обработать звук с микрофона",
"Gemini audio playback fell behind. Reconnect to continue.": "Воспроизведение звука Gemini отстаёт. Подключитесь заново, чтобы продолжить.",
"Gemini token response did not include an ephemeral credential": "В ответе Gemini нет временных учётных данных",
"Invalid Gemini connection endpoint": "Недопустимый адрес подключения к Gemini",
"Voice command context is too large": "Контекст голосовой команды слишком велик",
"Gemini reliable transport exceeded its queue limit. Start voice again.": "Очередь надёжной передачи Gemini превысила лимит. Запустите голосовой режим заново.",
"Gemini reliable transport did not drain. Start voice again.": "Очередь надёжной передачи Gemini не освободилась. Запустите голосовой режим заново.",
"Gemini token request failed (HTTP {status})": "Не удалось запросить токен Gemini (HTTP {status})",
"Gemini token response did not include a valid model": "В ответе Gemini нет допустимой модели",
"Gemini token response did not include session settings": "В ответе Gemini нет настроек сеанса",
"Gemini session settings do not match the input mode": "Настройки сеанса Gemini не соответствуют режиму ввода",
"Gemini could not submit this input. Start voice again.": "Gemini не удалось отправить эти данные. Запустите голосовой режим заново.",
"Connecting to Gemini": "Подключение к Gemini",
"Gemini returned an oversized message": "Gemini вернул слишком большое сообщение",
"Gemini Live connection failed. Check network access, then start voice again.": "Не удалось подключиться к Gemini Live. Проверьте доступ к сети и запустите голосовой режим заново.",
"Gemini Live connection ended. Start voice again for a new conversation.": "Соединение с Gemini Live завершено. Запустите голосовой режим заново для нового разговора.",
"Gemini Live could not start. Start voice again to request a fresh session.": "Не удалось запустить Gemini Live. Запустите голосовой режим заново, чтобы запросить новый сеанс.",
"Start voice again for a fresh token and a new conversation.": "Запустите голосовой режим заново, чтобы получить новый токен и начать новый разговор.",
"Start voice again to request a fresh session credential.": "Запустите голосовой режим заново, чтобы запросить новые учётные данные сеанса.",
"Microphone permission was denied. Allow microphone access and try again.": "Доступ к микрофону запрещён. Разрешите доступ и повторите попытку.",
"Gemini connection timed out. Try again.": "Время ожидания подключения к Gemini истекло. Повторите попытку.",
"Gemini voice could not start": "Не удалось запустить голосовой режим Gemini",
"Gemini did not finish the interrupted reply. Start voice again.": "Gemini не завершил прерванный ответ. Запустите голосовой режим заново.",
"Gemini session expiring; finishing the current reply": "Сеанс Gemini истекает; завершается текущий ответ",
"Gemini rejected the voice session. Start voice again to request a fresh session.": "Gemini отклонил голосовой сеанс. Запустите голосовой режим заново, чтобы запросить новый сеанс.",
"Start voice again for a fresh session credential. If it repeats, check Gemini model and API access.": "Запустите голосовой режим заново для получения новых учётных данных. Если ошибка повторится, проверьте модель Gemini и доступ к API.",
"Gemini action history is full. Start voice again for a fresh session.": "История действий Gemini заполнена. Запустите голосовой режим заново для нового сеанса.",
"Gemini requested too many simultaneous actions": "Gemini запросил слишком много одновременных действий",
"Voice text command is too long": "Текст голосовой команды слишком длинный",
"Gemini voice is not connected": "Голосовой режим Gemini не подключён",
"Gemini interrupted input exceeded its buffer or wait limit. Start voice again.": "Прерванный ввод Gemini превысил лимит буфера или ожидания. Запустите голосовой режим заново.",
"Gemini Live connection is expiring. Start voice again for a new conversation.": "Соединение с Gemini Live истекает. Запустите голосовой режим заново для нового разговора.",
"Gemini voice sessions have a time limit. When a session expires, start voice again for a new conversation.": "Время голосовых сеансов Gemini ограничено. Когда сеанс истечёт, запустите голосовой режим заново для нового разговора.",
"Gemini Live voice control": "Голосовое управление через Gemini Live",
"Voice provider": "Поставщик голосового сервиса",
"Changing provider reconnects voice if it is on.": "Смена поставщика переподключает голосовой режим, если он включён.",
"Gemini voice could not connect. Try again.": "Не удалось подключить голосовой режим Gemini. Повторите попытку.",
"Gemini voice requires a same-origin app request": "Голосовой режим Gemini требует запрос из того же источника, что и приложение",
"Add a Gemini API key in Provider Settings": "Добавьте ключ API Gemini в настройках поставщиков",
"The Gemini Live model configuration is invalid": "Недопустимая конфигурация модели Gemini Live",
"Gemini voice is starting too often. Try again shortly.": "Голосовой режим Gemini запускается слишком часто. Повторите попытку чуть позже.",
"Gemini quota is unavailable. Try again later.": "Квота Gemini недоступна. Повторите попытку позже.",
"Gemini access was refused. Check the API key and model access.": "Доступ к Gemini запрещён. Проверьте ключ API и доступ к модели.",
"Gemini Live is unavailable for the configured model.": "Gemini Live недоступен для выбранной модели.",
"Token request is too large": "Запрос токена слишком велик",
"Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON"
}
+9 -1
View File
@@ -19,7 +19,7 @@ import { hasProxySignals } from './localRequestGate.mjs';
/** Longest accepted key/token value. Real provider keys are all far shorter. */
export const KEY_SETUP_VALUE_LIMIT = 512;
/** Most env vars accepted in one save. The registry defines ten. */
/** Most env vars accepted in one save. */
export const KEY_SETUP_UPDATE_LIMIT = 16;
/** Header line written above keys the panel appends to a .env file. */
@@ -65,6 +65,14 @@ export const KEY_SETUP_KEYS = Object.freeze([
envVars: Object.freeze(['OPENAI_API_KEY']),
tier: 'metered',
}),
Object.freeze({
id: 'gemini',
title: 'GEMINI', // i18n-ignore: provider product name
unlocks: msg('Gemini Live voice control'),
getUrl: 'https://aistudio.google.com/apikey',
envVars: Object.freeze(['GEMINI_API_KEY']),
tier: 'metered',
}),
Object.freeze({
id: 'aisstream',
title: 'AISSTREAM', // i18n-ignore: provider product name
+44 -10
View File
@@ -3,6 +3,7 @@ import {
FRAME_SIGNATURE_H,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
PROJECTION_MATTE_COLOR,
PROJECTION_TEXTURE_SWAP_MS,
ACTIVE_FRAME_REFRESH_MS,
FRAME_ENDPOINT,
@@ -12,6 +13,37 @@ import {
PLACEHOLDER_REPAINT_MS,
} from './policy.js';
/**
* Contain-fits a source frame into the projection canvas: scaled to the
* largest size that fits, centered, with the margins left for the matte fill.
* Falls back to the full canvas when either dimension is missing or zero.
*
* @param {number} sourceW - Source width in pixels.
* @param {number} sourceH - Source height in pixels.
* @returns {{x:number,y:number,w:number,h:number}} Draw rectangle in canvas pixels.
*/
export function fitToProjection(sourceW, sourceH) {
if (!sourceW || !sourceH)
return {
x: 0,
y: 0,
w: PROJECTION_CANVAS_WIDTH,
h: PROJECTION_CANVAS_HEIGHT,
};
const scale = Math.min(
PROJECTION_CANVAS_WIDTH / sourceW,
PROJECTION_CANVAS_HEIGHT / sourceH,
);
const w = Math.round(sourceW * scale);
const h = Math.round(sourceH * scale);
return {
x: Math.round((PROJECTION_CANVAS_WIDTH - w) / 2),
y: Math.round((PROJECTION_CANVAS_HEIGHT - h) / 2),
w,
h,
};
}
export function createFrames({ state: layerState, services, parts, source }) {
/**
* FNV-1a over the RGB channels of a downsampled frame. Pure (takes the raw
@@ -182,7 +214,7 @@ export function createFrames({ state: layerState, services, parts, source }) {
ctx.clearRect(0, 0, w, h);
const g = ctx.createLinearGradient(0, 0, w, h);
g.addColorStop(0, '#05111a');
g.addColorStop(1, '#01070c');
g.addColorStop(1, PROJECTION_MATTE_COLOR);
ctx.fillStyle = g;
ctx.fillRect(0, 0, w, h);
@@ -320,7 +352,14 @@ export function createFrames({ state: layerState, services, parts, source }) {
// base color (planeMaterial color = WHITE, alpha .95) for the frame or
// two Cesium needs to rebind, which IS the periodic white flash from the
// owner field tests (2026-07-04 and 2026-07-30).
const signature = projectionFrameSignature(runtime);
const fit = fitToProjection(
runtime.image.naturalWidth,
runtime.image.naturalHeight,
);
// Pixels are hashed at a fixed size, so the fit carries the aspect.
const pixels = projectionFrameSignature(runtime);
const signature =
pixels === null ? null : `${pixels}:${fit.w}x${fit.h}`;
runtime.drawnImageStamp = runtime.imageStamp;
if (signature !== null && signature === runtime.lastFrameSignature) {
// Identical pixels — leave the canvas, and therefore the bound
@@ -328,19 +367,14 @@ export function createFrames({ state: layerState, services, parts, source }) {
return;
}
runtime.lastFrameSignature = signature;
runtime.ctx.clearRect(
0,
0,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
);
runtime.ctx.drawImage(
runtime.image,
runtime.ctx.fillStyle = PROJECTION_MATTE_COLOR;
runtime.ctx.fillRect(
0,
0,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
);
runtime.ctx.drawImage(runtime.image, fit.x, fit.y, fit.w, fit.h);
runtime.canvasStamp = (runtime.canvasStamp || 0) + 1;
runtime.lastPlaceholderPaintAt = 0;
}
+141
View File
@@ -0,0 +1,141 @@
// src/layers/cctv/frames.test.mjs
// Still-frame fit (#1201): stills draw at their source aspect, centered on the
// 1920x1080 projection canvas, instead of being stretched to fill it.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { createFrames, fitToProjection } from './frames.js';
import {
FRAME_SIGNATURE_W,
FRAME_SIGNATURE_H,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
PROJECTION_MATTE_COLOR,
} from './policy.js';
// The expected rectangles below are written for a 1920x1080 canvas.
assert.equal(PROJECTION_CANVAS_WIDTH, 1920);
assert.equal(PROJECTION_CANVAS_HEIGHT, 1080);
const FULL_CANVAS = {
x: 0,
y: 0,
w: PROJECTION_CANVAS_WIDTH,
h: PROJECTION_CANVAS_HEIGHT,
};
test('a 4:3 still is pillarboxed at full height', () => {
assert.deepEqual(fitToProjection(640, 480), {
x: 240,
y: 0,
w: 1440,
h: 1080,
});
});
test('a source wider than 16:9 is letterboxed at full width', () => {
assert.deepEqual(fitToProjection(3840, 1080), {
x: 0,
y: 270,
w: 1920,
h: 540,
});
});
test('a 16:9 still fills the canvas', () => {
assert.deepEqual(fitToProjection(1920, 1080), FULL_CANVAS);
});
test('a missing or zero dimension falls back to the full canvas', () => {
assert.deepEqual(fitToProjection(0, 480), FULL_CANVAS);
assert.deepEqual(fitToProjection(640, 0), FULL_CANVAS);
assert.deepEqual(fitToProjection(640, undefined), FULL_CANVAS);
assert.deepEqual(fitToProjection(undefined, undefined), FULL_CANVAS);
});
// Draw path: the fit reaches drawImage, and the unchanged-frame skip keys on the
// fitted size as well as the pixels (the 64x36 hash alone drops aspect).
function stillHarness() {
const calls = [];
const ctx = {
fillStyle: null,
fillRect: (...rect) => calls.push(['fillRect', ctx.fillStyle, ...rect]),
clearRect: () => {},
drawImage: (_image, ...rect) => calls.push(['drawImage', ...rect]),
};
// Every frame downsamples to the same pixels, so only the size differs.
const pixels = new Uint8ClampedArray(
FRAME_SIGNATURE_W * FRAME_SIGNATURE_H * 4,
).fill(7);
const runtime = {
mode: 'image',
ctx,
image: { naturalWidth: 0, naturalHeight: 0 },
imageReady: true,
// In flight, so refreshProjectionImage returns before touching image.src.
imageLoading: true,
imageStamp: 0,
drawnImageStamp: -1,
lastFrameSignature: null,
canvasStamp: 0,
signatureCtx: {
clearRect: () => {},
drawImage: () => {},
getImageData: () => ({ data: pixels }),
},
};
const record = { camera: { id: 'still' }, projection: runtime };
const { drawProjectionFrame } = createFrames({
state: { _healthById: new Map(), _activeCameraId: null },
services: {},
parts: {},
source: {},
});
const load = (width, height) => {
runtime.image = { naturalWidth: width, naturalHeight: height };
runtime.imageStamp += 1;
calls.length = 0;
drawProjectionFrame(record);
return calls.slice();
};
return { runtime, load };
}
test('a still is drawn into its fitted rectangle over a full matte', () => {
const { runtime, load } = stillHarness();
assert.deepEqual(load(640, 480), [
[
'fillRect',
PROJECTION_MATTE_COLOR,
0,
0,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
],
['drawImage', 240, 0, 1440, 1080],
]);
assert.equal(runtime.canvasStamp, 1);
});
test('same pixels at the same size skip the redraw', () => {
const { runtime, load } = stillHarness();
load(640, 480);
assert.deepEqual(load(640, 480), []);
assert.equal(runtime.canvasStamp, 1);
});
test('same pixels at a new aspect redraw at the new fit', () => {
const { runtime, load } = stillHarness();
load(640, 480);
assert.deepEqual(load(1920, 1080), [
[
'fillRect',
PROJECTION_MATTE_COLOR,
0,
0,
PROJECTION_CANVAS_WIDTH,
PROJECTION_CANVAS_HEIGHT,
],
['drawImage', 0, 0, 1920, 1080],
]);
assert.equal(runtime.canvasStamp, 2);
});
+3
View File
@@ -35,6 +35,9 @@ export const PROJECTION_CANVAS_WIDTH = 1920;
export const PROJECTION_CANVAS_HEIGHT = 1080;
/** Matte behind fitted stills; the placeholder gradient's dark end. */
export const PROJECTION_MATTE_COLOR = '#01070c';
// Downsample grid for the unchanged-frame signature (drawProjectionFrame).
// 64x36 keeps the 16:9 aspect and reads ~9 KB per check versus the 8.3 MB a
// full-resolution compare would touch.
+5 -1
View File
@@ -17,6 +17,10 @@ export function createStandaloneTools(options) {
},
},
...options,
voice: { toolCatalog: loadToolCatalog, ...options?.voice },
voice: {
supportedProviders: ['openai', 'gemini'],
toolCatalog: loadToolCatalog,
...options?.voice,
},
});
}
@@ -0,0 +1,364 @@
import assert from 'node:assert/strict';
import { createVoiceSession } from '../voice/session.js';
import { createGevActionRunner } from '../voice/gevActions.js';
import { createGeminiSession } from '../voice/geminiSession.js';
import { createNarrationScheduler } from '../voice/narration.js';
export const turnOfEventLoop = () =>
new Promise((resolve) => setImmediate(resolve));
function deferred() {
let resolve;
const promise = new Promise((done) => {
resolve = done;
});
return { promise, resolve };
}
function ownedClock() {
let now = 0;
let nextId = 0;
const timers = new Map();
return {
timers,
now: () => now,
setTimer(fn, delay = 0) {
const id = ++nextId;
timers.set(id, { fn, at: now + Math.max(0, delay) });
return id;
},
clearTimer: (id) => timers.delete(id),
advance(ms) {
const until = now + ms;
let callbacks = 0;
for (;;) {
const due = [...timers]
.filter(([, timer]) => timer.at <= until)
.sort((a, b) => a[1].at - b[1].at || a[0] - b[0])[0];
if (!due) break;
assert.ok(
++callbacks <= 1000,
'fixture timer queue must remain bounded',
);
timers.delete(due[0]);
now = due[1].at;
due[1].fn();
}
now = until;
},
};
}
/**
* Real common session -> real Gemini adapter -> real action runner.
* Only devices, transport, passive UI and wall-clock facilities are fakes.
* The optional result gate delays return AFTER the actual action commits.
*/
export function nativeIntegrationFixture(t) {
const clock = ownedClock();
const events = [];
const sockets = [];
const tracks = [];
const audios = [];
const requests = [];
const runnerCalls = [];
const runnerJobs = [];
const resultGates = new Map();
const acquisitions = { microphone: 0, audio: 0, token: 0, socket: 0 };
const state = { height: 1000, renders: 0, zooms: [], earconStarts: 0 };
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: state.height },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(metres) {
position.x -= metres;
state.height -= metres;
this.positionCartographic.height = state.height;
state.zooms.push({ direction: 'in', metres });
},
zoomOut(metres) {
position.x += metres;
state.height += metres;
this.positionCartographic.height = state.height;
state.zooms.push({ direction: 'out', metres });
},
},
scene: {
canvas: {
clientWidth: 800,
clientHeight: 600,
addEventListener() {},
removeEventListener() {},
},
requestRender() {
state.renders++;
},
},
};
const dataManager = { layers: new Map() };
const actualRunner = createGevActionRunner({
viewer,
dataManager,
styleManager: {},
});
const runner = (name, args, options) => {
runnerCalls.push({ name, args, options });
const job = (async () => {
// No fake action result: the camera and envelope come from production.
const result = await actualRunner(name, args, options);
const gate = resultGates.get(options.callId);
if (gate) {
gate.entered.resolve(result);
await gate.release.promise;
}
return result;
})();
runnerJobs.push(job);
return job;
};
class Socket {
constructor(url) {
acquisitions.socket++;
this.url = url;
this.readyState = 0;
this.bufferedAmount = 0;
this.sent = [];
sockets.push(this);
queueMicrotask(() => {
if (this.closed) return;
this.readyState = 1;
this.onopen?.();
});
}
send(raw) {
assert.equal(
this.readyState,
1,
'sending requires the owned open socket',
);
const message = JSON.parse(raw);
this.sent.push(message);
if (message.setup)
queueMicrotask(() => this.message({ setupComplete: {} }));
}
message(message) {
this.onmessage?.({ data: JSON.stringify(message) });
}
close() {
this.closed = true;
this.readyState = 3;
}
}
const session = createVoiceSession({
runner,
createAdapter: (hooks) =>
createGeminiSession({
...hooks,
ui: { root: { remove() {} } },
dataManager,
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
WebSocketClass: Socket,
getUserMedia: async () => {
acquisitions.microphone++;
const track = {
enabled: true,
readyState: 'live',
stopCalls: 0,
stop() {
this.stopCalls++;
this.readyState = 'ended';
},
};
tracks.push(track);
return { getTracks: () => [track], getAudioTracks: () => [track] };
},
fetchImpl: async (url, init) => {
acquisitions.token++;
requests.push({ url, init });
const manual = JSON.parse(init.body).inputMode === 'push-to-talk';
return Response.json({
token: 'auth_tokens/fixture-not-a-credential',
model: 'gemini-3.8-live',
websocketUrl:
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained',
config: {
generationConfig: { responseModalities: ['AUDIO'] },
...(manual
? {
realtimeInputConfig: {
automaticActivityDetection: { disabled: true },
},
}
: {}),
},
});
},
createInput: ({ readStream, operations }) => ({
...operations,
setMicrophoneEnabled(enabled) {
for (const track of readStream()?.getAudioTracks() || [])
track.enabled = enabled;
},
updateVoiceButtonLabel() {},
setVoiceSpeaker() {},
startVoiceVisualizer() {},
startAssistantVoiceVisualizer() {},
cancelPushToTalkHold() {},
stopVoiceVisualizer() {},
detachBindings() {},
bindPushToTalkShortcut() {},
resetSession() {
this.pushToTalkMode =
this.pushToTalkKeyHeld =
this.spaceKeyHeld =
false;
},
}),
createAudio: (callbacks) => {
acquisitions.audio++;
const audio = {
pending: false,
closed: false,
outputStream: {},
async initialize() {},
connectMicrophone(stream, enabled, metadata) {
this.stream = stream;
this.setMicrophoneEnabled(enabled, metadata);
},
setMicrophoneEnabled(enabled, metadata) {
this.enabled = enabled;
this.metadata = metadata;
},
play() {
this.pending = true;
},
clearPlayback() {
this.pending = false;
},
drain() {
this.pending = false;
callbacks.onDrain();
},
diagnostics: () => ({ outputChunks: 0 }),
async close() {
this.closed = true;
this.pending = false;
return true;
},
};
audios.push(audio);
return audio;
},
createNarration: (options) =>
createNarrationScheduler({
...options,
now: clock.now,
setTimer: clock.setTimer,
clearTimer: clock.clearTimer,
}),
createProgressEarcon: () => ({
available: () => true,
start() {
state.earconStarts++;
},
stop() {},
dispose() {},
}),
captureViewport: async () => null,
}),
});
session.subscribe((event) => events.push(event));
const controller = session.adapter.controller;
t.after(async () => {
session.destroy();
for (const gate of resultGates.values()) gate.release.resolve();
await Promise.allSettled(runnerJobs);
actualRunner.dispose();
assert.equal(clock.timers.size, 0, 'fixture must leave no owned timer');
assert.ok(audios.every((audio) => audio.closed));
assert.ok(tracks.every((track) => track.readyState === 'ended'));
assert.ok(sockets.every((socket) => socket.closed));
});
return {
session,
controller,
state,
events,
sockets,
tracks,
audios,
requests,
acquisitions,
runnerCalls,
clock,
holdResult(callId) {
const gate = { entered: deferred(), release: deferred() };
resultGates.set(callId, gate);
return { ready: gate.entered.promise, release: gate.release.resolve };
},
async start() {
await session.start();
assert.equal(session.state, 'listening');
assert.equal(controller.getDiagnostics().connected, true);
return sockets.at(-1);
},
activity(socket = sockets.at(-1)) {
socket.message({ voiceActivity: { type: 'ACTIVITY_START' } });
socket.message({ voiceActivity: { type: 'ACTIVITY_END' } });
},
tool(callId, socket = sockets.at(-1)) {
socket.message({
toolCall: {
functionCalls: [
{
id: callId,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
// A deterministic settlement boundary, not a fake action dispatcher.
return controller.toolQueue;
},
outputs(socket = sockets.at(-1)) {
return socket.sent.flatMap(
(message) => message.toolResponse?.functionResponses || [],
);
},
complete(socket = sockets.at(-1)) {
socket.message({ serverContent: { turnComplete: true } });
},
};
}
export function assertSuccessfulZoom(h, socket, callId, beforeHeight) {
assert.equal(h.state.height, beforeHeight * 0.75);
assert.ok(h.state.renders > 0);
const calls = h.events.filter(
(event) => event.type === 'action-call' && event.callId === callId,
);
const results = h.events.filter(
(event) => event.type === 'action-result' && event.callId === callId,
);
assert.equal(calls.length, 1);
assert.equal(results.length, 1);
assert.equal(calls[0].name, 'adjust_camera_zoom');
assert.deepEqual(calls[0].arguments, { direction: 'in', amount: 'little' });
assert.equal(results[0].result.ok, true);
assert.equal(results[0].result.action, 'adjust_camera_zoom');
assert.equal(results[0].result.beforeHeightM, Math.round(beforeHeight));
assert.equal(results[0].result.afterHeightM, Math.round(h.state.height));
const outputs = h.outputs(socket).filter((output) => output.id === callId);
assert.equal(outputs.length, 1);
assert.equal(outputs[0].name, 'adjust_camera_zoom');
assert.equal(outputs[0].response.ok, true);
assert.equal(outputs[0].response.action, 'adjust_camera_zoom');
assert.equal(outputs[0].response.afterHeightM, Math.round(h.state.height));
}
+251
View File
@@ -0,0 +1,251 @@
import { GeminiSessionController } from '../voice/geminiSession.js';
import { createNarrationScheduler } from '../voice/narration.js';
const endpoint =
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
const credential = {
token: 'auth_tokens/test',
model: 'gemini-3.8-live',
websocketUrl: endpoint,
config: { generationConfig: { responseModalities: ['AUDIO'] } },
};
const manualCredential = {
...credential,
config: {
...credential.config,
realtimeInputConfig: { automaticActivityDetection: { disabled: true } },
},
};
const tick = () => new Promise((resolve) => setImmediate(resolve));
function deferred() {
let resolve;
const promise = new Promise((done) => {
resolve = done;
});
return { promise, resolve };
}
function harness(overrides = {}) {
const { autoSetup = true, ...controllerOverrides } = overrides;
const events = [];
const sockets = [];
const media = [];
const tracks = [];
const captures = [];
const calls = [];
const requests = [];
const state = { playing: 0, ducked: false, stopped: 0 };
let clock = 0;
let nextTimer = 0;
const timers = new Map();
const earcon = { playing: false, starts: 0, disposals: 0 };
const advance = (ms) => {
const target = clock + ms;
for (;;) {
const due = [...timers]
.filter(([, timer]) => timer.at <= target)
.sort((a, b) => a[1].at - b[1].at)[0];
if (!due) break;
timers.delete(due[0]);
clock = due[1].at;
due[1].fn();
}
clock = target;
};
let radioControl;
const radio = {
setVoiceDucked(value) {
state.ducked = value;
},
pause() {},
subscribePlaybackControls(callback) {
radioControl = callback;
return () => {
radioControl = null;
};
},
async playForVoice() {
state.playing++;
return true;
},
stopPlayback() {
state.stopped++;
},
};
class Socket {
constructor(url) {
this.url = url;
this.readyState = 0;
this.bufferedAmount = 0;
this.sent = [];
sockets.push(this);
queueMicrotask(() => {
this.readyState = 1;
this.onopen?.();
});
}
send(raw) {
const value = JSON.parse(raw);
this.sent.push(value);
if (value.setup && autoSetup)
queueMicrotask(() => this.message({ setupComplete: {} }));
}
message(value) {
this.onmessage?.({ data: JSON.stringify(value) });
}
error() {
this.onerror?.();
}
remoteClose() {
this.readyState = 3;
this.onclose?.();
}
close() {
this.readyState = 3;
this.closed = true;
}
}
const makeStream = () => {
const track = {
enabled: true,
readyState: 'live',
stop() {
this.readyState = 'ended';
},
};
tracks.push(track);
return { getTracks: () => [track], getAudioTracks: () => [track] };
};
const c = new GeminiSessionController({
emit: (event) => events.push(event),
runAction: async (name, args, options) => {
calls.push({ name, args, options });
return { ok: true, action: name };
},
ui: { root: { remove() {} } },
radioLayer: radio,
now: () => clock,
setTimer: (fn, ms) => {
const id = ++nextTimer;
timers.set(id, { fn, at: clock + ms });
return id;
},
clearTimer: (id) => timers.delete(id),
WebSocketClass: Socket,
fetchImpl: async (_url, init) => {
requests.push(init);
return Response.json(
JSON.parse(init.body).inputMode === 'push-to-talk'
? manualCredential
: credential,
);
},
getUserMedia: async () => makeStream(),
createInput: ({ readStream, operations }) => ({
...operations,
setMicrophoneEnabled(enabled) {
for (const track of readStream()?.getAudioTracks() || [])
track.enabled = enabled;
},
updateVoiceButtonLabel() {},
setVoiceSpeaker() {},
startVoiceVisualizer() {},
startAssistantVoiceVisualizer() {},
cancelPushToTalkHold() {},
stopVoiceVisualizer() {},
detachBindings() {},
bindPushToTalkShortcut() {},
resetSession() {
this.pushToTalkMode = false;
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
},
}),
createNarration: (options) =>
createNarrationScheduler({
...options,
now: () => clock,
setTimer: (fn, ms) => {
const id = ++nextTimer;
timers.set(id, { fn, at: clock + ms });
return id;
},
clearTimer: (id) => timers.delete(id),
}),
createProgressEarcon: ({ isMicrophoneMuted }) => ({
available: isMicrophoneMuted,
start() {
earcon.playing = true;
earcon.starts++;
},
stop() {
earcon.playing = false;
},
dispose() {
this.stop();
earcon.disposals++;
},
}),
createAudio: (callbacks) => {
const audio = {
callbacks,
pending: false,
closed: false,
played: [],
async initialize() {},
connectMicrophone(stream, enabled, metadata) {
media.push({ stream, enabled });
this.connected = true;
this.setMicrophoneEnabled(enabled, metadata);
},
setMicrophoneEnabled(value, metadata) {
if (!this.connected) return;
if (value && !this.enabled) {
this.metadata = metadata;
callbacks.onAudioStart(metadata);
}
if (!value && this.enabled) callbacks.onAudioEnd(this.metadata);
this.enabled = value;
},
play(data, mime) {
this.pending = true;
this.played.push({ data, mime });
},
drain() {
this.pending = false;
callbacks.onDrain();
},
clearPlayback() {
this.pending = false;
this.cleared = true;
},
diagnostics() {
return { outputChunks: this.played.length };
},
close() {
this.closed = true;
},
};
captures.push(audio);
return audio;
},
...controllerOverrides,
});
return {
c,
sockets,
tracks,
makeStream,
events,
media,
captures,
calls,
requests,
state,
advance,
timers,
earcon,
radioControl: (value) => radioControl?.(value),
};
}
export { harness, tick, deferred };
+12 -1
View File
@@ -88,8 +88,19 @@ test('root config retains existing named exports and standalone provider order',
for (const [name, value] of Object.entries(providers))
assert.equal(compatibility[name], value, name);
const config = standaloneConfig({ mode: 'test' });
const standaloneProviders = config.plugins
.slice(3, -5)
.map((plugin) => plugin.name);
assert.equal(
standaloneProviders.filter((name) => name === 'gemini-live-proxy').length,
1,
);
assert.equal(
standaloneProviders.indexOf('gemini-live-proxy'),
standaloneProviders.indexOf('openai-realtime-proxy') + 1,
);
assert.deepEqual(
config.plugins.slice(3, -5).map((plugin) => plugin.name),
standaloneProviders.filter((name) => name !== 'gemini-live-proxy'),
providers.localProviderPlugins().map((plugin) => plugin.name),
);
assert.equal(config.plugins.at(-6).name, 'gev-key-setup');
+92
View File
@@ -140,3 +140,95 @@ html:not([lang='en'])
white-space: normal;
overflow-wrap: anywhere;
}
#gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: 0px;
}
/* Provider choice belongs to the existing voice control, not a separate panel. */
.gev-voice-provider-field {
position: absolute;
bottom: calc(100% + var(--gev-provider-lift, 0px));
z-index: 9;
left: 0;
right: 0;
display: grid;
gap: 4px;
padding: 8px 6px;
border: 1px solid var(--glass-border);
border-radius: 8px;
background: var(--bg-panel, #09121b);
color: var(--text-secondary);
font-family: var(--font-mono);
font-size: 0.5rem;
opacity: 0;
pointer-events: none;
}
.gev-voice-provider-field[hidden] {
display: none;
}
.gev-voice-provider-field select {
min-width: 0;
width: 100%;
min-height: 30px;
color: var(--text-primary);
background: #09121b;
border: 1px solid var(--glass-border);
border-radius: 4px;
font: inherit;
font-size: 0.65rem;
}
#gev-voice-control:hover .gev-voice-provider-field,
#gev-voice-control:focus-within .gev-voice-provider-field {
opacity: 1;
pointer-events: auto;
}
#command-dock
#gev-voice-control[data-provider-choice='true']
.gev-voice-help-tray,
#command-dock #gev-voice-control[data-provider-choice='true'] .gev-voice-card,
#command-dock
#gev-voice-control[data-provider-choice='true']
.gev-voice-error-tray {
bottom: calc(100% + 4.8rem + var(--gev-provider-lift));
}
#command-dock.dock-has-pinned-tray
#gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: calc(1.7rem + var(--dock-pinned-stack-height));
}
/* Pinned trays separate the chooser from its hover owner. Keep it reachable
across that gap without requiring keyboard focus to hold it open. */
#command-dock.dock-has-pinned-tray .gev-voice-provider-field {
opacity: 1;
pointer-events: auto;
}
@media (max-width: 900px) {
/* The attribution clearance also creates a gap for mouse users in a
narrow window, so the raised chooser must not depend on root hover. */
.gev-voice-provider-field {
opacity: 1;
pointer-events: auto;
}
#command-dock #gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: 3rem;
}
#command-dock.dock-has-pinned-tray
#gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: calc(3.5rem + var(--dock-pinned-stack-height));
}
}
@media (max-width: 720px) {
#command-dock #gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: calc(2vh + 2.5rem);
}
#command-dock.dock-has-pinned-tray
#gev-voice-control[data-provider-choice='true'] {
--gev-provider-lift: calc(2vh + 3rem + var(--dock-pinned-stack-height));
}
}
@media (hover: none) {
.gev-voice-provider-field {
opacity: 1;
pointer-events: auto;
}
}
+9 -1
View File
@@ -1,8 +1,16 @@
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createRealtimeSession } from './realtimeSession.js';
import { createGeminiSession } from './geminiSession.js';
import { createProviderCommands } from './providerCommands.js';
/** Default composition; callers may supply another session adapter factory. */
export function createVoiceCommands(options) {
export function createVoiceCommands(options = {}) {
// Generic hosts retain the existing default; standalone opts in explicitly.
if (!options.createSession && options.supportedProviders !== undefined)
return createProviderCommands(options, {
bind: bindVoiceCommands,
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
});
return bindVoiceCommands({
createSession: createRealtimeSession,
...options,
+6
View File
@@ -39,6 +39,10 @@ export function createVoiceControl({ reset = false } = {}) {
root.dataset.status = 'idle';
root.dataset.speaker = 'idle';
root.innerHTML = `
<label id="gev-voice-provider-field" class="gev-voice-provider-field" hidden>
<span>${t('Voice provider')}</span>
<select id="gev-voice-provider" aria-label="${t('Voice provider')}" title="${t('Changing provider reconnects voice if it is on.')}"></select>
</label>
<div class="gev-voice-heading">
<div class="gev-voice-kicker">${t('AI AGENT')}</div>
<div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
@@ -110,6 +114,8 @@ export function createVoiceControl({ reset = false } = {}) {
}
return {
root,
providerField: root.querySelector('#gev-voice-provider-field'),
providerSelect: root.querySelector('#gev-voice-provider'),
button: root.querySelector('#gev-voice-button'),
buttonLabel: root.querySelector('.gev-mic-label'),
status: root.querySelector('#gev-voice-status'),
+142
View File
@@ -0,0 +1,142 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
deferred,
tick,
} from '../testSupport/geminiNativeHarness.mjs';
function unreadResponse(bytes = null) {
const state = { reads: 0, cancelled: 0 };
const response = new Response(
new ReadableStream(
{
pull(controller) {
state.reads++;
if (bytes) controller.enqueue(bytes);
},
cancel() {
state.cancelled++;
},
},
{ highWaterMark: 0 },
),
);
return { response, state };
}
test('already aborted token request does not acquire or read a response', async (t) => {
let fetches = 0;
const h = harness({
fetchImpl: () => {
fetches++;
throw Error('must not fetch');
},
});
t.after(() => h.c.stop());
const abort = new AbortController();
abort.abort();
await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' });
assert.equal(fetches, 0);
});
for (const phase of [
'before response',
'between resolution and continuation',
]) {
test(`aborted token acquisition closes the unread body ${phase}`, async (t) => {
const gate = deferred();
const h = harness({ fetchImpl: () => gate.promise });
t.after(() => h.c.stop());
const abort = new AbortController();
const { response, state } = unreadResponse();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
if (phase === 'before response') abort.abort();
gate.resolve(response);
if (phase !== 'before response') queueMicrotask(() => abort.abort());
await result;
await tick();
assert.deepEqual(state, { reads: 0, cancelled: 1 });
});
}
test('token body cap cancels a native response stream', async (t) => {
const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1));
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
await assert.rejects(
h.c.requestToken(new AbortController().signal),
/too large|exceed|limit/i,
);
assert.equal(state.cancelled, 1);
assert.equal(state.reads, 1);
});
test('token body read ends promptly on caller abort and releases the reader', async (t) => {
const { response, state } = unreadResponse();
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
const abort = new AbortController();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
await tick();
assert.equal(state.reads, 1);
abort.abort();
await result;
assert.equal(state.cancelled, 1);
assert.equal(response.body.locked, false);
});
test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => {
const gate = deferred();
const h = harness({ getUserMedia: () => gate.promise });
t.after(() => h.c.stop());
const stream = h.makeStream();
const start = h.c.start();
gate.resolve(stream);
queueMicrotask(() => h.c.stop());
await start;
await tick();
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.requests.length, 0);
assert.equal(h.sockets.length, 0);
assert.equal(h.captures[0].closed, true);
h.c.getUserMedia = async () => h.makeStream();
await h.c.start();
assert.equal(h.c.status, 'listening');
assert.equal(h.tracks[1].readyState, 'live');
assert.equal(h.captures[1].closed, false);
});
test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => {
const gate = deferred();
const h = harness();
t.after(() => h.c.stop());
const createAudio = h.c.createAudio;
let first = true;
h.c.createAudio = (options) => {
const audio = createAudio(options);
if (first) {
first = false;
audio.initialize = () => gate.promise;
audio.connectMicrophone = () =>
assert.fail('retired initialization must not connect');
}
return audio;
};
const oldStart = h.c.start();
await tick();
h.c.stop();
await h.c.start();
gate.resolve();
await oldStart;
await tick();
assert.equal(h.c.status, 'listening');
assert.equal(h.captures[0].closed, true);
assert.equal(h.captures[1].closed, false);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.tracks[1].readyState, 'live');
});
+380
View File
@@ -0,0 +1,380 @@
import { msg } from '../i18n/marker.js';
const MAX_AUDIO_BYTES = 2 * 1024 * 1024;
const MAX_QUEUED_SECONDS = 60;
const MAX_QUEUED_SOURCES = 256;
const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz.
const PTT_ENERGY_FLOOR = 0.002;
const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4;
/** Admit manual input only after sustained energy; not an utterance detector. */
function pttCapture(captureId) {
return {
captureId,
admitted: false,
buffers: [],
bytes: 0,
samples: 0,
energy: 0,
aboveFloor: 0,
voicedWindows: 0,
};
}
function hasPttEnergy(state, pcm) {
for (let index = 0; index + 1 < pcm.byteLength; index += 2) {
const sample = pcm.getInt16(index, true) / 32768;
state.energy += sample * sample;
if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++;
if (++state.samples !== PTT_WINDOW_SAMPLES) continue;
// Requiring occupancy as well as RMS rejects isolated high-peak key clicks.
const voiced =
state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 &&
state.aboveFloor >= PTT_WINDOW_SAMPLES / 8;
state.voicedWindows = voiced ? state.voicedWindows + 1 : 0;
state.samples = state.energy = state.aboveFloor = 0;
if (state.voicedWindows >= 2) return true;
}
return false;
}
export function pcmBytesToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let offset = 0; offset < bytes.length; offset += 8192) {
binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
}
return btoa(binary);
}
export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') {
if (
typeof data !== 'string' ||
data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3)
)
throw new Error(msg('Gemini returned an oversized audio chunk'));
const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType);
if (!match)
throw new Error(msg('Gemini returned an unsupported audio format'));
const sampleRate = Number(match[1] || 24000);
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000)
throw new Error(msg('Gemini returned an invalid audio sample rate'));
const binary = atob(data);
if (binary.length > MAX_AUDIO_BYTES)
throw new Error(msg('Gemini returned an oversized audio chunk'));
if (!binary.length || binary.length % 2)
throw new Error(msg('Gemini returned incomplete PCM audio'));
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index++)
bytes[index] = binary.charCodeAt(index);
const view = new DataView(bytes.buffer);
const samples = new Float32Array(bytes.length / 2);
for (let index = 0; index < samples.length; index++)
samples[index] = view.getInt16(index * 2, true) / 32768;
return { samples, sampleRate };
}
/** Own capture and playback independently from the Live protocol. */
export function createGeminiAudio({
pushToTalk = false,
onAudioStart,
onAudio,
onAudioEnd,
onAudioDiscard,
onDrain,
onError,
AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext,
AudioWorkletNodeClass = globalThis.AudioWorkletNode,
} = {}) {
if (!AudioContextClass || !AudioWorkletNodeClass)
throw new Error(
msg('This browser does not support Gemini microphone audio'),
);
let captureContext;
let playbackContext;
let output;
let meter;
let resumed;
const closeContext = (context) => {
try {
return Promise.resolve(context?.close());
} catch (error) {
return Promise.reject(error);
}
};
try {
// Capture at the device rate; the worklet explicitly resamples to 16 kHz.
captureContext = new AudioContextClass();
playbackContext = new AudioContextClass({ sampleRate: 24000 });
// Resume both synchronously within the mic gesture, before any awaits.
const resume = (context) => {
try {
return Promise.resolve(context.resume());
} catch (error) {
return Promise.reject(error);
}
};
resumed = Promise.all([resume(captureContext), resume(playbackContext)]);
// Permission may remain pending before initialize observes this failure.
void resumed.catch(() => {});
output = playbackContext.createGain();
meter = playbackContext.createMediaStreamDestination();
output.connect(playbackContext.destination);
output.connect(meter);
} catch (error) {
output?.disconnect();
meter?.disconnect();
meter?.stream.getTracks?.().forEach((track) => track.stop());
void Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]);
throw error;
}
let closed = false;
let closeResult;
let capture = null;
let microphone = null;
let mute = null;
let nextTime = 0;
let inputChunks = 0;
let inputPeak = 0;
let nonSilentInputChunks = 0;
let outputChunks = 0;
let outputSamples = 0;
let completedOutputSamples = 0;
let discardedOutputSamples = 0;
let pendingPtt = null;
let latestPttCaptureId = 0;
let suppressedPttCaptures = 0;
let admittedPttCaptures = 0;
const sources = new Map();
const emitAudio = (buffer, metadata) =>
onAudio?.(
{ data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' },
metadata,
);
function clearPlayback() {
const oldSources = [...sources];
sources.clear();
nextTime = playbackContext.currentTime;
for (const [source, sampleCount] of oldSources) {
discardedOutputSamples += sampleCount;
source.onended = null;
try {
source.stop();
} catch {
/* Already ended. */
}
source.disconnect();
}
}
return {
outputStream: meter.stream,
get pending() {
return sources.size > 0;
},
async initialize(signal) {
try {
await Promise.all([
resumed,
captureContext.audioWorklet.addModule(
new URL('./geminiCaptureWorklet.js', import.meta.url),
),
]);
signal?.throwIfAborted();
if (closed) throw new DOMException('Voice stopped', 'AbortError');
} catch (error) {
await this.close();
throw error;
}
},
connectMicrophone(stream, enabled, metadata) {
if (closed) throw new DOMException('Voice stopped', 'AbortError');
capture = new AudioWorkletNodeClass(
captureContext,
'gev-gemini-capture',
{
numberOfInputs: 1,
numberOfOutputs: 1,
outputChannelCount: [1],
},
);
capture.port.onmessage = ({ data }) => {
if (closed) return;
// Capture ownership is transport metadata, never part of Live audio.
if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0)
return;
const metadata = { captureId: data.captureId };
try {
if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) {
inputChunks++;
const pcm = new DataView(data.buffer);
let peak = 0;
for (let index = 0; index + 1 < pcm.byteLength; index += 2)
peak = Math.max(
peak,
Math.abs(pcm.getInt16(index, true)) / 32768,
);
inputPeak = Math.max(inputPeak, peak);
if (peak >= 0.01) nonSilentInputChunks++;
if (!pushToTalk) emitAudio(data.buffer, metadata);
else {
const state = pendingPtt;
if (state?.captureId !== data.captureId) return;
if (state.admitted) emitAudio(data.buffer, metadata);
else {
state.buffers.push(data.buffer);
state.bytes += data.buffer.byteLength;
if (hasPttEnergy(state, pcm)) {
state.admitted = true;
admittedPttCaptures++;
// Context and activityStart are gated together with PCM.
onAudioStart?.(metadata);
for (const buffer of state.buffers) {
if (closed || pendingPtt !== state) break;
emitAudio(buffer, metadata);
}
state.buffers = [];
state.bytes = 0;
} else {
while (state.bytes > PTT_PRE_ROLL_BYTES) {
state.bytes -= state.buffers.shift().byteLength;
}
}
}
}
} else if (data?.type === 'start') {
if (!pushToTalk) onAudioStart?.(metadata);
else if (data.captureId > latestPttCaptureId) {
latestPttCaptureId = data.captureId;
pendingPtt = pttCapture(data.captureId);
}
} else if (data?.type === 'end') {
if (!pushToTalk) onAudioEnd?.(metadata);
else if (pendingPtt?.captureId === data.captureId) {
const admitted = pendingPtt.admitted;
pendingPtt = null;
if (admitted) onAudioEnd?.(metadata);
else {
suppressedPttCaptures++;
onAudioDiscard?.(metadata);
}
}
}
} catch (error) {
onError?.(error);
}
};
capture.onprocessorerror = () => {
if (!closed)
onError?.(new Error(msg('Microphone audio processing failed')));
};
microphone = captureContext.createMediaStreamSource(stream);
mute = captureContext.createGain();
mute.gain.value = 0;
capture.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
microphone.connect(capture);
capture.connect(mute);
mute.connect(captureContext.destination);
},
setMicrophoneEnabled(enabled, metadata) {
if (closed) return;
capture?.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
},
play(data, mimeType) {
if (closed) return;
// Tiny chunks can exhaust source nodes long before the duration limit.
if (sources.size >= MAX_QUEUED_SOURCES)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const { samples, sampleRate } = decodeGeminiPcm(data, mimeType);
const start = Math.max(playbackContext.currentTime + 0.02, nextTime);
if (
start + samples.length / sampleRate - playbackContext.currentTime >
MAX_QUEUED_SECONDS
)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const buffer = playbackContext.createBuffer(
1,
samples.length,
sampleRate,
);
buffer.copyToChannel(samples, 0);
const source = playbackContext.createBufferSource();
source.buffer = buffer;
source.connect(output);
sources.set(source, samples.length);
source.onended = () => {
if (!sources.delete(source)) return;
completedOutputSamples += samples.length;
source.disconnect();
if (!closed && !sources.size) onDrain?.();
};
nextTime = start + buffer.duration;
outputChunks++;
outputSamples += samples.length;
source.start(start);
},
clearPlayback,
diagnostics() {
return {
inputChunks,
inputPeak,
nonSilentInputChunks,
outputChunks,
outputSamples,
completedOutputSamples,
discardedOutputSamples,
queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime),
contextState:
captureContext.state === playbackContext.state
? captureContext.state
: 'suspended',
captureContextState: captureContext.state,
playbackContextState: playbackContext.state,
captureSampleRate: captureContext.sampleRate,
playbackSampleRate: playbackContext.sampleRate,
inputSampleRate: 16000,
suppressedPttCaptures,
admittedPttCaptures,
pttBufferedBytes: pendingPtt?.bytes || 0,
};
},
close() {
if (closed) return closeResult;
closed = true;
pendingPtt = null;
clearPlayback();
if (capture) {
capture.port.onmessage = null;
capture.onprocessorerror = null;
capture.port.close();
capture.disconnect();
}
microphone?.disconnect();
mute?.disconnect();
output.disconnect();
meter.disconnect();
meter.stream.getTracks?.().forEach((track) => track.stop());
closeResult = Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]).then((results) =>
results.every((result) => result.status === 'fulfilled'),
);
return closeResult;
},
};
}
File diff suppressed because it is too large. Load diff
+791
View File
@@ -0,0 +1,791 @@
// Native automatic activity, reply ownership and expiry lifecycle regressions.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
const vad = (ws, type) => ws.message({ voiceActivity: { type } });
const onset = (ws) => {
vad(ws, 'ACTIVITY_START');
vad(ws, 'ACTIVITY_END');
};
const split = (ws) => {
onset(ws);
onset(ws);
};
const audio = {
serverContent: {
modelTurn: {
parts: [
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
],
},
},
};
const done = (ws) => ws.message({ serverContent: { turnComplete: true } });
const tools = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'out', amount: 'little' },
},
],
},
});
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
function close(h) {
h.c.stop();
assert.equal(h.timers.size, 0, 'owned timers released');
}
test('N1 VAD-only split admits the first real answer and completes after playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message(audio);
ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } });
done(ws);
assert.equal(h.captures[0].played.length, 1);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => {
const h = harness();
try {
await h.c.start();
split(h.sockets[0]);
h.advance(5001);
assert.equal(h.c.status, 'listening');
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.retiredBoundaryWait, null);
} finally {
close(h);
}
});
test('N3 server interruption before output still fences until ordered turnComplete', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'retired-before-boundary');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
assert.equal(responses(ws).length, 0);
ws.message({
serverContent: { ...audio.serverContent, turnComplete: true },
});
assert.equal(
h.captures[0].played.length,
0,
'coalesced retired content is discarded',
);
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
close(h);
}
});
test('N4 generationComplete is server evidence and next onset retires it', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
assert.ok(h.c.retiredBoundaryWait);
ws.message(audio);
tools(ws, 'old-generation');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
done(ws);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
assert.ok(h.c.goAway);
assert.equal(ws.closed, undefined);
h.advance(5001);
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
assert.equal(
h.c.isActive(),
true,
'pending actual PCM drains before expiry',
);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
assert.equal(ws.closed, true);
} finally {
close(h);
}
});
test('N6 first server tool after VAD-only split executes exactly once', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
assert.equal(responses(ws).at(-1).response.ok, true);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
} finally {
close(h);
}
});
test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-old');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 1);
assert.equal(h.calls.length, 0);
ws.message({ serverContent: { interrupted: true } });
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
ws.message(audio);
assert.equal(h.captures[0].played.length, 2);
} finally {
close(h);
}
});
test('C2 first server tool in flight is aborted and never answered after new onset', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'old-running');
await tick();
assert.equal(entered.length, 1);
onset(ws);
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0);
assert.equal(h.c.retiredReplyPending, true);
} finally {
gate.resolve();
close(h);
}
});
test('C5 typed request before first server output still retires on next onset', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
assert.equal(h.c.sendTextCommand('Zoom in'), true);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-typed');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
} finally {
close(h);
}
});
test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const oldOwner = h.c.automaticActivityOwner;
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
assert.notEqual(successor, oldOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const owner = h.c.automaticActivityOwner;
ws.message(audio);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredAutomaticActivityOwner, owner);
done(ws);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), false, 'no invented successor wait');
} finally {
close(h);
}
});
test('N9 coalesced old completion and new VAD preserves the new owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
vad(ws, 'ACTIVITY_END');
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.c.automaticTurnOpen, true);
assert.ok(h.c.automaticActivityOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('N10 stop, restart and destroy release automatic owner identities and timers', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
const first = h.c.automaticActivityOwner;
h.c.stop();
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.c.automaticActivitySequence, 0);
await h.c.start();
onset(h.sockets[1]);
assert.notEqual(h.c.automaticActivityOwner, first);
assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch);
h.c.stop({ removeUi: true });
assert.equal(h.c.disposed, true);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom in'), true);
assert.equal(h.c.sendTextCommand('Zoom out'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
const before = ws.sent.filter((x) => x.realtimeInput?.audio).length;
const metadata = { captureId: h.c.captureOwner.captureId };
h.captures[0].callbacks.onAudio(
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
metadata,
);
assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before);
assert.equal(
h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio),
false,
);
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, {
nonPttOwner: 1,
closedActivity: 0,
});
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
done(ws);
assert.equal(
ws.sent.filter((x) => x.realtimeInput?.audio).length,
before,
'no wrong-owner PCM replay after boundaries',
);
} finally {
close(h);
}
});
test('N12 goAway before old completion still waits for the newer automatic owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(
h.c.isActive(),
true,
'old boundary during goAway cannot end successor wait',
);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
for (const interruptedBeforeExpiry of [false, true])
test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
if (interruptedBeforeExpiry)
ws.message({ serverContent: { interrupted: true } });
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(
h.c.isActive(),
false,
'no successor finishes at its actual boundary',
);
} finally {
close(h);
}
});
test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'expiring-old-tool');
await tick();
assert.equal(entered.length, 1);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(
responses(ws).filter((x) => x.id === 'expiring-old-tool').length,
0,
);
assert.equal(h.c.isActive(), false);
} finally {
gate.resolve();
close(h);
}
});
test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ goAway: { timeLeft: '30s' } });
const turn = h.c.turn,
sent = ws.sent.length;
assert.equal(h.c.bargeIn(), false);
assert.equal(h.c.sendTextCommand('Zoom out'), false);
h.c.interrupt('push-to-talk');
assert.equal(h.c.turn, turn);
assert.equal(ws.sent.length, sent);
assert.equal(
h.tracks.every((t) => !t.enabled),
true,
);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.isActive(), false);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
ws.message(audio);
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.replyPending, true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(
h.c.replyPending,
true,
'old boundary cannot clear submitted successor',
);
assert.equal(h.c.sendTextCommand('Zoom out second request'), true);
assert.ok(h.c.currentTypedInput());
const played = a.played.length;
ws.message(audio);
tools(ws, 'late-ptt-predecessor');
await h.c.toolQueue;
assert.equal(a.played.length, played);
assert.equal(
h.calls.length,
0,
'no old tool under newer local typed input',
);
done(ws);
tools(ws, 'current-typed');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput().text, 'Zoom out second request');
} finally {
close(h);
}
});
for (const expiryBeforeBoundary of [false, true])
test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
const typed = h.c.currentTypedInput();
assert.ok(typed);
assert.equal(h.c.replyPending, true);
if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.replyPending, true);
if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true, 'submitted text must finish');
tools(ws, 'valid-successor');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput(), typed);
ws.message(audio);
done(ws);
assert.equal(h.c.isActive(), true);
h.captures[0].drain();
assert.equal(h.c.isActive(), false);
} finally {
close(h);
}
});
for (const repeat of [false, true])
test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out A'), true);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.sendTextCommand('Zoom out B'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
assert.ok(h.c.currentTypedInput());
onset(ws);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.deferredInput, null);
if (repeat) {
ws.message({ goAway: { timeLeft: '20s' } });
ws.message({ goAway: { timeLeft: '10s' } });
}
assert.equal(h.c.goAway.supersededSuccessor, true);
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
tools(ws, 'superseded-A');
await h.c.toolQueue;
assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority');
assert.equal(
h.c.isActive(),
false,
'ordered boundary closes superseded forwarded predecessor',
);
} finally {
close(h);
}
});
test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
h.sockets[0].message({ goAway: { timeLeft: '30s' } });
h.advance(29999);
assert.equal(h.c.isActive(), true);
h.advance(1);
assert.equal(h.c.isActive(), false);
assert.equal(h.timers.size, 0);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { turnComplete: true },
});
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
assert.equal(h.c.replyPending, true);
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.replyPending, false);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.retiredBoundaryWait, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.userTurnPending, true);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, false);
assert.equal(a.played.length, 0);
assert.equal(h.c.userTurnPending, false);
} finally {
close(h);
}
});
test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'pending-terminal-tool');
await tick();
assert.equal(entered.length, 1);
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.userTurnPending, false);
gate.resolve();
await h.c.toolQueue;
h.c.finishPlayback();
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up');
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
gate.resolve();
close(h);
}
});
test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
h.captures[0].drain();
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, true);
} finally {
close(h);
}
});
test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
assert.equal(h.c.userTurnPending, true);
done(ws);
assert.equal(h.c.userTurnPending, true);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
} finally {
close(h);
}
});
test('T-C7 reentrant silent completion observer keeps new input pending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const emit = h.c.emit.bind(h.c);
h.c.emit = (event) => {
emit(event);
if (event.type === 'completion')
h.c.sendTextCommand('Zoom out reentrant');
};
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.userTurnPending, true);
assert.ok(h.c.currentTypedInput());
assert.equal(h.c.modelComplete, false);
} finally {
close(h);
}
});
+125
View File
@@ -0,0 +1,125 @@
const OUTPUT_RATE = 16000;
const FILTER_TAPS = 64;
const FILTER_PHASES = 128;
// A causal, windowed-sinc low-pass filter avoids folding device frequencies
// above 8 kHz into speech. Precompute phases; process allocates no filter data.
function resamplingFilters(inputRate) {
const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate);
const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES);
for (let phase = 0; phase < FILTER_PHASES; phase++) {
let total = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES;
const sinc =
distance === 0
? 2 * cutoff
: Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance);
const window =
0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1));
const weight = sinc * window;
filters[phase * FILTER_TAPS + tap] = weight;
total += weight;
}
for (let tap = 0; tap < FILTER_TAPS; tap++)
filters[phase * FILTER_TAPS + tap] /= total;
}
return filters;
}
/** PCM microphone capture. Only explicitly enabled frames leave this processor. */
class GeminiCaptureProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.enabled = false;
this.captureId = null;
this.samples = new Int16Array(1024);
this.length = 0;
this.inputRate = sampleRate;
this.filters =
this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate);
this.history = new Float32Array(FILTER_TAPS);
this.resetResampler();
this.port.onmessage = ({ data }) => {
if (data?.enabled === true) {
if (
this.enabled ||
!Number.isSafeInteger(data.captureId) ||
data.captureId <= 0
)
return;
this.captureId = data.captureId;
this.resetResampler();
this.enabled = true;
this.port.postMessage({ type: 'start', captureId: this.captureId });
} else if (
data?.enabled === false &&
this.enabled &&
data.captureId === this.captureId
) {
this.flush();
this.port.postMessage({ type: 'end', captureId: this.captureId });
this.enabled = false;
this.captureId = null;
this.length = 0;
// Do not drain filter tails into a later physical microphone capture.
this.resetResampler();
}
};
}
resetResampler() {
this.phase = 0;
this.historyIndex = 0;
this.history.fill(0);
}
appendSample(sample) {
const value = Math.max(-1, Math.min(1, sample));
this.samples[this.length++] = Math.round(
value * (value < 0 ? 32768 : 32767),
);
if (this.length === this.samples.length) this.flush();
}
flush() {
if (!this.length) return;
const buffer = new ArrayBuffer(this.length * 2);
const view = new DataView(buffer);
for (let i = 0; i < this.length; i++)
view.setInt16(i * 2, this.samples[i], true);
this.port.postMessage(
{ type: 'audio', buffer, captureId: this.captureId },
[buffer],
);
this.length = 0;
}
process(inputs) {
if (!this.enabled) return true;
const channel = inputs[0]?.[0];
if (!channel) return true;
for (const sample of channel) {
if (!this.filters) {
this.appendSample(sample);
continue;
}
this.history[this.historyIndex] = sample;
this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS;
this.phase += OUTPUT_RATE;
while (this.phase >= this.inputRate) {
this.phase -= this.inputRate;
const phase = Math.min(
FILTER_PHASES - 1,
Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE),
);
let value = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const index =
(this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS;
value +=
this.history[index] * this.filters[phase * FILTER_TAPS + tap];
}
this.appendSample(value);
}
}
return true;
}
}
registerProcessor('gev-gemini-capture', GeminiCaptureProcessor);
+285
View File
@@ -0,0 +1,285 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_PCM = 'BAAFAA==';
const TEXT = 'Zoom in for the owned typed successor';
function exposeCapture(h) {
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
audio.captureCallbacks = callbacks;
return audio;
};
}
async function startManual(h) {
await h.session.start({ pushToTalk: true });
assert.equal(h.session.state, 'listening');
assert.equal(h.controller.input.pushToTalkMode, true);
assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false);
return h.sockets.at(-1);
}
// Explicit fake worklet callbacks retain the native physical-capture path;
// this helper does not populate controller queues or ownership fields itself.
function speakAndRelease(h, data) {
const c = h.controller;
const audio = h.audios.at(-1);
c.input.pushToTalkKeyHeld = true;
c.input.setMicrophoneEnabled(true);
const metadata = audio.metadata;
assert.ok(Number.isSafeInteger(metadata?.captureId));
audio.captureCallbacks.onAudioStart(metadata);
audio.captureCallbacks.onAudio(
{ data, mimeType: 'audio/pcm;rate=16000' },
metadata,
);
c.input.pushToTalkKeyHeld = false;
c.input.setMicrophoneEnabled(false);
audio.captureCallbacks.onAudioEnd(metadata);
return metadata;
}
async function prepareDeferredPtt(h) {
exposeCapture(h);
const socket = await startManual(h);
speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply.
speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A.
const wireBeforeC = socket.sent.length;
const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary.
const owner = h.controller.deferredInput;
assert.equal(owner?.kind, 'push-to-talk');
assert.equal(owner.captureId, metadata.captureId);
assert.equal(
socket.sent.length,
wireBeforeC,
'C must still be wholly deferred',
);
assert.deepEqual(
owner.messages.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
return { socket, owner, metadata };
}
async function assertHealthySuccessor(h, socket, callId) {
const beforeHeight = h.state.height;
await h.tool(callId, socket);
assertSuccessfulZoom(h, socket, callId, beforeHeight);
assert.equal(h.controller.pendingTools, 0);
h.complete(socket); // Native post-tool response boundary, with no output PCM.
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.session.state, 'listening');
}
for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) {
test(
`F04-5 deferred PTT flush loses its owner on ${mode}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket: oldSocket, owner } = await prepareDeferredPtt(h);
const oldAudio = h.audios.at(-1);
let observed = 0;
let sentAtObserverReturn;
let restart;
let stateAtObserverReturn;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== '' ||
observed
)
return;
observed++;
if (mode === 'typed replacement') {
assert.equal(h.session.sendText(TEXT), true);
} else {
h.session.stop();
if (mode === 'Stop+restart')
restart = h.session.start({ pushToTalk: true });
}
sentAtObserverReturn = oldSocket.sent.length;
stateAtObserverReturn = {
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
};
});
t.after(unsubscribe);
h.complete(oldSocket); // A's boundary starts flushing C. C's start emits.
assert.equal(
observed,
1,
'must enter the actual flush/onset observer boundary',
);
assert.ok(
stateAtObserverReturn,
'observer must complete its replacement action',
);
assert.deepEqual(
oldSocket.sent.slice(sentAtObserverReturn),
[],
'retired C must send neither queued PCM nor an extra end after the observer returns',
);
assert.deepEqual(
{
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
},
stateAtObserverReturn,
'old flush must not write successor state',
);
assert.equal(h.controller.deferredInput, null);
assert.equal(
h.controller.typedIntent?.text ?? null,
mode === 'typed replacement' ? TEXT : null,
);
let successorSocket;
if (mode === 'typed replacement') {
assert.equal(h.controller.epoch, owner.epoch);
assert.notEqual(h.controller.turn, owner.turn);
assert.equal(
oldSocket.sent.some(
(message) => message.realtimeInput?.audio?.data === OLD_PCM,
),
false,
);
const typedIndex = oldSocket.sent.findIndex((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some((part) => part.text === TEXT),
),
);
assert.ok(
typedIndex >= 0,
'owned replacement must actually reach transport',
);
// The replacement legitimately closes the already-forwarded C start once.
assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd);
h.complete(oldSocket); // B's remaining retired boundary, deliberately separate.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
successorSocket = oldSocket;
} else {
assert.equal(oldAudio.closed, true);
assert.equal(oldSocket.closed, true);
if (!restart) restart = h.session.start({ pushToTalk: true });
await restart;
successorSocket = h.sockets.at(-1);
assert.notEqual(successorSocket, oldSocket);
assert.equal(
successorSocket.sent.some((message) => message.realtimeInput?.audio),
false,
);
assert.equal(
successorSocket.sent.some(
(message) => message.realtimeInput?.activityEnd,
),
false,
);
assert.equal(h.session.sendText(TEXT), true);
}
await assertHealthySuccessor(
h,
successorSocket,
`healthy-after-deferred-${mode}`,
);
},
);
}
test(
'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
assert.equal(h.session.sendText('A'), true);
assert.equal(h.session.sendText('B'), true);
assert.equal(h.session.sendText('C deferred text'), true);
assert.equal(h.controller.deferredInput?.kind, 'user-text');
let restart;
let observed = 0;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== 'C deferred text'
)
return;
observed++;
h.session.stop();
restart = h.session.start();
});
t.after(unsubscribe);
h.complete(oldSocket);
assert.equal(observed, 1);
assert.equal(
h.controller.replyPending,
false,
'old onSent wrapper must not resurrect a stopped reply',
);
await restart;
assert.equal(h.controller.replyPending, false);
const socket = h.sockets.at(-1);
assert.equal(
socket.sent.length,
1,
'replacement connection should contain setup only',
);
assert.equal(h.session.sendText(TEXT), true);
await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text');
},
);
test(
'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket } = await prepareDeferredPtt(h);
const before = socket.sent.length;
h.complete(socket);
assert.deepEqual(
socket.sent
.slice(before)
.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio?.data === OLD_PCM
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
assert.equal(h.controller.deferredInput, null);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, true);
h.complete(socket); // B's boundary; C remains the owned submitted input.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred');
},
);
+287
View File
@@ -0,0 +1,287 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const resultFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-result' && event.callId === id,
);
const callFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-call' && event.callId === id,
);
test(
'native tool reaches actual common session and camera runner without a caption',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws); // Native activity is input framing, not transcript admission.
await h.tool('ordinary', ws);
assertSuccessfulZoom(h, ws, 'ordinary', 1000);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.runnerCalls[0].options.callId, 'ordinary');
assert.equal(h.runnerCalls[0].options.signal.aborted, false);
assert.equal(h.runnerCalls[0].options.isCurrent(), true);
h.complete(ws);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
for (const cancellation of ['stop', 'native interruption']) {
test(
`${cancellation} discards a stale completed action result; successor changes camera once`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
h.activity(oldSocket);
const gate = h.holdResult('old');
const oldDispatch = h.tool('old', oldSocket);
const committedResult = await gate.ready;
assert.equal(committedResult.ok, true);
assert.equal(
h.state.height,
750,
'the legitimate first action committed before cancellation',
);
assert.equal(callFor(h, 'old').length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'stop') h.session.stop();
else
oldSocket.message({
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.runnerCalls[0].options.signal.aborted, true);
assert.equal(h.runnerCalls[0].options.isCurrent(), false);
await oldDispatch; // Cancellation must settle before the held result returns.
gate.release();
await turnOfEventLoop();
assert.equal(
h.state.height,
750,
'settling a cancelled result adds no camera mutation',
);
assert.equal(h.state.zooms.length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'native interruption')
assert.ok(h.events.some((event) => event.type === 'interruption'));
const successorSocket =
cancellation === 'stop' ? await h.start() : oldSocket;
h.activity(successorSocket);
await h.tool('successor', successorSocket);
assertSuccessfulZoom(h, successorSocket, 'successor', 750);
assert.equal(h.state.zooms.length, 2);
assert.equal(h.runnerCalls.length, 2);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === 'old'),
false,
);
h.complete(successorSocket);
},
);
}
for (const phase of ['common connecting', 'adapter connecting']) {
test(
`Stop from ${phase} observer prevents acquisition and permits fresh Start`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
let stopped = false;
const remove = h.session.subscribe((event) => {
const matchingDetail =
phase === 'common connecting'
? event.detail === undefined
: typeof event.detail === 'string' && event.detail.length > 0;
if (
!stopped &&
event.type === 'state' &&
event.state === 'connecting' &&
matchingDetail
) {
stopped = true;
h.session.stop();
}
});
await h.session.start();
remove();
assert.equal(
stopped,
true,
'the chosen production observer boundary was exercised',
);
assert.deepEqual(h.acquisitions, {
microphone: 0,
audio: 0,
token: 0,
socket: 0,
});
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const ws = await h.start();
h.activity(ws);
await h.tool('fresh-after-start-cancel', ws);
assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000);
h.complete(ws);
},
);
}
test(
'Stop from common action-call observer prevents runner and leaves no unhandled rejection',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
const remove = h.session.subscribe((event) => {
if (event.type === 'action-call' && event.callId === 'cancel-at-call')
h.session.stop();
});
h.activity(ws);
await h.tool('cancel-at-call', ws);
// Do not install a swallowing process rejection listener. node:test records
// an unhandled rejection as a failure; these checkpoints let it surface here.
await turnOfEventLoop();
await turnOfEventLoop();
remove();
assert.equal(callFor(h, 'cancel-at-call').length, 1);
assert.deepEqual(resultFor(h, 'cancel-at-call'), []);
assert.equal(h.runnerCalls.length, 0);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const successor = await h.start();
h.activity(successor);
await h.tool('after-call-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000);
h.complete(successor);
},
);
test(
'Stop from executing observer cannot recreate narration or dispatch the runner',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
let stopped = false;
const remove = h.session.subscribe((event) => {
if (!stopped && event.type === 'state' && event.state === 'executing') {
stopped = true;
h.session.stop();
}
});
h.activity(ws);
await h.tool('cancel-at-executing', ws);
remove();
assert.equal(stopped, true);
assert.equal(h.runnerCalls.length, 0);
assert.deepEqual(callFor(h, 'cancel-at-executing'), []);
assert.deepEqual(resultFor(h, 'cancel-at-executing'), []);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(ws.closed, true);
assert.ok(h.audios.every((audio) => audio.closed));
assert.ok(h.tracks.every((track) => track.readyState === 'ended'));
const before = {
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
};
h.clock.advance(60_000);
assert.deepEqual(
{
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
},
before,
);
const successor = await h.start();
h.activity(successor);
await h.tool('after-executing-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000);
h.complete(successor);
},
);
test(
'native idle observer restart remains connecting and public Stop cancels its pending permission',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
await h.start();
let grant;
const permission = new Promise((resolve) => {
grant = resolve;
});
h.controller.getUserMedia = () => permission;
const states = [];
let restarted = false;
let startup;
const remove = h.session.subscribe((event) => {
if (event.type === 'state') states.push(event.state);
if (!restarted && event.type === 'state' && event.state === 'idle') {
restarted = true;
startup = h.session.start();
}
});
h.session.stop();
remove();
assert.equal(restarted, true);
assert.equal(states.filter((state) => state === 'idle').length, 1);
assert.equal(h.session.state, 'connecting');
assert.equal(h.controller.status, 'connecting');
assert.equal(
h.session.isActive(),
true,
'the common button/facade must offer Stop',
);
h.session.stop();
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.status, 'idle');
const lateTrack = {
readyState: 'live',
stopCalls: 0,
stop() {
this.stopCalls++;
this.readyState = 'ended';
},
};
h.tracks.push(lateTrack);
grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] });
await startup;
await turnOfEventLoop();
assert.equal(lateTrack.stopCalls, 1);
assert.equal(
h.requests.length,
1,
'cancelled restart must not mint another token',
);
assert.equal(
h.sockets.length,
1,
'cancelled restart must not open another socket',
);
assert.equal(h.session.state, 'idle');
},
);
@@ -0,0 +1,85 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { harness } from '../testSupport/geminiNativeHarness.mjs';
const caption = (socket, text) =>
socket.message({ serverContent: { inputTranscription: { text } } });
const userEvents = (events) =>
events.filter(
(event) => event.type === 'transcript' && event.role === 'user',
);
test('native caption events preserve received advisory segments without granting tool authority', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const start = h.events.length;
for (const text of ['little. ', 'Zoom in a ']) caption(ws, text);
const received = userEvents(h.events.slice(start));
assert.deepEqual(
received.map((event) => event.text),
['little. ', 'Zoom in a '],
);
assert.equal(
received.every(
(event) =>
event.delta === true &&
event.final === false &&
event.attribution === 'unverified' &&
event.responseId === null,
),
true,
);
assert.equal(h.c.currentTypedInput(), null);
ws.message({
toolCall: {
functionCalls: [
{
id: 'ordinary-native',
name: 'get_current_view_state',
args: {},
},
],
},
});
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(
userEvents(h.events.slice(start)).some((event) => event.final),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('native old socket captions cannot enter a restarted controller event stream', async () => {
const h = harness();
try {
await h.c.start();
const old = h.sockets[0];
caption(old, 'old partial');
h.c.stop();
await h.c.start();
const start = h.events.length;
const current = h.sockets[1];
caption(current, 'Zoom ');
caption(old, 'stale old socket');
caption(current, 'in a little.');
current.message({ serverContent: { turnComplete: true } });
assert.deepEqual(
userEvents(h.events.slice(start)).map((event) => event.text),
['Zoom ', 'in a little.'],
);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.captures[0].closed, true);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
+667
View File
@@ -0,0 +1,667 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { withToolCatalog } from './gevRealtime.js';
const audioContent = (data) => ({
modelTurn: {
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
},
});
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
for (const [id, name, args] of [
['H01', 'get_entity_context', {}],
['H02', 'control_radio', { action: 'status' }],
]) {
test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
const ready = deferred(),
query = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return args.action === 'play'
? Promise.resolve({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
})
: query.promise;
},
});
h.c.radio.radioLayer.playForVoice = () => ready.promise;
try {
await h.c.start();
h.c.sendTextCommand('Play radio');
const ws = h.sockets[0];
request(ws, 'owner', 'control_radio', { action: 'play' });
await h.c.toolQueue;
ws.message({
serverContent: { ...audioContent('confirmation'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(h.c.radio.radioHandoffInFlight, true);
request(ws, 'query', name, args);
await tick();
assert.equal(
executions.length,
2,
'current native query may start after provider completion',
);
const epoch = h.c.radio.radioHandoffEpoch;
const pending = h.c.radio.pendingRadioPlaybackResult;
const attempt = h.c.radio.radioHandoffAttemptId;
const ownerCurrent = executions[0].options.isCurrent();
ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
await h.c.toolQueue;
assert.equal(executions[1].options.signal.aborted, true);
assert.equal(executions[1].options.isCurrent(), false);
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
assert.equal(executions[0].options.signal.aborted, false);
assert.equal(executions[0].options.isCurrent(), ownerCurrent);
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(h.state.stopped, 0);
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
assert.equal(responses(ws)[0].response.ok, true);
query.resolve({ ok: true });
ready.resolve(true);
await tick();
assert.equal(h.c.status, 'idle');
assert.equal(h.state.ducked, false);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, prior] of [
['H03', 'playing'],
['H04', 'before-first-output'],
]) {
test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
if (prior === 'playing')
ws.message({ serverContent: audioContent('old') });
else {
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
}
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
h.c.input.setMicrophoneEnabled(false);
const turn = h.c.turn,
played = audio.played.length,
firstEvent = h.events.length;
ws.message({
serverContent: {
...audioContent('late-old'),
outputTranscription: { text: 'retired output' },
inputTranscription: { text: 'new request' },
},
});
request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
await h.c.toolQueue;
assert.equal(audio.played.length, played);
assert.equal(audio.pending, false);
assert.equal(h.calls.length, 0);
assert.equal(
h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
false,
);
assert.equal(
h.events
.slice(firstEvent)
.some((e) => e.role === 'user' && e.text === 'new request'),
true,
);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.turn, turn);
ws.message({ serverContent: audioContent('after-interrupted') });
assert.equal(
audio.played.length,
played,
'interrupted alone cannot release the fence',
);
ws.message({
serverContent: {
...audioContent('boundary-old'),
outputTranscription: { text: 'still retired' },
turnComplete: true,
},
});
assert.equal(
audio.played.length,
played,
'coalesced old boundary output is discarded',
);
assert.equal(
h.events.slice(firstEvent).some((e) => e.type === 'completion'),
false,
);
caption(ws, 'unrelated advisory words');
ws.message({
toolCall: {
functionCalls: [
{
id: 'retired-call',
name: 'fly_to_location',
args: { query: 'replayed old target' },
},
{
id: 'new-call',
name: 'fly_to_location',
args: { query: 'new target' },
},
],
},
});
await h.c.toolQueue;
assert.deepEqual(
h.calls.map((c) => c.args.query),
['new target'],
);
assert.equal(h.calls[0].options.userTurn, undefined);
ws.message({
serverContent: {
...audioContent('new'),
outputTranscription: { text: 'new reply' },
turnComplete: true,
},
});
assert.equal(audio.played.at(-1).data, 'new');
audio.drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0],
audio = h.captures[0];
h.c.sendTextCommand('Show the current view');
const response = h.c.currentTypedInput();
assert.equal(response.text, 'Show the current view');
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
response.text,
);
request(ws, 'first');
await h.c.toolQueue;
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({
serverContent: {
...audioContent('confirmation'),
outputTranscription: { text: 'Here is the current view' },
turnComplete: true,
},
});
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(
h.events.findLast((e) => e.role === 'assistant').responseId,
response.responseId,
);
audio.drain();
const completion = h.events.findLast((e) => e.type === 'completion');
assert.equal(completion.responseId, response.responseId);
assert.equal(completion.spoken, true);
assert.equal(completion.audioDrained, true);
assert.equal(h.c.currentTypedInput(), response);
assert.equal(response.text, 'Show the current view');
request(ws, 'same-turn');
await h.c.toolQueue;
assert.equal(h.calls[1].options.userTurn, undefined);
assert.equal(h.c.currentTypedInput(), response);
h.c.sendTextCommand('New request');
assert.notEqual(h.c.currentTypedInput(), response);
assert.equal(h.c.currentTypedInput().text, 'New request');
assert.equal(h.calls[0].options.isCurrent(), false);
assert.equal(h.calls[1].options.isCurrent(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return options.callId === 'old'
? gate.promise
: Promise.resolve({ ok: true });
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
assert.equal(h.c.pendingTools, 0);
await oldWork;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
request(ws, 'fresh');
await h.c.toolQueue;
assert.equal(executions.at(-1).options.callId, 'fresh');
assert.equal(h.c.pendingTools, 0);
gate.resolve({ ok: true });
await tick();
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['fresh'],
);
ws.message({
serverContent: { ...audioContent('fresh reply'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return gate.promise;
},
});
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
h.c.input.bargeIn();
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
caption(ws, 'Show the current view');
assert.equal(h.c.currentTypedInput(), null);
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
'Show the current view',
);
request(ws, 'held');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(executions.length, 1);
assert.equal(executions[0].options.userTurn, undefined);
assert.equal(executions[0].options.isCurrent(), true);
h.c.sendTextCommand('new intent');
await oldWork;
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
gate.resolve({ ok: true });
await tick();
assert.equal(
responses(ws).some((r) => r.id === 'held'),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
for (const [id, restart] of [
['H11', false],
['H12', true],
]) {
test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
const gate = deferred(),
mutations = [],
executions = [];
const h = harness({
runAction: async (name, args, options) => {
executions.push(options);
await gate.promise;
if (options.isCurrent()) mutations.push(name);
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const pending = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
h.c.stop();
const boundary = h.events.length;
const successor = restart ? h.c.start() : null;
await pending;
assert.equal(
h.events
.slice(boundary)
.some((e) => e.type === 'state' && e.state === 'listening'),
false,
);
assert.equal(executions[0].signal.aborted, true);
assert.equal(executions[0].isCurrent(), false);
assert.equal(h.c.pendingTools, 0);
if (successor) await successor;
const laterBoundary = h.events.length;
gate.resolve();
caption(ws, 'late old speech');
await tick();
assert.deepEqual(mutations, []);
assert.equal(
h.events.slice(laterBoundary).some((e) => e.type === 'state'),
false,
);
assert.equal(responses(ws).length, 0);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, boundary] of [
['H14', 'interrupted'],
['H15', 'natural'],
]) {
test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name, args) => {
mutations.push({ name, args });
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(options);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
ws.message({ serverContent: audioContent('old reply') });
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({
serverContent: {
...(boundary === 'interrupted' ? { interrupted: true } : {}),
turnComplete: true,
},
});
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
caption(ws, 'zoom in');
request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
await tick();
assert.equal(
mutations.length,
1,
'native action executes without a transcript wait',
);
assert.equal(observed[0].userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
responses(ws).some((r) => r.response.voiceIntentUnverified),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, exit] of [
['H18', 'stop'],
['H19', 'replacement'],
]) {
test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name) => {
mutations.push(name);
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(name);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
if (exit === 'stop') h.c.stop();
else h.c.sendTextCommand('Show the whole globe');
caption(ws, 'zoom in');
await oldWork;
assert.deepEqual(mutations, []);
assert.deepEqual(observed, []);
assert.equal(h.c.pendingTools, 0);
if (exit === 'replacement') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'fresh-camera', 'zoom_to_globe');
await h.c.toolQueue;
assert.deepEqual(mutations, ['zoom_to_globe']);
assert.deepEqual(observed, ['zoom_to_globe']);
assert.equal(responses(ws).at(-1).response.ok, true);
}
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const settlement of ['queued', 'running']) {
test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
const result = deferred();
let executions = 0;
const h = harness({
runAction: () => {
executions++;
return result.promise;
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
request(ws, 'cancelled-query');
if (settlement === 'running') await tick();
ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
ws.message({ serverContent: { turnComplete: true } });
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions, settlement === 'queued' ? 0 : 1);
assert.deepEqual(responses(ws), []);
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.equal(h.c.pendingCompletionStatus, null);
result.resolve({ ok: true });
await tick();
h.captures[0].drain();
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.deepEqual(responses(ws), []);
} finally {
result.resolve({ ok: true });
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const order of ['before', 'after']) {
test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
const h = harness();
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.controlQueue.length, 1);
assert.equal(responses(ws).length, 0);
if (order === 'after')
ws.message({ serverContent: { turnComplete: true } });
ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
if (order === 'before')
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.c.controlQueue.length, 0);
assert.equal(h.c.controlTimer, null);
assert.equal(h.c.pendingCompletionStatus, null);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
ws.bufferedAmount = 0;
h.advance(1000);
h.captures[0].drain();
assert.deepEqual(responses(ws), []);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const successor of ['input', 'stop']) {
test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
let h;
let successorTurn;
h = harness({
emit: (event) => {
h.events.push(event);
if (event.type !== 'completion') return;
h.sockets[0].bufferedAmount = 0;
if (successor === 'stop') h.c.stop();
else {
h.c.sendTextCommand('Show the whole globe');
successorTurn = h.c.turn;
}
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
ws.message({ serverContent: { turnComplete: true } });
const callCount = h.calls.length;
ws.message({
toolCallCancellation: { ids: ['queued-output'] },
serverContent: {
outputTranscription: { text: 'Retired packet caption' },
turnComplete: true,
},
toolCall: {
functionCalls: [
{ id: 'retired-tool', name: 'get_current_view_state', args: {} },
],
},
});
await h.c.toolQueue;
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.calls.length, callCount);
assert.deepEqual(responses(ws), []);
assert.ok(
!h.events.some((e) =>
JSON.stringify(e).includes('Retired packet caption'),
),
);
if (successor === 'input') {
assert.equal(h.c.turn, successorTurn);
assert.equal(h.c.modelComplete, false);
assert.equal(h.c.replyPending, true);
} else assert.equal(h.c.isActive(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
+734
View File
@@ -0,0 +1,734 @@
// Native ownership replacements for historical Radio regression cases.
// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
import { createVoiceSession } from './session.js';
import { DataLayerManager } from '../data/manager.js';
const viewerFixture = () => ({
clock: { onTick: { addEventListener: () => () => {} } },
camera: { moveEnd: { addEventListener() {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
});
const request = (ws, id, name, args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []);
const responseFor = (ws, id) =>
responses(ws).findLast((response) => response.id === id)?.response;
const speechTurn = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
function drainRadioReply(h) {
h.sockets.at(-1).message({
serverContent: {
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
turnComplete: true,
},
});
h.captures.at(-1).drain();
}
function radioSiblingFixture() {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const lookupEntered = deferred();
const lookupReleased = deferred();
const state = {
stationCount: 2,
selected: { id: 'existing-station', name: 'Existing Station' },
audioState: 'stopped',
};
const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() };
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause({ origin } = {}) {
if (origin === 'voice') trace.pauses++;
state.audioState = 'paused';
return true;
},
stopPlayback() {
trace.stops++;
state.audioState = 'stopped';
return true;
},
selectRequestedStation() {
trace.selections++;
state.selected = { id: 'late-station', name: 'Late Station' };
return state.selected;
},
subscribePlaybackControls: () => () => {},
};
manager.register(
Object.assign(radio, {
id: 'radio',
name: 'Radio',
updateInterval: -1,
}),
);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode(_query, { signal }) {
trace.lookupSignal = signal;
lookupEntered.resolve();
// Deliberately ignore abort: production currentness must still prevent
// this late service completion from committing a station selection.
await lookupReleased.promise;
return { place: { lat: 10, lng: 20, label: 'Deferred Place' } };
},
},
});
const h = harness({
dataManager: manager,
radioLayer: radio,
runAction(name, args, options) {
trace.options.set(options.callId, options);
return runner(name, args, options);
},
});
return {
h,
manager,
runner,
radio,
trace,
state,
lookupEntered,
lookupReleased,
};
}
for (const action of ['select', 'play']) {
for (const stronger of ['stop', 'pause', 'disable', 'off']) {
test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => {
const f = radioSiblingFixture();
const { h, trace, state } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
const turn = h.c.turn;
request(ws, 'slow-selection', 'control_radio', {
action,
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'query-sibling', 'control_radio', { action: 'status' });
request(
ws,
'stronger-radio',
stronger === 'off' ? 'set_layer_visibility' : 'control_radio',
stronger === 'off'
? { layerId: 'radio', enabled: false }
: { action: stronger },
);
await tick();
assert.equal(
responseFor(ws, 'stronger-radio')?.ok,
true,
'stronger control must finish before lookup release',
);
assert.equal(trace.lookupSignal.aborted, true);
await h.c.toolQueue;
assert.equal(
h.c.turn,
turn,
'sibling control cannot invent a new turn',
);
assert.equal(trace.selections, 0);
assert.equal(state.selected.id, 'existing-station');
assert.equal(responseFor(ws, 'slow-selection').ok, false);
assert.match(responseFor(ws, 'slow-selection').error, /cancelled/);
assert.equal(
responseFor(ws, 'slow-selection').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(responseFor(ws, 'query-sibling').ok, true);
assert.equal(trace.options.get('query-sibling').signal.aborted, false);
assert.equal(
responseFor(ws, 'stronger-radio').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(trace.stops, stronger === 'stop' ? 1 : 0);
assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0);
f.lookupReleased.resolve();
await tick();
assert.equal(trace.selections, 0, 'late lookup cannot commit');
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
for (const id of ['slow-selection', 'query-sibling', 'stronger-radio'])
assert.equal(responses(ws).filter((r) => r.id === id).length, 1);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
}
for (const guard of ['malformed', 'replay', 'invalid-arguments']) {
test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
if (guard === 'replay') {
request(ws, 'stronger-radio', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
}
speechTurn(ws);
if (guard === 'replay')
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
const stops = trace.stops;
const handoffEpoch = h.c.radio.radioHandoffEpoch;
request(
ws,
guard === 'malformed' ? '\n' : 'stronger-radio',
'control_radio',
{
action: 'stop',
...(guard === 'invalid-arguments'
? { latitude: 91, longitude: 0 }
: {}),
},
);
await tick();
assert.equal(trace.lookupSignal.aborted, false);
assert.equal(trace.stops, stops);
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
if (guard === 'invalid-arguments') {
assert.equal(responseFor(ws, 'stronger-radio').ok, false);
assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/);
}
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(
trace.selections,
1,
'older eligible work retains authority',
);
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
test('native failed Stop releases its reservation and lets the older selection finish', async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'failed-stop', 'control_radio', { action: 'stop' });
await tick();
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.equal(
trace.lookupSignal.aborted,
false,
'provisional Stop cannot revoke work',
);
assert.equal(h.c.pendingTools, 2);
stopReleased.resolve(false);
await tick();
assert.equal(responseFor(ws, 'failed-stop').ok, false);
assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(trace.lookupSignal.aborted, false);
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(trace.selections, 1);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
test('native pending Stop preserves selection completed before its own successful settlement', async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
f.lookupReleased.resolve();
await tick();
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.ok(h.c.radio.pendingRadioPlaybackResult);
stopReleased.resolve(true);
await h.c.toolQueue;
assert.equal(responseFor(ws, 'pending-stop').ok, true);
assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station');
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
for (const restart of [false, true]) {
test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
const oldDrain = h.c.toolQueue;
const stopOptions = f.trace.options.get('pending-stop');
assert.equal(h.c.pendingTools, 2);
h.c.stop();
assert.equal(stopOptions.signal.aborted, true);
assert.equal(f.trace.lookupSignal.aborted, true);
if (restart) {
await h.c.start();
request(h.sockets[1], 'new-query', 'control_radio', {
action: 'status',
});
await h.c.toolQueue;
assert.equal(responseFor(h.sockets[1], 'new-query').ok, true);
}
stopReleased.resolve(true);
f.lookupReleased.resolve();
await oldDrain;
await tick();
assert.equal(responseFor(ws, 'pending-stop'), undefined);
assert.equal(responseFor(ws, 'slow-selection'), undefined);
assert.equal(f.trace.selections, 0);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.isActive(), restart);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight']) {
test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const ready = deferred();
const stopped = [];
let preflight;
f.radio.playForVoice = () => ready.promise;
f.radio.stopPlayback = (options) => {
stopped.push(options);
return true;
};
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'prepared-play', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.equal(
responseFor(ws, 'prepared-play').radioPlaybackRequested,
true,
);
if (phase === 'preflight')
preflight = h.c.radio.startPendingRadioHandoff();
const attemptId = h.c.radio.radioHandoffAttemptId;
request(ws, 'stronger-stop', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
assert.equal(responseFor(ws, 'stronger-stop').ok, true);
assert.equal(f.state.selected.id, 'existing-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup');
assert.deepEqual(
cleanups,
phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [],
);
ready.resolve(true);
await preflight;
await tick();
assert.equal(
h.c.isActive(),
true,
'late old readiness cannot stop voice',
);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.pendingTools, 0);
} finally {
ready.resolve(false);
h.c.stop();
await preflight;
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight', 'settling']) {
test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const prepared = deferred();
const settleResult = deferred();
const playbackReady = deferred();
let playCalls = 0;
let stopCalls = 0;
let selections = 0;
let activeOptions;
const state = {
stationCount: 2,
selected: null,
audioState: 'stopped',
voiceDucked: false,
};
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked(value) {
state.voiceDucked = value;
},
pause() {
state.audioState = 'paused';
},
subscribePlaybackControls() {
return () => {};
},
selectRequestedStation(_criteria, options) {
assert.equal(options.autoplay, false);
selections++;
state.selected = { id: 'station-1', name: 'Station One' };
return state.selected;
},
playForVoice() {
playCalls++;
return playbackReady.promise;
},
stopPlayback() {
stopCalls++;
state.audioState = 'stopped';
},
};
manager.register({
id: 'radio',
name: 'Radio',
updateInterval: -1,
...radio,
});
const runner = createGevActionRunner({ viewer, dataManager: manager });
let h;
let preflight;
const session = createVoiceSession({
runner: async (name, args, options) => {
activeOptions = options;
const result = await runner(name, args, options);
prepared.resolve(result);
if (phase === 'settling') await settleResult.promise;
return result;
},
createAdapter: (shared) => {
h = harness({ ...shared, dataManager: manager, radioLayer: radio });
return {
start: (settings) => h.c.start(settings),
stop: (settings) => h.c.stop(settings),
sendText: (text) => h.c.sendTextCommand(text),
sendMapEvent: (event) => h.c.notifyMapEvent(event),
};
},
});
try {
await session.start();
const ws = h.sockets[0];
assert.equal(session.sendText('Select Station One on Radio'), true);
request(ws, 'prepared-radio', 'control_radio', {
action: 'select',
stationQuery: 'Station One',
});
const actual = await prepared.promise;
assert.equal(actual.ok, true);
assert.equal(actual.radioPlaybackRequested, true);
assert.equal(actual.stationId, 'station-1');
assert.equal(manager.isEnabled('radio'), true);
if (phase !== 'settling') {
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
}
if (phase === 'preflight') {
preflight = h.c.radio.startPendingRadioHandoff();
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(playCalls, 1);
}
// Native call ownership revokes only this deferred playback. No caption
// parsing or caption-driven provider policy participates in cancellation.
ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } });
if (phase === 'settling')
assert.equal(activeOptions.signal.aborted, true);
settleResult.resolve();
await h.c.toolQueue;
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(manager.isEnabled('radio'), true);
assert.equal(state.selected.id, 'station-1');
assert.equal(selections, 1);
const modelResult = responseFor(ws, 'prepared-radio');
if (phase === 'settling')
assert.equal(
modelResult,
undefined,
'cancelled active call never sends a late result',
);
else {
assert.equal(modelResult.ok, true);
assert.equal(modelResult.stationId, 'station-1');
}
const stopsAfterCancellation = stopCalls;
assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0);
drainRadioReply(h);
playbackReady.resolve(true);
await preflight;
await tick();
assert.equal(
playCalls,
phase === 'preflight' ? 1 : 0,
'drain cannot revive cancelled playback',
);
assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice');
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(session.sendText('Play radio'), true);
request(ws, 'fresh-radio', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
drainRadioReply(h);
await tick();
assert.equal(playCalls, phase === 'preflight' ? 2 : 1);
assert.equal(
h.c.isActive(),
false,
'fresh typed Radio handoff alone ends voice',
);
} finally {
settleResult.resolve();
playbackReady.resolve(false);
session.destroy();
await preflight;
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
// A stronger Radio control skips the ordered queue. An older Radio call still
// queued behind a slow sibling must not start after that control succeeded,
// while a failed control or a later call in the same turn keeps its order.
for (const [label, ids, stopOk, expectPending] of [
[
'older queued select is revoked by a later successful stop',
['slow', 'sel', 'stop'],
true,
false,
],
[
'a failed stop leaves the older queued select current',
['slow', 'sel', 'stop'],
false,
true,
],
[
'a select after the stop in the same turn still runs',
['slow', 'stop', 'sel'],
true,
true,
],
]) {
test(`stronger Radio ordering: ${label}`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const state = {
stationCount: 2,
selected: { id: 'existing', name: 'Existing' },
audioState: 'stopped',
};
const radio = {
id: 'radio',
name: 'Radio',
updateInterval: -1,
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause() {
state.audioState = 'paused';
return true;
},
stopPlayback() {
state.audioState = 'stopped';
return stopOk;
},
selectRequestedStation() {
state.selected = { id: 'late', name: 'Late' };
return state.selected;
},
async playForVoice() {
return true;
},
subscribePlaybackControls: () => () => {},
};
manager.register(radio);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode() {
return { place: { lat: 10, lng: 20, label: 'Place' } };
},
},
});
const slow = deferred();
const h = harness({
dataManager: manager,
radioLayer: radio,
async runAction(name, args, options) {
if (name === 'slow_sibling') {
await slow.promise;
return { ok: true, action: 'slow_sibling' };
}
if (name === 'control_radio' && args.action === 'stop' && !stopOk)
return {
ok: false,
action: 'control_radio',
error: 'Radio stop failed',
};
return runner(name, args, options);
},
});
const calls = {
slow: { id: 'slow', name: 'slow_sibling', args: {} },
sel: {
id: 'sel',
name: 'control_radio',
args: { action: 'select', locationQuery: 'Place' },
},
stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } },
};
try {
await manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } });
await tick();
assert.equal(
responseFor(ws, 'stop')?.ok,
stopOk,
'stop settles without waiting',
);
slow.resolve();
await h.c.toolQueue;
await tick();
assert.equal(
Boolean(h.c.radio.pendingRadioPlaybackResult),
expectPending,
);
assert.equal(responseFor(ws, 'sel')?.ok, expectPending);
} finally {
slow.resolve();
h.c.stop();
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
+280
View File
@@ -0,0 +1,280 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
const tool = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const complete = (ws) => ws.message({ serverContent: { turnComplete: true } });
const oldAudio = {
serverContent: {
modelTurn: {
parts: [
{
inlineData: {
data: 'AAAA',
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
},
};
const outputs = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
// Socket/media are deterministic. Mutations use the production action runner.
function fixture({ holdFirst = false } = {}) {
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: 1000 },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(n) {
position.x -= n;
state.height -= n;
this.positionCartographic.height = state.height;
},
zoomOut(n) {
position.x += n;
state.height += n;
this.positionCartographic.height = state.height;
},
},
scene: {
canvas: {
clientWidth: 800,
clientHeight: 600,
addEventListener() {},
removeEventListener() {},
},
requestRender() {
state.renders++;
},
},
};
const runner = createGevActionRunner({
viewer,
dataManager: { layers: new Map() },
styleManager: {
setVoiceProvider: (p) => state.providerWrites.push(p),
setPanelCollapsed: (...args) => state.panels.push(args),
},
});
const entered = [];
const gate = deferred();
const h = harness({
runAction: async (name, args, options) => {
entered.push({ name, args, options });
if (holdFirst && entered.length === 1) await gate.promise;
if (!options.isCurrent())
return { ok: false, action: name, changed: false };
return runner(name, args, options);
},
});
return {
...h,
state,
entered,
gate,
close() {
gate.resolve();
h.c.stop();
runner.dispose();
assert.equal(h.timers.size, 0);
assert.deepEqual(state.providerWrites, []);
assert.deepEqual(state.panels, []);
},
};
}
for (const [id, order] of [
['H20', 'before-boundary'],
['H21', 'after-boundary'],
]) {
test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(oldAudio);
onset(ws);
caption(ws, 'zoom in');
if (order === 'before-boundary') {
tool(ws, 'ambiguous-before');
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.entered.length, 0);
assert.equal(outputs(ws).length, 0);
}
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
if (order === 'after-boundary') {
tool(ws, 'native-after');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 1);
assert.equal(outputs(ws).at(-1).response.ok, true);
complete(ws);
}
onset(ws);
caption(ws, 'zoom in');
tool(ws, 'fresh-clean-onset');
await h.c.toolQueue;
assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5);
assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2);
assert.equal(h.entered.at(-1).options.userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some((o) => o.id === 'ambiguous-before'),
false,
);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok,
true,
);
} finally {
h.close();
}
});
}
test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'ordinary-before-caption-resolution');
await tick();
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.c.pendingTools, 0);
h.advance(1499);
caption(ws, 'Austin');
h.advance(5000);
tool(ws, 'released-sibling');
await h.c.toolQueue;
assert.equal(h.state.height, 562.5);
assert.equal(h.entered.length, 2);
assert.equal(
h.entered.every((c) => c.options.userTurn === undefined),
true,
);
assert.equal(outputs(ws).length, 2);
assert.equal(
outputs(ws).every((o) => o.response.ok),
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.status, 'listening');
} finally {
h.close();
}
});
for (const [id, exit] of [
['H32', 'stop'],
['H33', 'restart'],
['H34', 'new-input'],
]) {
test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => {
const h = fixture({ holdFirst: true });
try {
await h.c.start();
let ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'old-owned-action');
await tick();
assert.equal(h.entered.length, 1);
assert.equal(h.c.pendingTools, 1);
assert.equal(h.state.height, 1000);
const oldWork = h.c.toolQueue;
const oldMessage = ws.onmessage;
const oldOwner = h.entered[0].options;
if (exit === 'new-input') h.c.sendTextCommand('zoom out');
else h.c.stop();
assert.equal(oldOwner.signal.aborted, true);
assert.equal(oldOwner.isCurrent(), false);
if (exit === 'restart') {
await h.c.start();
ws = h.sockets.at(-1);
}
if (exit !== 'new-input')
oldMessage({
data: JSON.stringify({
toolCall: {
functionCalls: [
{
id: 'late-old-call',
name: 'adjust_camera_zoom',
args: { direction: 'in' },
},
],
},
}),
});
caption(ws, 'Gemini');
h.advance(1500);
h.gate.resolve();
await oldWork;
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some(
(o) => o.id === 'old-owned-action' || o.id === 'late-old-call',
),
false,
);
if (exit === 'stop') {
assert.equal(h.c.status, 'idle');
assert.equal(h.c.isActive(), false);
} else {
if (exit === 'new-input') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
} else h.c.sendTextCommand('zoom out');
tool(ws, 'fresh-after-retirement');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 2);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-after-retirement').response
.ok,
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.isActive(), true);
}
} finally {
h.close();
}
});
}
+529
View File
@@ -0,0 +1,529 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_TEXT = 'Old packet caption.';
const OLD_PCM = 'AAABAA==';
const POST_TOOL_PCM = 'AgADAA==';
const OLD_CALL = 'old-coalesced-call';
function coalescedPacket(callId = OLD_CALL) {
return {
serverContent: {
outputTranscription: { text: OLD_TEXT },
modelTurn: {
parts: [
{ inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } },
],
},
turnComplete: true,
},
toolCall: {
functionCalls: [
{
id: callId,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
};
}
// Local test observations wrap only injected device fakes and the actual
// scheduler methods, delegating unchanged. No frozen fixture file is modified.
function observePacketEffects(h) {
const played = [];
const narration = { audible: 0, toolStarted: 0, toolFinished: 0 };
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
const play = audio.play.bind(audio);
audio.play = (data, mimeType) => {
played.push({ audio, data, mimeType });
return play(data, mimeType);
};
return audio;
};
for (const method of Object.keys(narration)) {
const invoke = h.controller.narration[method].bind(h.controller.narration);
h.controller.narration[method] = (...args) => {
narration[method]++;
return invoke(...args);
};
}
return { played, narration };
}
function packetState(h) {
const c = h.controller;
return {
epoch: c.epoch,
inputGeneration: c.inputGeneration,
turn: c.turn,
completedTurns: c.completedTurns,
pendingTools: c.pendingTools,
modelComplete: c.modelComplete,
pendingCompletionStatus: c.pendingCompletionStatus,
postToolOutput: c.postToolOutput,
modelSpoke: c.modelSpoke,
assistant: c.transcripts.assistant,
replyPending: c.replyPending,
retiredReplyPending: c.retiredReplyPending,
typedText: c.typedIntent?.text ?? null,
typedResponseId: c.typedIntent?.responseId ?? null,
narrationTools: c.narration.state().tools,
narrationTimers: c.narration.state().pendingTimers,
ownedTimers: h.clock.timers.size,
};
}
const completions = (h) =>
h.events.filter((event) => event.type === 'completion');
const toolEvents = (h, callId) =>
h.events.filter(
(event) =>
['action-call', 'action-result', 'progress'].includes(event.type) &&
event.callId === callId,
);
function assertOldRemainderAbsent(h, observed) {
assert.equal(h.state.height, 1000);
assert.deepEqual(h.state.zooms, []);
assert.deepEqual(h.runnerCalls, []);
assert.deepEqual(toolEvents(h, OLD_CALL), []);
assert.deepEqual(observed.played, []);
assert.deepEqual(observed.narration, {
audible: 0,
toolStarted: 0,
toolFinished: 0,
});
assert.deepEqual(completions(h), []);
assert.equal(h.controller.completedTurns, 0);
assert.equal(h.controller.pendingTools, 0);
assert.equal(
h.events.filter(
(event) =>
event.type === 'transcript' &&
event.role === 'assistant' &&
event.final &&
event.text === OLD_TEXT,
).length,
0,
);
for (const socket of h.sockets)
assert.equal(
h.outputs(socket).some((output) => output.id === OLD_CALL),
false,
);
}
for (const mode of ['Stop', 'Stop then Start', 'typed successor']) {
test(
`assistant transcript observer ${mode} fences the rest of a coalesced native packet`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const oldSocket = await h.start();
const oldAudio = h.audios[0];
h.activity(oldSocket);
const before = packetState(h);
let reached = false;
let restart;
let typedAccepted;
let afterObserver;
const remove = h.session.subscribe((event) => {
if (
reached ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.final ||
event.text !== OLD_TEXT
)
return;
reached = true;
if (mode === 'typed successor') {
typedAccepted = h.session.sendText('Zoom in for the typed successor');
} else {
h.session.stop();
if (mode === 'Stop then Start') restart = h.session.start();
}
afterObserver = packetState(h);
});
oldSocket.message(coalescedPacket());
remove();
assert.equal(
reached,
true,
'the actual common transcript observer ran synchronously',
);
// A retired reply may consume its own received terminal, but cannot
// complete or overwrite the successor. Stop/restart permits no old writes.
const afterPacketExpected =
mode === 'typed successor'
? {
...afterObserver,
retiredReplyPending: false,
ownedTimers: afterObserver.ownedTimers - 1,
}
: afterObserver;
assert.deepEqual(packetState(h), afterPacketExpected);
await h.controller.toolQueue;
await turnOfEventLoop();
if (restart) await restart;
assertOldRemainderAbsent(h, observed);
let successorSocket;
if (mode === 'typed successor') {
assert.equal(typedAccepted, true);
assert.equal(
h.controller.epoch,
before.epoch,
'typed replacement shares the socket epoch',
);
assert.ok(h.controller.inputGeneration > before.inputGeneration);
assert.ok(h.controller.turn > before.turn);
assert.deepEqual(packetState(h), afterPacketExpected);
assert.equal(
h.controller.typedIntent.text,
'Zoom in for the typed successor',
);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(oldAudio.closed, false);
const typedMessages = oldSocket.sent.filter((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some(
(part) => part.text === 'Zoom in for the typed successor',
),
),
);
assert.equal(typedMessages.length, 1);
// The received packet already carried the old terminal. No extra
// provider boundary may be invented before testing the actual successor.
assert.equal(h.controller.retiredBoundaryWait, null);
assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.completedTurns, 0);
assert.deepEqual(completions(h), []);
successorSocket = oldSocket;
} else {
assert.equal(oldSocket.closed, true);
assert.equal(oldAudio.closed, true);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
if (mode === 'Stop') {
assert.equal(h.session.state, 'idle');
successorSocket = await h.start();
} else {
assert.equal(h.session.state, 'listening');
assert.equal(h.sockets.length, 2);
assert.equal(h.audios.length, 2);
assert.notEqual(h.audios[1], oldAudio);
assert.equal(h.audios[1].closed, false);
assert.equal(h.audios[1].pending, false);
successorSocket = h.sockets[1];
}
h.activity(successorSocket);
}
await h.tool('owned-after-packet-cancel', successorSocket);
assertSuccessfulZoom(
h,
successorSocket,
'owned-after-packet-cancel',
1000,
);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.runnerCalls.length, 1);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === OLD_CALL),
false,
);
h.complete(successorSocket);
assert.equal(completions(h).length, 1);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
}
test(
'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const ws = await h.start();
h.activity(ws);
ws.message(coalescedPacket('ordinary-coalesced'));
assert.equal(
h.controller.completedTurns,
1,
'first generation terminal is counted once',
);
assert.equal(h.controller.pendingTools, 1);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM],
);
assert.equal(observed.played[0].audio, h.audios[0]);
assert.deepEqual(completions(h), [], 'tool and playback are still pending');
await h.controller.toolQueue;
assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000);
assert.equal(h.controller.pendingTools, 0);
assert.equal(observed.narration.toolStarted, 1);
assert.equal(observed.narration.toolFinished, 1);
assert.equal(h.controller.narration.state().tools, 0);
h.audios[0].drain();
assert.equal(
h.controller.modelComplete,
false,
'function response waits for the post-tool generation',
);
assert.equal(h.controller.completedTurns, 1);
assert.deepEqual(
completions(h),
[],
'draining first-generation PCM cannot finish the tool reply',
);
// The accepted contract has two generations: the function response prompts
// a fresh model reply. This boundary is intentional, not a stale-packet fix.
ws.message({
serverContent: {
outputTranscription: { text: 'The requested zoom is complete.' },
modelTurn: {
parts: [
{
inlineData: {
data: POST_TOOL_PCM,
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
turnComplete: true,
},
});
assert.equal(h.controller.completedTurns, 2);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM, POST_TOOL_PCM],
);
assert.deepEqual(
completions(h),
[],
'post-tool playback must drain before completion',
);
h.audios[0].drain();
assert.equal(completions(h).length, 1);
assert.equal(completions(h)[0].status, 'completed');
assert.equal(completions(h)[0].spoken, true);
assert.equal(completions(h)[0].audioDrained, true);
assert.equal(h.controller.pendingCompletionStatus, null);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(observed.narration.audible, 2);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.outputs(ws).length, 1);
h.audios[0].drain();
assert.equal(
completions(h).length,
1,
'a duplicate device drain cannot complete twice',
);
},
);
// Real native message-type fields are separate packets here; no extra old
// completion is inserted before the accepted successor.
const BOUNDARY_OLD = 'Prior reply caption.';
const BOUNDARY_NEXT = 'Zoom in for the typed successor';
const boundaryZoomCall = (id) => ({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
function boundarySnapshot(h) {
const c = h.controller;
return {
retiredReplyPending: c.retiredReplyPending,
retiredBoundaryWait: Boolean(c.retiredBoundaryWait),
replyPending: c.replyPending,
typedText: c.typedIntent?.text ?? null,
completions: h.events.filter((event) => event.type === 'completion').length,
};
}
for (const mode of [
'partial assistant transcript',
'input transcript',
'interruption event',
'listening event',
'final assistant transcript control',
]) {
test(
`received native terminal preserves typed successor after ${mode}`,
{ concurrency: false, timeout: 5000 },
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let oldDispatch, gate;
const interruptionMode =
mode === 'interruption event' || mode === 'listening event';
if (interruptionMode) {
h.session.sendText('Prior zoom request');
gate = h.holdResult('already-committed-old-tool');
oldDispatch = h.tool('already-committed-old-tool', ws);
await gate.ready;
} else if (mode === 'input transcript') {
ws.message({
serverContent: {
outputTranscription: { text: 'Previously emitted output' },
},
});
}
let entered = 0,
accepted;
const remove = h.session.subscribe((event) => {
if (entered) return;
const matches =
mode === 'interruption event'
? event.type === 'interruption' && event.reason === 'user-speech'
: mode === 'listening event'
? event.type === 'state' && event.state === 'listening'
: mode === 'input transcript'
? event.type === 'transcript' &&
event.role === 'user' &&
event.text === 'Old input caption'
: event.type === 'transcript' &&
event.role === 'assistant' &&
event.text === BOUNDARY_OLD &&
event.final ===
(mode === 'final assistant transcript control');
if (!matches) return;
entered++;
accepted = h.session.sendText(BOUNDARY_NEXT);
});
if (interruptionMode)
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
else
ws.message({
serverContent: {
...(mode === 'input transcript'
? { inputTranscription: { text: 'Old input caption' } }
: {}),
outputTranscription: { text: BOUNDARY_OLD },
turnComplete: true,
},
});
remove();
if (oldDispatch) {
await oldDispatch;
gate.release();
}
await turnOfEventLoop();
const afterPacket = boundarySnapshot(h);
const beforeSuccessorHeight = h.state.height;
// No extra synthetic old boundary. This is the next reply to the accepted text.
ws.message({
serverContent: {
outputTranscription: { text: 'Handling your new zoom.' },
},
});
ws.message(boundaryZoomCall('typed-successor-call'));
await h.controller.toolQueue;
await turnOfEventLoop();
ws.message({ serverContent: { turnComplete: true } });
const afterSuccessor = boundarySnapshot(h);
assert.equal(entered, 1);
assert.equal(accepted, true);
assert.equal(
afterPacket.retiredReplyPending,
false,
'received old terminal must settle its retirement',
);
assert.equal(afterPacket.retiredBoundaryWait, false);
assert.equal(afterPacket.typedText, BOUNDARY_NEXT);
assert.equal(
h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call')
.length,
1,
'accepted typed successor must reach the actual runner',
);
assert.equal(h.state.height, beforeSuccessorHeight * 0.75);
assert.equal(afterSuccessor.completions, 1);
assert.equal(afterSuccessor.replyPending, false);
assert.equal(
h.controller.cancelledCalls.has('typed-successor-call'),
false,
);
},
);
}
for (const terminalAlreadyApplied of [false, true]) {
test(
`typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let entered = false;
const remove = h.session.subscribe((event) => {
if (
entered ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.text !== 'Boundary control' ||
event.final !== terminalAlreadyApplied
)
return;
entered = true;
assert.equal(h.session.sendText('First typed replacement'), true);
if (terminalAlreadyApplied)
assert.equal(h.session.sendText('Second typed replacement'), true);
});
ws.message({
serverContent: {
outputTranscription: { text: 'Boundary control' },
...(terminalAlreadyApplied ? { turnComplete: true } : {}),
},
});
remove();
assert.equal(entered, true);
// With no old terminal received, or when that terminal was consumed
// before the observer, a real predecessor boundary is still outstanding.
assert.equal(h.controller.retiredReplyPending, true);
assert.ok(h.controller.retiredBoundaryWait);
assert.equal(completions(h).length, 0);
assert.equal(h.runnerCalls.length, 0);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.retiredBoundaryWait, null);
await h.tool('after-real-pending-boundary', ws);
assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000);
h.complete(ws);
assert.equal(completions(h).length, 1);
},
);
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+3 -5
View File
@@ -61,10 +61,8 @@ export function initGevVoiceCommands(options) {
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
commands.session?.signal?.addEventListener(
'abort',
() => pointer?.destroy(),
{ once: true },
);
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
}
+65 -1
View File
@@ -182,6 +182,63 @@ export function pointerChip(snapshot) {
};
}
/** Dedupe key of the clearing pointer_context item. */
export const POINTER_NONE_KEY = 'none';
/**
* The voice-card event for a turn's pointer: a gesture reveals its target at
* once; open-mic and typed turns reveal it only through announcePointer.
* @param {object|null} snapshot
* @param {boolean} gesture
* @returns {{announced: object|null, pointer: object|null}}
*/
export function pointerTurnEvent(snapshot, gesture) {
const chip = gesture ? pointerChip(snapshot) : null;
return chip
? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } }
: { announced: null, pointer: null };
}
/**
* The pointer_context item a turn should send, or null when there is nothing
* new. Without a gesture the model learns only what kind of thing is pointed
* at, never which. Losing a sent pointer produces one clearing item.
* @param {object|null} snapshot
* @param {{at: string, gesture: boolean, sentKey: string|null}} turn
* @returns {{item: object, key: string}|null}
*/
export function pointerTurnContext(snapshot, { at, gesture, sentKey }) {
const chip = pointerChip(snapshot);
const item = !chip
? sentKey && sentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: {
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return null;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
return key === sentKey ? null : { item, key };
}
/**
* The voice-card event revealing a pointer a tool resolved, once per snapshot.
* @param {object|null} snapshot
* @param {object|null} announced Snapshot already shown.
* @returns {object|null}
*/
export function pointerAnnouncement(snapshot, announced) {
const chip = pointerChip(snapshot);
return chip && snapshot !== announced
? { ...chip, screenPx: snapshot.screenPx }
: null;
}
/**
* Track the pointer over one element (the globe canvas).
* @param {object} options
@@ -330,7 +387,14 @@ export function createPointerTracker({
* (open mic only when it moved recently: `recentMs`). The result becomes
* the turn's referent (null when stale).
*/
beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) {
beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) {
// Delayed admitted audio keeps its capture's pointer without replacing a
// newer physical hold. Undefined preserves existing caller behavior.
if (capturedSnapshot !== undefined) {
const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot;
active = snapshot?.fresh ? snapshot : null;
return snapshot;
}
const reuse =
at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS;
const snapshot = reuse ? held.snapshot : capture(at, { recentMs });
+112
View File
@@ -6,8 +6,13 @@ import {
POINTER_HOLD_MS,
buildPointerSnapshot,
createPointerTracker,
POINTER_NONE_KEY,
formatPointerContext,
pointerAnnouncement,
pointerChip,
pointerContextItem,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
function harness({ pick } = {}) {
@@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => {
assert.equal(h.tracker.capture('keydown').fresh, false);
});
test('delayed speech uses its released capture without replacing a newer physical hold', () => {
const h = harness();
h.move(700, 250);
const capturedA = h.tracker.hold();
h.tracker.releaseHold();
h.move(200, 100);
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
const heldB = h.tracker.hold();
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
assert.equal(h.tracker.isHeld(), true);
assert.equal(h.tracker.beginTurn('speech_start'), heldB);
h.tracker.destroy();
});
test('explicit stale captures clear the referent while a silent hold leaves it alone', () => {
const h = harness();
h.move(700, 250);
const current = h.tracker.beginTurn('text');
h.move(200, 100);
h.tracker.hold();
h.tracker.releaseHold();
assert.equal(h.tracker.activeSnapshot(), current);
for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) {
h.tracker.beginTurn('text');
h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(h.tracker.activeSnapshot(), null);
}
h.tracker.destroy();
});
test('a capture arriving after destruction cannot restore an active referent', () => {
const h = harness();
h.move(700, 250);
const capturedSnapshot = h.tracker.hold();
assert.equal(capturedSnapshot.fresh, true);
h.tracker.destroy();
const result = h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(result.fresh, false);
assert.equal(h.tracker.activeSnapshot(), null);
assert.equal(h.tracker.isHeld(), false);
assert.equal(h.tracker.beginTurn('speech_start').fresh, false);
});
test('format: the model item is compact inert JSON; labels are bounded data', () => {
const snapshot = buildPointerSnapshot({
at: 'speech_start',
@@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => {
assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' });
assert.equal(pointerChip(null), null);
});
test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => {
const snapshot = {
fresh: true,
at: 'keydown',
entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' },
lat: 10,
lon: 20,
screen: { x: 5, y: 6 },
screenPx: { x: 5, y: 6 },
};
const shown = pointerTurnEvent(snapshot, true);
assert.equal(shown.announced, snapshot);
assert.deepEqual(shown.pointer, {
kind: 'this',
label: 'UAL1',
screenPx: { x: 5, y: 6 },
});
assert.deepEqual(pointerTurnEvent(snapshot, false), {
announced: null,
pointer: null,
});
const gesture = pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: null,
});
assert.deepEqual(gesture.item, pointerContextItem(snapshot));
assert.equal(
pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: gesture.key,
}),
null,
);
assert.deepEqual(
pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null })
.item,
{ type: 'pointer_context', at: 'text', pointing: 'aircraft' },
);
const cleared = pointerTurnContext(null, {
at: 'text',
gesture: false,
sentKey: gesture.key,
});
assert.deepEqual(cleared, {
item: { type: 'pointer_context', at: 'text', target: 'none' },
key: POINTER_NONE_KEY,
});
for (const sentKey of [null, POINTER_NONE_KEY])
assert.equal(
pointerTurnContext(null, { at: 'text', gesture: false, sentKey }),
null,
);
assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer);
assert.equal(pointerAnnouncement(snapshot, snapshot), null);
assert.equal(pointerAnnouncement(null, null), null);
});
+21
View File
@@ -0,0 +1,21 @@
export const VOICE_PROVIDERS = Object.freeze({
openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name
gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name
});
/** Optional providers require an explicit host opt-in and an available factory. */
export function supportedVoiceProviders(
requested = ['openai'],
factories = {},
) {
if (!Array.isArray(requested))
throw new TypeError('supportedProviders must be an array');
const selected = [...new Set(requested)].filter(
(name) =>
Object.hasOwn(VOICE_PROVIDERS, name) &&
typeof factories[name] === 'function',
);
if (!selected.length)
throw new Error('No supported voice provider is configured');
return Object.freeze(selected);
}
+161
View File
@@ -0,0 +1,161 @@
import { supportedVoiceProviders } from './providerCapabilities.js';
const STORAGE_KEY = 'gev.voice.provider';
export function readVoiceProvider(storage, supported = ['openai', 'gemini']) {
let saved;
try {
saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY);
} catch {
/* Storage is optional. */
}
return supported.includes(saved)
? saved
: supported.includes('openai')
? 'openai'
: supported[0];
}
/** Keep one public controller identity while replacing each provider's lifetime. */
export function createProviderCommands(options, { bind, factories, storage }) {
const supportedProviders = supportedVoiceProviders(
options.supportedProviders ?? Object.keys(factories),
factories,
);
window.__gevVoiceCommands?.stop?.({ removeUi: true });
let provider = readVoiceProvider(storage, supportedProviders);
let current;
let disposed = false;
const lifetime = new AbortController();
let mountGeneration = 0;
let voiceEnabled = false;
const stored = () => {
try {
return storage ?? globalThis.localStorage;
} catch {
return null;
}
};
const stop = (settings = {}) => {
if (disposed) return;
voiceEnabled = false;
if (settings.removeUi) {
disposed = true;
options.signal?.removeEventListener('abort', destroy);
}
try {
current?.session.stop(settings);
} finally {
if (settings.removeUi) {
lifetime.abort();
options.runner.dispose?.();
}
}
};
const destroy = () => stop({ removeUi: true });
const selectProvider = (next) => {
if (disposed || !supportedProviders.includes(next) || next === provider)
return false;
const resume = voiceEnabled && current.session.isActive();
// Carry the input mode, never a held key, microphone or conversation.
const pushToTalk = Boolean(
current.session.adapter.getStartOptions?.().pushToTalk,
);
const generation = ++mountGeneration;
voiceEnabled = false;
current.session.destroy();
// Disposal suppresses the retired adapter's idle event. Clear the shared
// conversation explicitly before the new mount can use these owners.
if (disposed || generation !== mountGeneration) return false;
options.pointer?.clear();
options.referents?.clear();
options.runner.resetConversation?.();
if (disposed || generation !== mountGeneration) return false;
provider = next;
try {
stored()?.setItem(STORAGE_KEY, provider);
} catch {
/* Storage is optional. */
}
mount();
if (resume && !disposed && provider === next)
void current.session.start({ pushToTalk });
globalThis.document?.getElementById('gev-voice-provider')?.focus();
return true;
};
const handleSessionEvent = (event) => {
if (event?.type === 'state' && event.state === 'connecting')
voiceEnabled = true;
if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) {
voiceEnabled = false;
}
// The host still observes the mounted provider's neutral session events.
options.onSessionEvent?.(event);
};
const sendTextCommand = (text) => {
if (disposed) return false;
return current.session.sendText(text);
};
const api = {
// The shared action surface is independent of an active voice connection.
runner: options.runner,
supportedProviders,
signal: lifetime.signal,
get provider() {
return provider;
},
get session() {
return current?.session;
},
get status() {
return current?.status ?? current?.session.state ?? 'idle';
},
get state() {
return current?.session.state ?? 'idle';
},
get disposed() {
return disposed;
},
start: (settings) => !disposed && current.session.start(settings),
stop,
destroy,
isActive: () => !disposed && current.session.isActive(),
sendTextCommand,
sendText: sendTextCommand,
notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event),
};
// Existing diagnostic and controller accessors continue to reflect the active provider.
const facade = new Proxy(api, {
get(target, name, receiver) {
if (Reflect.has(target, name)) return Reflect.get(target, name, receiver);
const value = current?.[name];
return typeof value === 'function' ? value.bind(current) : value;
},
});
function mount() {
voiceEnabled = false;
const generation = ++mountGeneration;
current = bind({
...options,
signal: lifetime.signal,
resetExisting: false,
disposeRunner: false,
provider,
supportedProviders,
onProviderChange: selectProvider,
onSessionEvent: (event) => {
if (!disposed && generation === mountGeneration)
handleSessionEvent(event);
},
createSession: factories[provider],
});
window.__gevVoiceCommands = facade;
}
mount();
options.signal?.addEventListener('abort', destroy, { once: true });
if (options.signal?.aborted) destroy();
return facade;
}
+888
View File
@@ -0,0 +1,888 @@
// Imports intentionally target the candidate, never the immutable reference.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createProviderCommands,
readVoiceProvider,
} from './providerCommands.js';
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createVoiceCommands as createCommands } from './commands.js';
import { createGevActionRunner } from './gevActions.js';
import { createPointerTracker } from './pointerContext.js';
import { createReferentRegistry } from './referents.js';
import { describeUnsavedWork, t as translate } from '../i18n/index.js';
const PROVIDER_KEY = 'gev.voice.provider';
const settle = () => new Promise((resolve) => setImmediate(resolve));
function deferred() {
let resolve;
let reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
// Listener counts observe the actual binder's registrations/removals.
class Element extends EventTarget {
listeners = new Map();
attributes = new Map();
addEventListener(type, handler, options) {
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
this.listeners.get(type).add(handler);
super.addEventListener(type, handler, options);
}
removeEventListener(type, handler, options) {
this.listeners.get(type)?.delete(handler);
super.removeEventListener(type, handler, options);
}
listenerCount(type) {
return this.listeners.get(type)?.size || 0;
}
setAttribute(name, value) {
this.attributes.set(name, String(value));
}
focus() {
this.focusCount = (this.focusCount || 0) + 1;
}
}
for (const custom of [false, true]) {
test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => {
const prior = globalThis.window;
globalThis.window = new EventTarget();
let controls;
t.after(() => {
controls?.session.destroy();
globalThis.window = prior;
});
const ui = {
root: { dataset: {}, remove() {} },
button: new Element(),
status: {},
detail: {},
providerField: {},
};
let defaultCreated = 0;
let customCreated = 0;
const options = {
runner: async () => ({ ok: true }),
createControl: () => ui,
createController: ({ onSessionEvent }) => {
defaultCreated++;
return {
ui,
start({ pushToTalk }) {
this.pushToTalkMode = pushToTalk;
onSessionEvent({ type: 'state', state: 'listening' });
},
stop() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
sendTextCommand: (text) => `default:${text}`,
};
},
};
if (custom) {
options.supportedProviders = ['gemini'];
options.createSession = ({ emit }) => {
customCreated++;
return {
capabilities: { pushToTalk: false },
start() {
emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText: (text) => `custom:${text}`,
sendMapEvent: () => true,
};
};
}
controls = createCommands(options);
await controls.session.start({ pushToTalk: true });
assert.equal(defaultCreated, custom ? 0 : 1);
assert.equal(customCreated, custom ? 1 : 0);
assert.equal(
controls.session.sendText('test'),
`${custom ? 'custom' : 'default'}:test`,
);
assert.equal(ui.providerField.hidden, true);
assert.equal(ui.root.dataset.providerChoice, 'false');
if (!custom)
assert.deepEqual(controls.session.adapter.getStartOptions(), {
pushToTalk: true,
});
});
}
/** Real facade -> real binder -> real common session and real action runner.
* Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count
* calls to real pointer/referent/conversation cleanup; they do not replace it.
*/
function compositionFixture(
t,
{
supportedProviders = ['openai', 'gemini'],
factoryProviders = ['openai', 'gemini'],
savedProvider = null,
storageDenied = false,
gateStarts = () => false,
onSessionEvent = () => {},
} = {},
) {
const original = new Map(
['window', 'document'].map((key) => [
key,
Object.getOwnPropertyDescriptor(globalThis, key),
]),
);
const lifetime = new AbortController();
const views = [];
const instances = [];
const mounts = [];
const events = [];
const order = [];
const annotations = new Set();
const storageWrites = [];
const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 };
const state = { height: 1000, zooms: 0, renders: 0 };
let facade;
let pointer;
globalThis.window = new EventTarget();
globalThis.document = {
getElementById: (id) =>
id === 'gev-voice-provider' ? views.at(-1)?.providerSelect : null,
};
t.after(async () => {
try {
facade?.destroy();
pointer?.destroy();
for (const instance of instances)
for (const gate of instance.gates) gate.resolve();
await Promise.allSettled(instances.flatMap((instance) => instance.jobs));
} finally {
for (const [key, descriptor] of original) {
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
else delete globalThis[key];
}
}
});
const canvas = new EventTarget();
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: state.height },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(metres) {
position.x -= metres;
state.height -= metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
zoomOut(metres) {
position.x += metres;
state.height += metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
},
scene: {
canvas,
requestRender() {
state.renders++;
},
},
};
const dataManager = { layers: new Map() };
pointer = createPointerTracker({ element: null, now: () => 1000 });
const referents = createReferentRegistry();
const runner = createGevActionRunner({
viewer,
dataManager,
styleManager: {},
deixis: { pointer, referents },
});
for (const [object, method, counter] of [
[pointer, 'clear', 'pointer'],
[referents, 'clear', 'referents'],
[runner, 'resetConversation', 'conversation'],
[runner, 'dispose', 'dispose'],
]) {
const originalMethod = object[method];
object[method] = (...args) => {
counters[counter]++;
order.push(counter);
return originalMethod.apply(object, args);
};
}
const factories = Object.fromEntries(
factoryProviders.map((provider) => [
provider,
(hooks) => {
const id = instances.length;
let generation = 0;
const instance = {
id,
provider,
hooks,
starts: [],
stops: [],
jobs: [],
gates: [],
acquired: 0,
microphoneEnabled: false,
bound: 0,
input: { pushToTalk: false, held: false },
texts: [],
mapEvents: [],
};
instances.push(instance);
order.push(`factory:${id}`);
const adapter = {
identity: `${provider}:${id}`,
capabilities: {
pushToTalk: true,
costControls: provider === 'openai',
},
controller: {
providerDiagnostic() {
return this.session.adapter.identity;
},
},
start(settings = {}) {
const attempt = ++generation;
instance.starts.push({ ...settings });
instance.input.pushToTalk = Boolean(settings.pushToTalk);
hooks.emit({
type: 'state',
state: 'connecting',
detail: `adapter:${provider}`,
});
const job = (async () => {
if (attempt !== generation || hooks.signal.aborted) return;
if (gateStarts(instance)) {
const gate = deferred();
instance.gates.push(gate);
await gate.promise;
}
// Adapters own cancellation of their asynchronous acquisition.
if (attempt !== generation || hooks.signal.aborted) return;
instance.acquired++;
instance.microphoneEnabled =
!instance.input.pushToTalk || instance.input.held;
hooks.emit({
type: 'state',
state: 'listening',
detail: provider,
});
})();
instance.jobs.push(job);
return job;
},
stop(settings = {}) {
generation++;
instance.stops.push({ ...settings });
order.push(`stop:${id}`);
instance.input.pushToTalk = instance.input.held = false;
instance.microphoneEnabled = false;
},
getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }),
bindControls() {
instance.bound++;
},
sendText(text) {
instance.texts.push(text);
return `${provider}:${text}`;
},
sendMapEvent(event) {
instance.mapEvents.push(event);
return true;
},
};
instance.adapter = adapter;
return adapter;
},
]),
);
function createControl() {
const id = views.length;
order.push(`control:${id}`);
const providerSelect = new Element();
providerSelect.ownerDocument = { createElement: () => ({}) };
providerSelect.replaceChildren = (...options) => {
providerSelect.options = options;
};
const view = {
root: {
dataset: {},
classList: { remove() {} },
removeCount: 0,
remove() {
this.removeCount++;
order.push(`remove:${id}`);
},
},
button: new Element(),
providerSelect,
providerField: {},
status: {},
detail: {},
errorDetail: {},
tierButton: {},
costValue: {},
helpDetail: {},
card: {},
};
views.push(view);
return view;
}
function passiveView() {
return {
events: [],
outlines: [],
destroyed: 0,
handle(event) {
this.events.push(event);
},
outline(event) {
this.outlines.push(event);
},
destroy() {
this.destroyed++;
},
};
}
facade = createProviderCommands(
{
runner,
pointer,
referents,
dataManager,
signal: lifetime.signal,
supportedProviders,
createControl,
createCard() {
const card = passiveView();
views.at(-1).cardView = card;
return card;
},
createReticle() {
const reticle = passiveView();
views.at(-1).reticle = reticle;
return reticle;
},
annotations: {
onOutlineEvent(callback) {
annotations.add(callback);
return () => annotations.delete(callback);
},
},
onSessionEvent(event) {
events.push(event);
onSessionEvent(event, facade);
},
},
{
factories,
storage: {
getItem() {
if (storageDenied) throw Error('Storage denied');
return savedProvider;
},
setItem(key, value) {
if (storageDenied) throw Error('Storage denied');
storageWrites.push([key, value]);
savedProvider = value;
},
},
bind(options) {
// Keep the real callback so stale-event suppression tests exercise the
// facade's mount generation as well as the common session's disposal.
const mount = { options, callback: options.onSessionEvent };
mounts.push(mount);
mount.controls = bindVoiceCommands(options);
return mount.controls;
},
},
);
return {
facade,
runner,
pointer,
referents,
lifetime,
instances,
views,
mounts,
events,
order,
annotations,
counters,
storageWrites,
state,
select(provider) {
const select = views.at(-1).providerSelect;
select.value = provider;
select.dispatchEvent(new Event('change'));
},
seedConversation() {
pointer.beginTurn('text', {
capturedSnapshot: { fresh: true, at: 'text' },
});
referents.recordResult('frame_overhead', {
ok: true,
display: { title: 'One aircraft' },
referents: [
{ n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' },
],
});
assert.ok(pointer.activeSnapshot());
assert.equal(referents.list().length, 1);
},
};
}
test('provider preference uses only the supplied host capability list', () => {
const stored = (value) => ({ getItem: () => value });
assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai');
assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini');
assert.equal(
readVoiceProvider(stored('unknown'), ['gemini', 'openai']),
'openai',
);
assert.equal(
readVoiceProvider(stored('gemini'), ['openai', 'gemini']),
'gemini',
);
assert.equal(
readVoiceProvider(
{
getItem() {
throw Error('denied');
},
},
['gemini'],
),
'gemini',
);
});
for (const scenario of [
{
title: 'OpenAI-only host ignores saved Gemini',
requested: ['openai'],
saved: 'gemini',
expected: ['openai'],
provider: 'openai',
},
{
title: 'Gemini-only host falls back from saved OpenAI',
requested: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
{
title: 'dual host restores Gemini',
requested: ['openai', 'gemini'],
saved: 'gemini',
expected: ['openai', 'gemini'],
provider: 'gemini',
},
{
title: 'unknown, duplicate and unavailable factories are excluded',
requested: ['unknown', 'gemini', 'openai', 'gemini'],
factories: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
]) {
test(scenario.title, (t) => {
const f = compositionFixture(t, {
supportedProviders: scenario.requested,
factoryProviders: scenario.factories,
savedProvider: scenario.saved,
});
assert.deepEqual(f.facade.supportedProviders, scenario.expected);
assert.equal(Object.isFrozen(f.facade.supportedProviders), true);
assert.equal(f.facade.provider, scenario.provider);
assert.deepEqual(
f.views[0].providerSelect.options.map((option) => option.value),
scenario.expected,
);
assert.equal(
f.views[0].providerField.hidden,
scenario.expected.length === 1,
);
assert.equal(
f.views[0].root.dataset.providerChoice,
String(scenario.expected.length > 1),
);
assert.equal(f.instances.length, 1);
assert.equal(
f.instances[0].starts.length,
0,
'discovery must not start a provider',
);
assert.equal(f.facade.state, 'idle');
});
}
test('invalid or empty host support fails before mounting a control', () => {
let mounts = 0;
const dependencies = {
bind() {
mounts++;
},
factories: { openai() {} },
};
assert.throws(
() =>
createProviderCommands({ supportedProviders: 'gemini' }, dependencies),
/must be an array/,
);
assert.throws(
() =>
createProviderCommands(
{ supportedProviders: ['gemini', 'unknown'] },
dependencies,
),
/No supported voice provider/,
);
assert.equal(mounts, 0);
});
test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => {
const f = compositionFixture(t);
const firstSession = f.facade.session;
const outerSignal = f.facade.signal;
const first = f.instances[0];
// PTT controls can activate the adapter directly; the facade must learn
// activity from its common events, not only from facade.start().
first.input.held = true;
await first.adapter.start({ pushToTalk: true });
assert.equal(first.microphoneEnabled, true);
assert.equal(f.facade.isActive(), true);
f.seedConversation();
const before = { ...f.counters };
f.select('gemini');
await settle();
const second = f.instances[1];
assert.equal(window.__gevVoiceCommands, f.facade);
assert.equal(f.facade.runner, f.runner);
assert.equal(f.facade.signal, outerSignal);
assert.equal(outerSignal.aborted, false);
assert.equal(firstSession.disposed, true);
assert.equal(first.hooks.signal.aborted, true);
assert.deepEqual(first.stops, [{ removeUi: true }]);
assert.equal(first.input.held, false);
assert.equal(first.microphoneEnabled, false);
assert.deepEqual(second.starts, [{ pushToTalk: true }]);
assert.equal(second.input.held, false);
assert.equal(
second.microphoneEnabled,
false,
'a held key must never migrate to the new adapter',
);
assert.equal(f.facade.state, 'listening');
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1'));
assert.equal(f.views[0].button.listenerCount('click'), 0);
assert.equal(f.views[0].providerSelect.listenerCount('change'), 0);
assert.equal(f.views[1].providerSelect.listenerCount('change'), 1);
assert.equal(f.views[1].providerSelect.focusCount, 1);
assert.equal(f.views[1].tierButton.hidden, true);
assert.equal(f.views[1].costValue.hidden, true);
assert.equal(f.facade.providerDiagnostic(), 'gemini:1');
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(
f.counters[key],
before[key] + 1,
`one explicit replacement reset: ${key}`,
);
assert.equal(
f.counters.dispose,
0,
'provider replacement keeps the page runner',
);
assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]);
assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello');
assert.deepEqual(first.texts, []);
assert.deepEqual(second.texts, ['hello']);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(
result.ok,
true,
'the actual shared action runner remains usable after switching',
);
assert.equal(f.state.zooms, 1);
});
for (const initialState of ['idle', 'error']) {
test(`${initialState} selection never autostarts; manual start still works`, async (t) => {
const f = compositionFixture(t, { storageDenied: true });
if (initialState === 'error') {
await f.facade.start({ pushToTalk: true });
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
f.select('gemini');
await settle();
assert.equal(
f.facade.provider,
'gemini',
'denied storage cannot block a supported selection',
);
assert.equal(f.facade.state, 'idle');
assert.deepEqual(f.instances[1].starts, []);
f.select('gemini');
f.select('unknown');
assert.equal(
f.instances.length,
2,
'same and unsupported selections cannot remount',
);
assert.equal(f.facade.provider, 'gemini');
await f.facade.start({ pushToTalk: false });
assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]);
assert.equal(f.instances[1].microphoneEnabled, true);
});
}
for (const outcome of ['resolve', 'reject']) {
test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 0,
});
const oldStart = f.facade.start({ pushToTalk: false });
const old = f.instances[0];
assert.equal(f.facade.state, 'connecting');
f.select('gemini');
await settle();
assert.equal(f.facade.state, 'listening');
const current = f.instances[1];
const eventCount = f.events.length;
const currentStops = current.stops.length;
const currentCardCount = f.views[1].cardView.events.length;
old.gates[0][outcome](new Error('Late retired startup failure'));
await oldStart;
for (const state of ['idle', 'error', 'connecting']) {
old.hooks.emit({ type: 'state', state, detail: 'retired adapter' });
f.mounts[0].callback({
type: 'state',
state,
detail: 'retired forward callback',
});
}
assert.equal(f.events.length, eventCount);
assert.equal(f.views[1].cardView.events.length, currentCardCount);
assert.equal(f.facade.state, 'listening');
assert.equal(old.acquired, 0);
assert.equal(current.stops.length, currentStops);
assert.equal(current.starts.length, 1);
f.facade.stop();
f.mounts[0].callback({ type: 'state', state: 'connecting' });
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'a retired connecting callback cannot revive voice intent',
);
assert.equal(f.facade.state, 'idle');
});
}
test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 1,
});
await f.facade.start();
f.select('gemini');
const replacement = f.instances[1];
assert.equal(f.facade.state, 'connecting');
f.facade.stop();
replacement.gates[0].resolve();
await Promise.all(replacement.jobs);
assert.equal(replacement.acquired, 0);
assert.equal(f.facade.state, 'idle');
f.select('openai');
assert.deepEqual(f.instances[2].starts, []);
await f.facade.start();
assert.equal(
f.facade.state,
'listening',
'a later explicit start remains healthy',
);
});
for (const emitsIdle of [false, true]) {
test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => {
const f = compositionFixture(t);
if (emitsIdle) {
const instance = f.instances[0];
const originalStop = instance.adapter.stop;
instance.adapter.stop = function (settings = {}) {
originalStop.call(this, settings);
if (!settings.preserveStatus && !settings.removeUi)
instance.hooks.emit({ type: 'state', state: 'idle' });
};
}
for (const end of ['stop', 'error']) {
await f.facade.start();
f.seedConversation();
const before = { ...f.counters };
const generation = f.referents.generation();
if (end === 'stop') f.facade.stop();
else {
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
assert.equal(f.referents.generation(), generation + 1);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`);
}
});
}
test('outer destruction disposes the runner once and removes each provider binding', async (t) => {
const priorUnsaved = describeUnsavedWork();
const f = compositionFixture(t);
await f.facade.start();
f.select('gemini');
await settle();
assert.equal(f.annotations.size, 1);
assert.ok(
describeUnsavedWork().includes(translate('the active voice session')),
);
const oldCardEvents = f.views[0].cardView.events.length;
for (const callback of f.annotations) callback({ id: 'fixture-outline' });
assert.equal(f.views[0].cardView.outlines.length, 0);
assert.equal(f.views[1].cardView.outlines.length, 1);
assert.equal(f.instances[1].mapEvents.length, 1);
f.facade.destroy();
f.facade.destroy();
f.facade.stop({ removeUi: true });
f.lifetime.abort();
assert.equal(f.counters.dispose, 1);
assert.equal(f.facade.signal.aborted, true);
assert.equal(f.facade.disposed, true);
assert.equal(f.annotations.size, 0);
assert.deepEqual(describeUnsavedWork(), priorUnsaved);
const finalEvents = f.events.length;
const starts = f.instances.map((instance) => instance.starts.length);
for (const [index, view] of f.views.entries()) {
assert.equal(view.root.removeCount, 1);
assert.equal(view.cardView.destroyed, 1);
assert.equal(view.reticle.destroyed, 1);
assert.equal(view.button.listenerCount('click'), 0);
assert.equal(view.providerSelect.listenerCount('change'), 0);
assert.equal(f.instances[index].hooks.signal.aborted, true);
view.button.dispatchEvent(new Event('click'));
view.providerSelect.value = 'openai';
view.providerSelect.dispatchEvent(new Event('change'));
f.instances[index].hooks.emit({ type: 'state', state: 'connecting' });
f.mounts[index].callback({ type: 'state', state: 'connecting' });
}
assert.equal(f.events.length, finalEvents);
assert.equal(f.views[0].cardView.events.length, oldCardEvents);
assert.deepEqual(
f.instances.map((instance) => instance.starts.length),
starts,
);
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
assert.equal(f.facade.sendText('after destroy'), false);
assert.equal(f.facade.notifyMapEvent({}), false);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(result.cancelled, true);
assert.equal(
f.state.zooms,
0,
'the actual runner cannot move the camera after outer disposal',
);
});
for (const action of ['stop', 'destroy']) {
for (const phase of ['common', 'adapter']) {
test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => {
let intercepted = 0;
const f = compositionFixture(t, {
onSessionEvent(event, facade) {
if (
facade?.provider !== 'gemini' ||
event.type !== 'state' ||
event.state !== 'connecting'
)
return;
const matching =
phase === 'common'
? event.detail === undefined
: event.detail === 'adapter:gemini';
if (matching && intercepted++ === 0) facade[action]();
},
});
await f.facade.start({ pushToTalk: true });
f.select('gemini');
await settle();
const replacement = f.instances[1];
assert.equal(intercepted, 1);
assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1);
assert.equal(replacement.acquired, 0);
assert.equal(replacement.microphoneEnabled, false);
assert.equal(f.facade.isActive(), false);
assert.equal(f.facade.state, 'idle');
assert.equal(
f.events.filter(
(event) => event.state === 'listening' && event.detail === 'gemini',
).length,
0,
);
assert.equal(f.facade.disposed, action === 'destroy');
assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0);
if (action === 'stop') {
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'Stop cleared the intent to resume',
);
await f.facade.start({ pushToTalk: false });
assert.equal(f.facade.state, 'listening');
} else {
f.select('openai');
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
}
});
}
}
+18 -35
View File
@@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js';
import { createNarrationScheduler } from './narration.js';
import { createEarcon } from './earcon.js';
import {
POINTER_NONE_KEY,
POINTER_OPEN_MIC_RECENT_MS,
pointerChip,
pointerContextItem,
pointerAnnouncement,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
/** Dedupe key of the clearing pointer_context item. */
const POINTER_NONE_KEY = 'none';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
import { postDebugLog } from './realtimeDiagnostics.js';
@@ -287,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade {
} catch {
snapshot = null;
}
const chip = pointerChip(snapshot);
this.pointerAnnounced = gesture && chip ? snapshot : null;
this.emitSessionEvent({
type: 'pointer',
pointer:
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
const event = pointerTurnEvent(snapshot, gesture);
this.pointerAnnounced = event.announced;
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
const context = pointerTurnContext(snapshot, {
at,
gesture,
sentKey: this.pointerSentKey,
});
const item = !chip
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: // What kind of thing, never which: no label or coordinates.
{
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return snapshot;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
if (key === this.pointerSentKey) return snapshot;
if (this._turns.notifyMapEvent(item)) {
this.pointerSentKey = key;
if (!context) return snapshot;
if (this._turns.notifyMapEvent(context.item)) {
this.pointerSentKey = context.key;
const chip = context.key !== POINTER_NONE_KEY;
this.debugLog('pointer.context', {
at,
target: chip ? snapshot.target : 'none',
@@ -327,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade {
*/
announcePointer() {
const snapshot = this.pointer?.activeSnapshot?.() || null;
const chip = pointerChip(snapshot);
if (!chip || snapshot === this.pointerAnnounced) return false;
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
if (!pointer) return false;
this.pointerAnnounced = snapshot;
this.emitSessionEvent({
type: 'pointer',
pointer: { ...chip, screenPx: snapshot.screenPx },
});
this.emitSessionEvent({ type: 'pointer', pointer });
return true;
}
+22 -1
View File
@@ -247,7 +247,9 @@ export class RealtimeRadio {
handoffChannel?.readyState === 'open',
},
);
const stillCurrent = handoffEpoch === this.radioHandoffEpoch;
const stillCurrent =
handoffEpoch === this.radioHandoffEpoch &&
this.radioHandoffAttemptId === handoffAttemptId;
if (this.radioHandoffAttemptId === handoffAttemptId) {
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
@@ -266,6 +268,25 @@ export class RealtimeRadio {
}
}
/** Revoke one call's prepared/preflight result, preserving other owners. */
cancelPlaybackResult(result) {
if (!result) return;
if (this.pendingRadioPlaybackResult === result) {
this.pendingRadioPlaybackResult = null;
this.radioHandoffDeferredByReservation = false;
}
if (this.radioHandoffInFlightResult !== result) return;
const attemptId = this.radioHandoffAttemptId;
const shouldStopPlayback = this.radioHandoffInFlight;
this.radioHandoffInFlightResult = null;
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
// Keep the group epoch and sibling reservations; release ownership before
// Stop can synchronously notify playback observers.
if (shouldStopPlayback)
this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId });
}
/** Invalidate delayed Radio work inside the requested authority scope. */
abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) {
for (const [controller, metadata] of this.activeRadioToolControllers) {
+120
View File
@@ -0,0 +1,120 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { GevRealtimeController } from './realtimeController.js';
const tick = () => new Promise((resolve) => setImmediate(resolve));
const playback = (id) => ({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
id,
});
function fixture(t) {
const attempts = [],
stopped = [],
corrections = [],
states = [];
const controller = new GevRealtimeController({
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
remove() {},
},
status: {},
detail: {},
errorDetail: {},
},
runner: async () => ({ ok: false }),
debugSink: null,
onSessionEvent: (event) => states.push(event),
radioLayer: {
setVoiceDucked() {},
playForVoice: ({ attemptId }) =>
new Promise((resolve, reject) =>
attempts.push({ attemptId, resolve, reject }),
),
stopPlayback: (options) => stopped.push(options),
},
});
controller.queueResponseCreate = (message) => corrections.push(message);
controller.dc = { readyState: 'open', send() {}, close() {} };
t.after(() => controller.stop({ removeUi: true }));
return { controller, attempts, stopped, corrections, states };
}
test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A');
c.pendingRadioPlaybackResult = a;
const pending = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
assert.equal(c.pendingRadioPlaybackResult, a);
assert.equal(c.radioHandoffInFlight, false);
h.attempts[0].reject(new Error('obsolete frozen preflight'));
await pending;
assert.deepEqual(h.corrections, []);
assert.equal(
h.states.some((event) => event.detail === 'Radio did not start'),
false,
);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2);
assert.equal(c.radioHandoffInFlightResult, a);
assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId);
h.attempts[1].reject(new Error('current preflight failed'));
await tick();
assert.equal(h.corrections.length, 1);
assert.equal(c.status, 'listening');
});
test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => {
const h = fixture(t),
c = h.controller;
c.pendingRadioPlaybackResult = playback('A');
const old = c.startPendingRadioHandoff();
c.cancelRadioHandoff();
h.attempts[0].reject(new Error('cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
c.pendingRadioPlaybackResult = playback('B');
const current = c.startPendingRadioHandoff();
h.attempts[1].resolve(false);
await current;
assert.equal(h.corrections.length, 1);
});
test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A'),
b = playback('B');
c.pendingRadioPlaybackResult = a;
const old = c.startPendingRadioHandoff();
const siblingEpoch = c.radioHandoffEpoch;
c.pendingRadioPlaybackResult = b;
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(playback('unknown'));
assert.equal(c.radioHandoffEpoch, siblingEpoch);
assert.equal(c.pendingRadioPlaybackResult, b);
assert.equal(h.stopped.length, 1);
h.attempts[0].reject(new Error('late cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
const current = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
c._radio.cancelPlaybackResult(b);
assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true);
h.attempts[1].resolve(false);
await current;
assert.deepEqual(h.corrections, []);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed');
});
+1
View File
@@ -16,6 +16,7 @@ export function createRealtimeSession({
controller,
capabilities: { costControls: true, pushToTalk: true },
start: (settings) => controller.start(settings),
getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }),
stop: (settings) => controller.stop(settings),
sendText: (text) => controller.sendTextCommand(text),
sendMapEvent: (event) => controller.notifyMapEvent(event),
+1 -1
View File
@@ -137,7 +137,7 @@ export class RealtimeTurns {
content: [{ type: 'input_text', text: cleanText }],
},
};
this.sendRealtimeEvent(itemEvent, 'client.user_text');
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.requestUserTextResponse();
}
+268
View File
@@ -0,0 +1,268 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
function fixture({ pointer = null, onUserSend = () => {} } = {}) {
const sent = [];
const executed = [];
const rejectedTexts = new Set();
let rejectResponse = false;
const controller = new GevRealtimeController({
runner: async (name) => {
executed.push(name);
return { ok: true };
},
backend: {},
debugSink: null,
pointer,
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: {},
detail: {},
},
});
controller.dc = {
readyState: 'open',
send(body) {
const value = JSON.parse(body);
const text = value.item?.role === 'user' && value.item.content?.[0]?.text;
if (value.item?.role === 'user') onUserSend();
if (
rejectedTexts.has(text) ||
(rejectResponse && value.type === 'response.create')
)
throw new Error('Synthetic channel rejection');
sent.push(value);
},
close() {
this.readyState = 'closed';
},
};
controller.status = 'listening';
controller._turns.narration = null;
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
const created = (id) => event({ type: 'response.created', response: { id } });
const done = (id) =>
event({
type: 'response.done',
response: { id, status: 'completed', output: [] },
});
const creates = () =>
sent.filter(
(x) =>
x.type === 'response.create' &&
x.response?.metadata?.gev !== 'progress',
);
const texts = () =>
sent
.filter((x) => x.item?.role === 'user')
.map((x) => x.item.content[0].text);
return {
controller,
sent,
executed,
rejectedTexts,
event,
created,
done,
creates,
texts,
rejectResponses(value) {
rejectResponse = value;
},
};
}
test('rejected idle text schedules no answer, and a successful successor answers once', async () => {
const f = fixture();
try {
f.rejectedTexts.add('rejected');
assert.equal(f.controller.sendTextCommand('rejected'), undefined);
assert.equal(f.creates().length, 0);
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.controller.sendTextCommand('successor'), undefined);
assert.deepEqual(f.texts(), ['successor']);
assert.equal(f.creates().length, 1);
await f.created('successor');
await f.done('successor');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text behind an active response cannot queue an answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.queueResponseCreate('Old tool follow-up');
assert.equal(
f.controller.pendingResponseInstructions,
'Old tool follow-up',
);
const runningTool = new AbortController();
f.controller._turns.activeToolAbortControllers.add(runningTool);
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.pendingResponseInstructions, null);
assert.equal(f.controller._turns.isSupersededResponse('active'), true);
assert.equal(runningTool.signal.aborted, true);
const staleCall = {
type: 'response.function_call_arguments.done',
response_id: 'active',
call_id: 'stale-call',
name: 'fly_to_location',
arguments: '{"query":"Paris"}',
};
await f.event(staleCall);
await f.event(staleCall);
assert.deepEqual(f.executed, []);
const outputs = f.sent.filter(
(x) => x.item?.type === 'function_call_output',
);
assert.equal(outputs.length, 1);
assert.equal(outputs[0].item.call_id, 'stale-call');
assert.equal(JSON.parse(outputs[0].item.output).superseded, true);
await f.done('active');
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('rejected text still prepares narration and pointer context before sending', () => {
const steps = [];
const f = fixture({
pointer: {
beginTurn() {
steps.push('pointer');
return null;
},
clear() {},
},
onUserSend() {
steps.push('send');
},
});
f.controller._turns.narration = {
cancel() {
steps.push('cancel');
},
userTurnEnded() {
steps.push('user-turn-ended');
},
};
try {
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']);
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('earlier accepted deferred input survives a newer failed send and answers once', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(f.texts(), ['accepted']);
assert.equal(f.controller.pendingUserTextResponse, true);
assert.equal(f.creates().length, 0);
await f.done('active');
assert.equal(f.creates().length, 1);
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text preserves an outstanding create without adding deferred input', async () => {
const f = fixture();
try {
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.responseCreatePending, true);
assert.equal(f.controller.pendingUserTextResponse, false);
await f.created('accepted');
await f.done('accepted');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
for (const acceptedFirst of [false, true]) {
test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => {
const f = fixture();
try {
assert.equal(
f.controller._turns.speakProgressLine('Checking the map.'),
true,
);
await f.event({
type: 'response.created',
response: { id: 'progress', metadata: { gev: 'progress' } },
});
if (acceptedFirst) f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, acceptedFirst);
await f.event({
type: 'response.done',
response: {
id: 'progress',
status: 'cancelled',
metadata: { gev: 'progress' },
},
});
assert.equal(f.creates().length, acceptedFirst ? 1 : 0);
} finally {
f.controller.stop();
}
});
}
test('accepted text with a rejected response.create releases the pending latch for a successor', async () => {
const f = fixture();
try {
f.rejectResponses(true);
f.controller.sendTextCommand('accepted without response');
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.creates().length, 0);
f.rejectResponses(false);
f.controller.sendTextCommand('successor');
assert.deepEqual(f.texts(), ['accepted without response', 'successor']);
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('successful deferred typed burst still retains both items and coalesces one answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('first');
f.controller.sendTextCommand('second');
assert.deepEqual(f.texts(), ['first', 'second']);
assert.equal(f.creates().length, 0);
await f.done('active');
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
+9 -3
View File
@@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
emit({ type: 'state', state: 'connecting' });
const attempt = ++startAttempt;
emit({ type: 'state', state: 'connecting' });
if (disposed || attempt !== startAttempt) return;
try {
await adapter.start(options);
} catch (error) {
@@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
stop(options = {}) {
if (options.removeUi) return dispose();
if (disposed) return;
startAttempt++;
const attempt = ++startAttempt;
cancelActions();
try {
adapter.stop(options);
} finally {
if (!options.preserveStatus)
if (
!options.preserveStatus &&
!disposed &&
attempt === startAttempt &&
state !== 'idle'
)
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
}
},
+187
View File
@@ -6,6 +6,7 @@ import {
voiceCardView,
} from './voiceCardPresentation.js';
import { createVoiceSession } from './session.js';
import { createRealtimeSession } from './realtimeSession.js';
import { realtimeSessionEvent } from './realtimeEvents.js';
import { createVoiceCommands } from './sessionCommands.js';
import { describeUnsavedWork } from '../i18n/index.js';
@@ -441,3 +442,189 @@ test('a late rejection cannot settle a newer turn card', async () => {
);
f.session.destroy();
});
function startReentrancyFixture() {
let starts = 0;
let stops = 0;
const settings = [];
const session = createVoiceSession({
runner: async () => ({ ok: true }),
createAdapter: ({ emit }) => ({
async start(options) {
starts++;
settings.push(options);
emit({ type: 'state', state: 'listening' });
},
stop() {
stops++;
},
sendText() {},
sendMapEvent() {},
}),
});
return { session, settings, starts: () => starts, stops: () => stops };
}
for (const action of ['stop', 'destroy']) {
test(`initial connecting observer ${action} prevents adapter acquisition`, async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session[action]();
}
});
await f.session.start();
assert.equal(
f.starts(),
0,
'cancelled attempt must not call adapter.start',
);
assert.equal(f.stops(), 1);
assert.equal(f.session.state, 'idle');
if (action === 'stop') {
await f.session.start();
assert.equal(f.starts(), 1, 'later explicit Start remains usable');
assert.equal(f.session.state, 'listening');
}
f.session.destroy();
});
}
test('preserveStatus Stop still prevents adapter acquisition', async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop({ preserveStatus: true });
}
});
await f.session.start();
assert.equal(f.starts(), 0);
assert.equal(
f.session.state,
'connecting',
'preserved state is not ownership',
);
f.session.stop();
await f.session.start();
assert.equal(f.starts(), 1);
f.session.destroy();
});
test('Stop and explicit reentrant successor start acquire only for the successor', async () => {
const f = startReentrancyFixture();
let successor;
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop();
successor = f.session.start();
}
});
await f.session.start();
await successor;
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.destroy();
});
test('ordinary start forwards caller settings once and remains stoppable', async () => {
const f = startReentrancyFixture();
const settings = { pushToTalk: true };
await f.session.start(settings);
assert.equal(f.settings.length, 1);
assert.equal(f.settings[0], settings);
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.stop();
assert.equal(f.session.state, 'idle');
f.session.destroy();
});
for (const emitsIdle of [false, true]) {
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
let adapter;
const calls = [];
const events = [];
const failure = new Error('Controlled stop failure');
const session = createVoiceSession({
runner: async () => ({}),
createAdapter: ({ emit }) =>
(adapter = {
async start() {
emit({ type: 'state', state: 'listening' });
},
stop(settings = {}) {
calls.push({ receiver: this, settings });
if (settings.removeUi) return;
if (emitsIdle && !settings.preserveStatus)
emit({ type: 'state', state: 'idle' });
if (outcome === 'throw') throw failure;
return 'adapter-only return';
},
sendText() {},
sendMapEvent() {},
}),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
await session.start();
events.length = 0;
const settings =
outcome === 'preserveStatus' ? { preserveStatus: true } : {};
if (outcome === 'throw')
assert.throws(
() => session.stop(settings),
(error) => error === failure,
);
else assert.equal(session.stop(settings), undefined);
assert.equal(calls[0].receiver, adapter);
assert.equal(calls[0].settings, settings);
assert.equal(
session.state,
outcome === 'preserveStatus' ? 'listening' : 'idle',
);
assert.equal(
events.filter(
(event) => event.type === 'state' && event.state === 'idle',
).length,
outcome === 'preserveStatus' ? 0 : 1,
);
});
}
}
test('actual default OpenAI adapter stop publishes one common idle', (t) => {
const events = [];
const ui = {
root: {
dataset: {},
querySelectorAll: () => [],
classList: { remove() {} },
remove() {},
},
status: {},
detail: {},
};
const runner = async () => ({});
const session = createVoiceSession({
runner,
createAdapter: (hooks) =>
createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
// Exercise the real stop path without acquiring microphone/WebRTC resources.
session.adapter.controller.setStatus('listening', 'Controlled active state');
events.length = 0;
session.stop();
assert.equal(
events.filter((event) => event.type === 'state' && event.state === 'idle')
.length,
1,
);
assert.equal(session.state, 'idle');
assert.equal(session.adapter.controller.status, 'idle');
});
+52 -20
View File
@@ -2,6 +2,7 @@ import { createVoiceControl, voiceStatusLabel } from './control.js';
import { createVoiceSession } from './session.js';
import { VoiceCardControls } from './voiceCard.js';
import { PointerReticle } from './pointerReticle.js';
import { VOICE_PROVIDERS } from './providerCapabilities.js';
import { msg, registerUnsavedWork, t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
@@ -25,12 +26,18 @@ export function createVoiceCommands({
signal,
debugSink,
createControl = createVoiceControl,
resetExisting = true,
disposeRunner = true,
provider,
supportedProviders,
onProviderChange,
onSessionEvent,
createCard = (options) => new VoiceCardControls(options),
pointer = null,
referents = null,
createReticle = createDefaultReticle,
}) {
window.__gevVoiceCommands?.stop?.({ removeUi: true });
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({ reset: true });
const session = createVoiceSession({
runner,
@@ -50,6 +57,34 @@ export function createVoiceCommands({
});
const adapter = session.adapter;
const capabilities = adapter.capabilities || {};
const providerHandler = () => onProviderChange?.(ui.providerSelect.value);
if (ui.providerSelect && onProviderChange) {
if (supportedProviders && ui.providerSelect.replaceChildren) {
const doc = ui.providerSelect.ownerDocument;
ui.providerSelect.replaceChildren(
...supportedProviders.map((name) => {
const option = doc.createElement('option');
option.value = name;
option.textContent = VOICE_PROVIDERS[name].label;
return option;
}),
);
}
ui.providerSelect.value = provider;
ui.providerSelect.title = [
t('Changing provider reconnects voice if it is on.'),
capabilities.advisoryLimit,
]
.filter(Boolean)
.join(' ');
ui.providerSelect.addEventListener('change', providerHandler);
ui.root.dataset.provider = provider;
}
const hasProviderChoice = Boolean(
onProviderChange && supportedProviders?.length > 1,
);
if (ui.providerField) ui.providerField.hidden = !hasProviderChoice;
ui.root.dataset.providerChoice = String(hasProviderChoice);
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
if (!capabilities.pushToTalk) {
@@ -60,6 +95,7 @@ export function createVoiceCommands({
// Retain the existing controller's inspection surface for browser tools.
const controls = adapter.controller || session;
controls.session = session;
controls.ui = ui;
const updateStatus = session.subscribe((event) => {
if (event.type !== 'state') return;
ui.root.dataset.status = event.state;
@@ -101,6 +137,10 @@ export function createVoiceCommands({
});
controls.pointer = pointer;
controls.referents = referents;
// Forward last: a host observer may synchronously replace or stop this mount.
const forwardEvents = onSessionEvent
? session.subscribe(onSessionEvent)
: null;
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
card?.outline(event);
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
@@ -115,34 +155,26 @@ export function createVoiceCommands({
else void session.start({ pushToTalk: false });
};
ui.button.addEventListener('click', buttonHandler);
session.signal.addEventListener(
'abort',
() => {
ui.button.removeEventListener('click', buttonHandler);
unregisterUnsavedWork();
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePointer();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
ui.root.remove();
},
{ once: true },
);
if (session.disposed) {
let cleaned = false;
const cleanup = () => {
if (cleaned) return;
cleaned = true;
ui.button.removeEventListener('click', buttonHandler);
ui.providerSelect?.removeEventListener('change', providerHandler);
unregisterUnsavedWork();
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePointer();
forwardEvents?.();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
if (disposeRunner) runner.dispose?.();
ui.root.remove();
} else adapter.bindControls?.();
};
session.signal.addEventListener('abort', cleanup, { once: true });
if (session.disposed) cleanup();
else adapter.bindControls?.();
window.__gevVoiceCommands = controls;
return controls;
}