refactor(voice): separate Realtime state and resource owners

This commit is contained in:
Sameh Khamis committed 2026-09-15 12:05:01 -07:00
1 parent cd22ca3559
commit cc9a080f2d
22 files changed
+4194 -2803

No files matched your search

+219
View File
@@ -0,0 +1,219 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
import { RealtimeInput } from './realtimeInput.js';
import { RealtimeViewport } from './realtimeViewport.js';
import { resolveVoiceModel } from './voiceCost.js';
function installGlobals(t, values) {
for (const [name, value] of Object.entries(values)) {
const previous = Object.getOwnPropertyDescriptor(globalThis, name);
Object.defineProperty(globalThis, name, { configurable: true, writable: true, value });
t.after(() => {
if (previous) Object.defineProperty(globalThis, name, previous);
else delete globalThis[name];
});
}
}
function browser(t) {
const peers = [];
const streams = [];
class Peer {
constructor() { this.connectionState = 'connected'; peers.push(this); }
addTrack() {}
createDataChannel() {
this.channel = {
readyState: 'open', handlers: new Map(),
addEventListener(type, handler) { this.handlers.set(type, handler); },
send() {}, close() { this.readyState = 'closed'; },
};
return this.channel;
}
async createOffer() { return { sdp: 'offer' }; }
async setLocalDescription(offer) { this.localDescription = offer; }
async setRemoteDescription() {}
close() { this.connectionState = 'closed'; }
}
installGlobals(t, {
window: { RTCPeerConnection: Peer }, RTCPeerConnection: Peer,
document: {
querySelectorAll: () => [], body: { appendChild() {} },
createElement: () => ({ dataset: {}, style: {}, remove() {} }),
},
navigator: { mediaDevices: { async getUserMedia() {
const track = { stops: 0, stop() { this.stops++; } };
const stream = { getTracks: () => [track], getAudioTracks: () => [track], track };
streams.push(stream); return stream;
} } },
});
return { peers, streams, Peer };
}
test('retained peer and channel callbacks cannot act after stop or enter a replacement session', async (t) => {
const { peers, streams } = browser(t);
const actions = [];
const controller = new GevRealtimeController({
runner: async (name) => { actions.push(name); return { ok: true }; },
backend: {
async requestToken() { return { token: 'synthetic', model: resolveVoiceModel('mini').id }; },
async negotiate() { return 'answer'; },
},
debugSink: null,
ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: {}, detail: {} },
});
await controller.start();
const old = peers[0];
old.channel.handlers.get('open')();
assert.equal(controller.status, 'listening');
const message = { data: JSON.stringify({ type: 'response.function_call_arguments.done', name: 'get_entity_context', call_id: 'stale', arguments: '{}' }) };
controller.stop();
assert.doesNotThrow(() => old.ontrack({ streams: [{}] }));
await old.channel.handlers.get('message')(message);
await controller.start();
const current = peers[1];
current.channel.handlers.get('open')();
const audio = controller.audioEl;
old.channel.readyState = 'open'; // Model an already queued callback from the old transport.
old.channel.handlers.get('open')();
await old.channel.handlers.get('message')(message);
old.ontrack({ streams: [{}] });
old.onicecandidateerror({ errorText: 'stale error' });
assert.equal(controller.status, 'listening');
assert.equal(controller.pc, current);
assert.equal(controller.audioEl, audio);
assert.equal(audio.srcObject, undefined);
assert.deepEqual(actions, []);
const incoming = {};
current.ontrack({ streams: [incoming] });
assert.equal(audio.srcObject, incoming, 'the current peer still delivers audio');
controller.stop();
assert.deepEqual(streams.map(s => s.track.stops), [1, 1]);
});
test('an offer resolved after restart cannot change the replacement peer description', async (t) => {
const { peers, Peer } = browser(t);
let finishOldOffer;
const originalOffer = Peer.prototype.createOffer;
Peer.prototype.createOffer = function () {
if (peers.indexOf(this) === 0) return new Promise(resolve => { finishOldOffer = resolve; });
return originalOffer.call(this);
};
const controller = new GevRealtimeController({
runner: async () => ({ ok: true }),
backend: {
async requestToken() { return { token: 'synthetic', model: resolveVoiceModel('mini').id }; },
async negotiate() { return 'answer'; },
},
debugSink: null,
ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: {}, detail: {} },
});
const firstStart = controller.start();
while (!finishOldOffer) await new Promise(resolve => setImmediate(resolve));
controller.stop();
await controller.start();
const current = peers[1];
finishOldOffer({ sdp: 'superseded offer' });
await firstStart;
assert.equal(controller.pc, current);
assert.equal(current.localDescription.sdp, 'offer');
assert.equal(peers[0].localDescription, undefined);
controller.stop();
});
const localContext = { action: 'get_entity_context', scene: { basemap: { viewScale: 'local' } } };
test('a late viewport capture cannot publish into a replacement or reset conversation', async () => {
for (const reset of [false, true]) {
let channel = { readyState: 'open' };
let finish;
const sent = [];
const owner = new RealtimeViewport({
readChannel: () => channel,
capture: () => new Promise(resolve => { finish = resolve; }),
operations: { sendRealtimeEvent(message) { sent.push(message); return true; } },
});
const pending = owner.sendVisualContextIfUseful(localContext);
if (reset) owner.reset();
else channel = { readyState: 'open' };
finish('data:image/jpeg;base64,abc');
assert.equal(await pending, false);
assert.deepEqual(sent, []);
assert.equal(owner.lastViewportItemId, null);
}
});
test('a current viewport capture retains one image and replaces it through the protocol', async () => {
const channel = { readyState: 'open' };
const sent = [];
const owner = new RealtimeViewport({
readChannel: () => channel, capture: async () => 'data:image/jpeg;base64,abc',
operations: { sendRealtimeEvent(message) { sent.push(message); return true; } },
});
assert.equal(await owner.sendVisualContextIfUseful(localContext), true);
const first = owner.lastViewportItemId;
assert.equal(await owner.sendVisualContextIfUseful(localContext), true);
assert.equal(sent[1].type, 'conversation.item.delete');
assert.equal(sent[1].item_id, first);
assert.equal(sent[2].item.id, owner.lastViewportItemId);
owner.reset();
assert.equal(owner.lastViewportItemId, null);
assert.equal(owner.pendingViewportDeletes.size, 0);
});
test('audio meter initialization failure releases its newly acquired AudioContext', (t) => {
let closes = 0;
class AudioContext {
resume() { return Promise.resolve(); }
createAnalyser() { return {}; }
createMediaStreamSource() { throw new Error('analysis unavailable'); }
close() { closes++; return Promise.resolve(); }
}
installGlobals(t, { window: { AudioContext } });
const owner = new RealtimeInput({
readUi: () => ({ root: { querySelectorAll: () => [{ style: { setProperty() {}, removeProperty() {} } }] } }),
readStatus: () => 'listening', readStream: () => null, operations: {},
});
owner.startVoiceVisualizer({});
assert.equal(closes, 1);
assert.equal(owner.visualizerAudioContext, null);
owner.stopVoiceVisualizer();
assert.equal(closes, 1);
});
test('stopping one audio meter revokes retained frames without stopping another meter', (t) => {
const frames = [];
const contexts = [];
class AudioContext {
constructor() { this.reads = 0; this.closes = 0; contexts.push(this); }
resume() { return Promise.resolve(); }
createAnalyser() { return {
frequencyBinCount: 32,
getByteFrequencyData: data => { this.reads++; data.fill(30); },
}; }
createMediaStreamSource() { return { connect() {}, disconnect() {} }; }
close() { this.closes++; return Promise.resolve(); }
}
installGlobals(t, {
window: { AudioContext },
requestAnimationFrame: callback => { frames.push(callback); return frames.length; },
cancelAnimationFrame() {},
});
const create = () => new RealtimeInput({
readUi: () => ({ root: { querySelectorAll: () => [{ style: { setProperty() {}, removeProperty() {} } }] } }),
readStatus: () => 'listening', readStream: () => null, operations: {},
});
const first = create();
const second = create();
first.startVoiceVisualizer({}); second.startVoiceVisualizer({});
first.stopVoiceVisualizer();
const reads = contexts[0].reads;
frames[0]();
assert.equal(contexts[0].reads, reads);
assert.equal(frames.length, 2, 'a revoked frame cannot rearm itself');
frames[1]();
assert.equal(contexts[1].reads, 2);
assert.equal(frames.length, 3, 'the other meter retains its own render lifetime');
second.stopVoiceVisualizer();
assert.deepEqual(contexts.map(c => c.closes), [1, 1]);
});