mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
1135 lines
37 KiB
JavaScript
1135 lines
37 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import vm from 'node:vm';
|
|
import {
|
|
createGeminiAudio,
|
|
decodeGeminiPcm,
|
|
pcmBytesToBase64,
|
|
} from './geminiAudio.js';
|
|
|
|
test('PCM decoding preserves little-endian signed samples and rejects malformed audio', () => {
|
|
const data = new Uint8Array([0, 128, 0, 0, 255, 127]);
|
|
const decoded = decodeGeminiPcm(pcmBytesToBase64(data.buffer));
|
|
assert.equal(decoded.sampleRate, 24000);
|
|
assert.deepEqual([...decoded.samples], [-1, 0, 32767 / 32768]);
|
|
assert.throws(() => decodeGeminiPcm('AA=='), /incomplete/);
|
|
assert.throws(() => decodeGeminiPcm('AAAA', 'audio/mp3'), /unsupported/);
|
|
assert.throws(
|
|
() => decodeGeminiPcm('AAAA', 'audio/pcm;rate=1'),
|
|
/sample rate/,
|
|
);
|
|
});
|
|
|
|
test('PCM decoding accepts both rate endpoints and the exact decoded chunk limit', () => {
|
|
for (const rate of [8000, 96000]) {
|
|
const decoded = decodeGeminiPcm('AAA=', `audio/pcm;rate=${rate}`);
|
|
assert.equal(decoded.sampleRate, rate);
|
|
assert.deepEqual([...decoded.samples], [0]);
|
|
}
|
|
const limit = 2 * 1024 * 1024;
|
|
const maximum = decodeGeminiPcm(
|
|
pcmBytesToBase64(new ArrayBuffer(limit)),
|
|
'audio/pcm;rate=96000',
|
|
);
|
|
assert.equal(maximum.samples.length, limit / 2);
|
|
for (const size of [limit + 1, limit + 2])
|
|
assert.throws(
|
|
() => decodeGeminiPcm(pcmBytesToBase64(new ArrayBuffer(size))),
|
|
/oversized audio chunk/,
|
|
);
|
|
});
|
|
|
|
function createCaptureHarness(sampleRate = 16000) {
|
|
let Processor;
|
|
const messages = [];
|
|
vm.runInNewContext(
|
|
fs.readFileSync(
|
|
new URL('./geminiCaptureWorklet.js', import.meta.url),
|
|
'utf8',
|
|
),
|
|
{
|
|
sampleRate,
|
|
AudioWorkletProcessor: class {
|
|
constructor() {
|
|
this.port = { postMessage: (message) => messages.push(message) };
|
|
}
|
|
},
|
|
registerProcessor: (_name, Class) => {
|
|
Processor = Class;
|
|
},
|
|
},
|
|
);
|
|
return { capture: new Processor(), messages };
|
|
}
|
|
|
|
test('worklet orders start, bounded PCM and the final partial chunk before end', () => {
|
|
const { capture, messages } = createCaptureHarness();
|
|
capture.process([[new Float32Array([-1, 0, 1])]]);
|
|
assert.equal(messages.length, 0);
|
|
capture.port.onmessage({ data: { enabled: true, captureId: 1 } });
|
|
capture.process([[new Float32Array([-1, 0, 1])]]);
|
|
capture.port.onmessage({ data: { enabled: false, captureId: 1 } });
|
|
assert.deepEqual(
|
|
messages.map((message) => message.type),
|
|
['start', 'audio', 'end'],
|
|
);
|
|
const view = new DataView(messages[1].buffer);
|
|
assert.deepEqual(
|
|
[view.getInt16(0, true), view.getInt16(2, true), view.getInt16(4, true)],
|
|
[-32768, 0, 32767],
|
|
);
|
|
capture.process([[new Float32Array(2048)]]);
|
|
assert.equal(messages.length, 3);
|
|
});
|
|
|
|
test('worklet serializes rapid re-holds and ignores repeated enable or release', () => {
|
|
const { capture, messages } = createCaptureHarness();
|
|
let captureId = 0;
|
|
let gate = false;
|
|
const enable = (enabled) => {
|
|
if (enabled && !gate) captureId++;
|
|
gate = enabled;
|
|
capture.port.onmessage({ data: { enabled, captureId } });
|
|
};
|
|
enable(false);
|
|
enable(true);
|
|
enable(true);
|
|
capture.process([[new Float32Array(1025).fill(0.5)]]);
|
|
enable(false);
|
|
enable(false);
|
|
capture.process([[new Float32Array(2048).fill(1)]]);
|
|
enable(true);
|
|
capture.process([[new Float32Array([0.25])]]);
|
|
enable(false);
|
|
assert.deepEqual(
|
|
messages.map((message) => message.type),
|
|
['start', 'audio', 'audio', 'end', 'start', 'audio', 'end'],
|
|
);
|
|
assert.deepEqual(
|
|
messages
|
|
.filter((message) => message.type === 'audio')
|
|
.map((message) => message.buffer.byteLength),
|
|
[2048, 2, 2],
|
|
);
|
|
assert.equal(new DataView(messages[5].buffer).getInt16(0, true), 8192);
|
|
assert.deepEqual(
|
|
messages.map((message) => message.captureId),
|
|
[1, 1, 1, 1, 2, 2, 2],
|
|
);
|
|
enable(true);
|
|
enable(false);
|
|
assert.deepEqual(
|
|
messages.slice(-2).map((message) => message.type),
|
|
['start', 'end'],
|
|
);
|
|
assert.equal(capture.length, 0);
|
|
assert.equal(capture.enabled, false);
|
|
});
|
|
|
|
function createPlaybackHarness() {
|
|
const sources = [];
|
|
const buffers = [];
|
|
let drains = 0;
|
|
let context;
|
|
const contexts = [];
|
|
const node = () => ({
|
|
connect() {},
|
|
disconnect() {
|
|
this.disconnected = true;
|
|
},
|
|
gain: { value: 1 },
|
|
});
|
|
class Context {
|
|
constructor(options) {
|
|
context = this;
|
|
contexts.push(this);
|
|
this.options = options;
|
|
this.currentTime = 0;
|
|
this.sampleRate = options?.sampleRate ?? 48000;
|
|
this.state = 'running';
|
|
this.destination = {};
|
|
this.audioWorklet = { addModule: async () => {} };
|
|
}
|
|
async resume() {}
|
|
async close() {
|
|
this.state = 'closed';
|
|
}
|
|
createGain() {
|
|
return node();
|
|
}
|
|
createMediaStreamDestination() {
|
|
return { ...node(), stream: {} };
|
|
}
|
|
createBuffer(_channels, length, rate) {
|
|
const buffer = { duration: length / rate, copyToChannel() {} };
|
|
buffers.push(buffer);
|
|
return buffer;
|
|
}
|
|
createBufferSource() {
|
|
const source = {
|
|
...node(),
|
|
start(at) {
|
|
this.at = at;
|
|
},
|
|
stop() {
|
|
this.stopped = true;
|
|
},
|
|
};
|
|
sources.push(source);
|
|
return source;
|
|
}
|
|
}
|
|
const audio = createGeminiAudio({
|
|
AudioContextClass: Context,
|
|
AudioWorkletNodeClass: class {},
|
|
onDrain: () => {
|
|
drains++;
|
|
},
|
|
});
|
|
return {
|
|
audio,
|
|
sources,
|
|
buffers,
|
|
contexts,
|
|
get drains() {
|
|
return drains;
|
|
},
|
|
get context() {
|
|
return context;
|
|
},
|
|
};
|
|
}
|
|
|
|
function outputDiagnostics(audio) {
|
|
const { outputSamples, completedOutputSamples, discardedOutputSamples } =
|
|
audio.diagnostics();
|
|
return { outputSamples, completedOutputSamples, discardedOutputSamples };
|
|
}
|
|
|
|
test('pending sources are capped before decoding or allocating and natural completion frees a slot', async (t) => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources, buffers } = playback;
|
|
t.after(() => audio.close());
|
|
const decode = t.mock.method(globalThis, 'atob', globalThis.atob);
|
|
await audio.initialize();
|
|
for (let index = 0; index < 256; index++)
|
|
audio.play('AAA=', 'audio/pcm;rate=96000');
|
|
assert.equal(audio.pending, true);
|
|
assert.ok(audio.diagnostics().queuedSeconds < 0.023);
|
|
assert.equal(decode.mock.calls.length, 256);
|
|
assert.equal(buffers.length, 256);
|
|
assert.equal(sources.length, 256);
|
|
const before = audio.diagnostics();
|
|
assert.throws(
|
|
() => audio.play('AAA=', 'audio/pcm;rate=96000'),
|
|
/Gemini audio playback fell behind\. Reconnect to continue\./,
|
|
);
|
|
assert.equal(decode.mock.calls.length, 256);
|
|
assert.equal(buffers.length, 256);
|
|
assert.equal(sources.length, 256);
|
|
assert.deepEqual(audio.diagnostics(), before);
|
|
|
|
const ended = sources[0].onended;
|
|
ended();
|
|
assert.equal(sources[0].disconnected, true);
|
|
assert.equal(playback.drains, 0);
|
|
audio.play('AAA=', 'audio/pcm;rate=96000');
|
|
assert.equal(decode.mock.calls.length, 257);
|
|
assert.equal(buffers.length, 257);
|
|
assert.equal(sources.length, 257);
|
|
ended();
|
|
assert.equal(audio.diagnostics().completedOutputSamples, 1);
|
|
assert.throws(() => audio.play('AAA='), /playback fell behind/);
|
|
assert.equal(decode.mock.calls.length, 257);
|
|
});
|
|
|
|
test('clearing and closing a full queue release every source and ignore late completion', async (t) => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources, buffers } = playback;
|
|
t.after(() => audio.close());
|
|
const decode = t.mock.method(globalThis, 'atob', globalThis.atob);
|
|
await audio.initialize();
|
|
for (let index = 0; index < 256; index++) audio.play('AAA=');
|
|
const staleEnded = sources.map((source) => source.onended);
|
|
audio.clearPlayback();
|
|
assert.equal(audio.pending, false);
|
|
assert.equal(audio.diagnostics().queuedSeconds, 0);
|
|
for (const source of sources) {
|
|
assert.equal(source.stopped, true);
|
|
assert.equal(source.disconnected, true);
|
|
assert.equal(source.onended, null);
|
|
}
|
|
for (let index = 0; index < 256; index++) audio.play('AAA=');
|
|
for (const ended of staleEnded) ended();
|
|
assert.equal(audio.pending, true);
|
|
assert.throws(() => audio.play('AAA='), /playback fell behind/);
|
|
assert.equal(decode.mock.calls.length, 512);
|
|
assert.equal(buffers.length, 512);
|
|
assert.equal(sources.length, 512);
|
|
const lateEnded = sources.slice(256).map((source) => source.onended);
|
|
await audio.close();
|
|
for (const source of sources) {
|
|
assert.equal(source.stopped, true);
|
|
assert.equal(source.disconnected, true);
|
|
assert.equal(source.onended, null);
|
|
}
|
|
for (const ended of [...staleEnded, ...lateEnded]) ended();
|
|
audio.play('not base64');
|
|
assert.equal(decode.mock.calls.length, 512);
|
|
assert.equal(buffers.length, 512);
|
|
assert.equal(sources.length, 512);
|
|
assert.equal(audio.pending, false);
|
|
assert.equal(playback.drains, 0);
|
|
assert.equal(playback.context.state, 'closed');
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 512,
|
|
completedOutputSamples: 0,
|
|
discardedOutputSamples: 512,
|
|
});
|
|
});
|
|
|
|
test('the duration limit still rejects long chunks below the source cap', async (t) => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources, buffers } = playback;
|
|
t.after(() => audio.close());
|
|
await audio.initialize();
|
|
const second = pcmBytesToBase64(new ArrayBuffer(16000));
|
|
for (let index = 0; index < 59; index++)
|
|
audio.play(second, 'audio/pcm;rate=8000');
|
|
assert.ok(Math.abs(audio.diagnostics().queuedSeconds - 59.02) < 1e-9);
|
|
assert.throws(
|
|
() => audio.play(second, 'audio/pcm;rate=8000'),
|
|
/playback fell behind/,
|
|
);
|
|
assert.equal(buffers.length, 59);
|
|
assert.equal(sources.length, 59);
|
|
});
|
|
|
|
test('audio playback records completed samples only as sources naturally end', async () => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources } = playback;
|
|
await audio.initialize();
|
|
audio.play('AAA=');
|
|
audio.play('AAA=');
|
|
assert.ok(sources[1].at > sources[0].at);
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 2,
|
|
completedOutputSamples: 0,
|
|
discardedOutputSamples: 0,
|
|
});
|
|
sources[0].onended();
|
|
assert.equal(playback.drains, 0);
|
|
assert.equal(audio.diagnostics().completedOutputSamples, 1);
|
|
sources[1].onended();
|
|
assert.equal(playback.drains, 1);
|
|
assert.equal(audio.pending, false);
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 2,
|
|
completedOutputSamples: 2,
|
|
discardedOutputSamples: 0,
|
|
});
|
|
await audio.close();
|
|
assert.equal(playback.context.state, 'closed');
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 2,
|
|
completedOutputSamples: 2,
|
|
discardedOutputSamples: 0,
|
|
});
|
|
});
|
|
|
|
test('interruption records queued samples as discarded and rejects stale ended callbacks', async () => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources } = playback;
|
|
await audio.initialize();
|
|
audio.play('AAA=');
|
|
audio.play(pcmBytesToBase64(new ArrayBuffer(4)));
|
|
sources[0].onended();
|
|
const staleEnded = sources[1].onended;
|
|
audio.clearPlayback();
|
|
assert.equal(sources[1].stopped, true);
|
|
assert.equal(sources[1].onended, null);
|
|
assert.equal(audio.pending, false);
|
|
assert.equal(audio.diagnostics().queuedSeconds, 0);
|
|
staleEnded();
|
|
audio.clearPlayback();
|
|
assert.equal(playback.drains, 0);
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 3,
|
|
completedOutputSamples: 1,
|
|
discardedOutputSamples: 2,
|
|
});
|
|
await audio.close();
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 3,
|
|
completedOutputSamples: 1,
|
|
discardedOutputSamples: 2,
|
|
});
|
|
});
|
|
|
|
test('closing queued playback cannot count stopped sources as completed audio', async () => {
|
|
const playback = createPlaybackHarness();
|
|
const { audio, sources } = playback;
|
|
await audio.initialize();
|
|
audio.play('AAA=');
|
|
audio.play(pcmBytesToBase64(new ArrayBuffer(4)));
|
|
const staleEnded = sources.map((source) => source.onended);
|
|
await audio.close();
|
|
for (const source of sources) {
|
|
assert.equal(source.stopped, true);
|
|
assert.equal(source.onended, null);
|
|
}
|
|
for (const ended of staleEnded) ended();
|
|
audio.play('AAA=');
|
|
await audio.close();
|
|
assert.equal(audio.pending, false);
|
|
assert.equal(playback.drains, 0);
|
|
assert.equal(audio.diagnostics().contextState, 'closed');
|
|
assert.equal(audio.diagnostics().queuedSeconds, 0);
|
|
assert.deepEqual(outputDiagnostics(audio), {
|
|
outputSamples: 3,
|
|
completedOutputSamples: 0,
|
|
discardedOutputSamples: 3,
|
|
});
|
|
});
|
|
|
|
test('capture diagnostics distinguish speech energy from silence without retaining audio', async () => {
|
|
let capture;
|
|
const emitted = [];
|
|
const order = [];
|
|
const identities = [];
|
|
const gates = [];
|
|
const node = () => ({ connect() {}, disconnect() {}, gain: { value: 1 } });
|
|
class Context {
|
|
constructor() {
|
|
this.currentTime = 0;
|
|
this.sampleRate = 16000;
|
|
this.state = 'running';
|
|
this.destination = {};
|
|
this.audioWorklet = { addModule: async () => {} };
|
|
}
|
|
async resume() {}
|
|
async close() {
|
|
this.state = 'closed';
|
|
}
|
|
createGain() {
|
|
return node();
|
|
}
|
|
createMediaStreamDestination() {
|
|
return { ...node(), stream: { getTracks: () => [] } };
|
|
}
|
|
createMediaStreamSource() {
|
|
return node();
|
|
}
|
|
}
|
|
class Worklet {
|
|
constructor() {
|
|
capture = this;
|
|
Object.assign(this, node());
|
|
this.port = {
|
|
postMessage(message) {
|
|
gates.push(message);
|
|
},
|
|
close() {},
|
|
};
|
|
}
|
|
}
|
|
const audio = createGeminiAudio({
|
|
AudioContextClass: Context,
|
|
AudioWorkletNodeClass: Worklet,
|
|
onAudioStart: (metadata) => {
|
|
identities.push(metadata.captureId);
|
|
order.push('start');
|
|
},
|
|
onAudio: (chunk, metadata) => {
|
|
identities.push(metadata.captureId);
|
|
emitted.push(chunk);
|
|
order.push('audio');
|
|
},
|
|
onAudioEnd: (metadata) => {
|
|
identities.push(metadata.captureId);
|
|
order.push('end');
|
|
},
|
|
});
|
|
await audio.initialize();
|
|
audio.connectMicrophone({}, true, { captureId: 11 });
|
|
const deliver = capture.port.onmessage;
|
|
deliver({ data: { type: 'start', captureId: 11 } });
|
|
capture.port.onmessage({
|
|
data: { type: 'audio', captureId: 11, buffer: new ArrayBuffer(4) },
|
|
});
|
|
const spoken = new ArrayBuffer(4);
|
|
new DataView(spoken).setInt16(0, 16384, true);
|
|
capture.port.onmessage({
|
|
data: { type: 'audio', captureId: 11, buffer: spoken },
|
|
});
|
|
deliver({ data: { type: 'end', captureId: 11 } });
|
|
assert.deepEqual(order, ['start', 'audio', 'audio', 'end']);
|
|
assert.deepEqual(identities, [11, 11, 11, 11]);
|
|
assert.deepEqual(gates, [{ enabled: true, captureId: 11 }]);
|
|
// Gate D does not relabel delayed B port messages.
|
|
audio.setMicrophoneEnabled(false, { captureId: 11 });
|
|
audio.setMicrophoneEnabled(true, { captureId: 12 });
|
|
deliver({ data: { type: 'start', captureId: 11 } });
|
|
deliver({ data: { type: 'audio', captureId: 11, buffer: spoken } });
|
|
deliver({ data: { type: 'end', captureId: 11 } });
|
|
assert.deepEqual(identities.slice(-3), [11, 11, 11]);
|
|
const count = order.length;
|
|
for (const captureId of [undefined, null, 0, -1, 1.5, '12', Infinity]) {
|
|
deliver({ data: { type: 'start', captureId } });
|
|
deliver({ data: { type: 'audio', captureId, buffer: spoken } });
|
|
deliver({ data: { type: 'end', captureId } });
|
|
}
|
|
assert.equal(order.length, count);
|
|
assert.equal(audio.diagnostics().inputChunks, 3);
|
|
assert.equal(audio.diagnostics().nonSilentInputChunks, 2);
|
|
assert.equal(audio.diagnostics().inputPeak, 0.5);
|
|
assert.equal(emitted.length, 3);
|
|
assert.equal(emitted[1].mimeType, 'audio/pcm;rate=16000');
|
|
assert.deepEqual(Object.keys(emitted[1]).sort(), ['data', 'mimeType']);
|
|
assert.equal(await audio.close(), true);
|
|
assert.equal(audio.diagnostics().contextState, 'closed');
|
|
deliver({ data: { type: 'start', captureId: 11 } });
|
|
deliver({ data: { type: 'audio', captureId: 11, buffer: spoken } });
|
|
deliver({ data: { type: 'end', captureId: 11 } });
|
|
assert.deepEqual(order, [
|
|
'start',
|
|
'audio',
|
|
'audio',
|
|
'end',
|
|
'start',
|
|
'audio',
|
|
'end',
|
|
]);
|
|
});
|
|
|
|
test('worklet rejects missing identity and stale release without changing the active capture', () => {
|
|
const { capture, messages } = createCaptureHarness();
|
|
for (const captureId of [undefined, null, 0, -1, 1.5, '1', Infinity])
|
|
capture.port.onmessage({ data: { enabled: true, captureId } });
|
|
capture.process([[new Float32Array([0.5])]]);
|
|
assert.equal(messages.length, 0);
|
|
capture.port.onmessage({ data: { enabled: true, captureId: 20 } });
|
|
capture.process([[new Float32Array([0.5])]]);
|
|
capture.port.onmessage({ data: { enabled: true, captureId: 21 } });
|
|
capture.port.onmessage({ data: { enabled: false, captureId: 19 } });
|
|
assert.equal(messages.length, 1);
|
|
capture.port.onmessage({ data: { enabled: false, captureId: 20 } });
|
|
assert.deepEqual(
|
|
messages.map((message) => [message.type, message.captureId]),
|
|
[
|
|
['start', 20],
|
|
['audio', 20],
|
|
['end', 20],
|
|
],
|
|
);
|
|
capture.port.onmessage({ data: { enabled: true, captureId: 21 } });
|
|
capture.port.onmessage({ data: { enabled: false, captureId: 20 } });
|
|
capture.process([[new Float32Array([0.25])]]);
|
|
capture.port.onmessage({ data: { enabled: false, captureId: 21 } });
|
|
assert.deepEqual(
|
|
messages.slice(3).map((message) => [message.type, message.captureId]),
|
|
[
|
|
['start', 21],
|
|
['audio', 21],
|
|
['end', 21],
|
|
],
|
|
);
|
|
});
|
|
|
|
function capturedPcm(messages, captureId) {
|
|
const samples = [];
|
|
for (const message of messages) {
|
|
if (message.type !== 'audio' || message.captureId !== captureId) continue;
|
|
const view = new DataView(message.buffer);
|
|
for (let index = 0; index < view.byteLength; index += 2)
|
|
samples.push(view.getInt16(index, true));
|
|
}
|
|
return samples;
|
|
}
|
|
|
|
function captureSignal(rate, signal, blockSizes = [128]) {
|
|
const harness = createCaptureHarness(rate);
|
|
harness.capture.port.onmessage({ data: { enabled: true, captureId: 1 } });
|
|
let block = 0;
|
|
for (let offset = 0; offset < signal.length;) {
|
|
const size = blockSizes[block++ % blockSizes.length];
|
|
harness.capture.process([[signal.subarray(offset, offset + size)]]);
|
|
offset += size;
|
|
}
|
|
harness.capture.port.onmessage({ data: { enabled: false, captureId: 1 } });
|
|
return { ...harness, pcm: capturedPcm(harness.messages, 1) };
|
|
}
|
|
|
|
for (const rate of [44100, 48000, 96000]) {
|
|
test(`worklet resamples ${rate} Hz to bounded 16 kHz PCM independently of block boundaries`, () => {
|
|
const signal = Float32Array.from(
|
|
{ length: rate },
|
|
(_, index) => 0.5 * Math.sin((2 * Math.PI * 1000 * index) / rate),
|
|
);
|
|
const fragmented = captureSignal(rate, signal, [1, 127, 257, 3, 128]);
|
|
const contiguous = captureSignal(rate, signal, [rate]);
|
|
assert.equal(fragmented.pcm.length, 16000);
|
|
assert.deepEqual(fragmented.pcm, contiguous.pcm);
|
|
assert.deepEqual(
|
|
fragmented.messages.map((message) => message.type).slice(0, 2),
|
|
['start', 'audio'],
|
|
);
|
|
assert.equal(fragmented.messages.at(-1).type, 'end');
|
|
assert.ok(fragmented.messages.every((message) => message.captureId === 1));
|
|
assert.ok(
|
|
fragmented.messages
|
|
.filter((message) => message.type === 'audio')
|
|
.every((message) => message.buffer.byteLength <= 2048),
|
|
);
|
|
const rms = Math.sqrt(
|
|
fragmented.pcm
|
|
.slice(256)
|
|
.reduce((total, sample) => total + (sample / 32768) ** 2, 0) /
|
|
(fragmented.pcm.length - 256),
|
|
);
|
|
assert.ok(
|
|
Math.abs(rms - Math.SQRT1_2 * 0.5) < 0.01,
|
|
`speech-band RMS ${rms}`,
|
|
);
|
|
// Check the actual waveform against its expected frequency and causal
|
|
// filter delay, rather than accepting any signal with the same energy.
|
|
const error = Math.sqrt(
|
|
fragmented.pcm.slice(256).reduce((total, sample, index) => {
|
|
const time = (index + 257) / 16000 - 32.5 / rate;
|
|
const expected = 0.5 * Math.sin(2 * Math.PI * 1000 * time);
|
|
return total + (sample / 32768 - expected) ** 2;
|
|
}, 0) /
|
|
(fragmented.pcm.length - 256),
|
|
);
|
|
assert.ok(error < 0.005, `resampled waveform error ${error}`);
|
|
});
|
|
}
|
|
|
|
for (const rate of [44100, 48000]) {
|
|
test(`worklet suppresses 12 kHz aliasing from a ${rate} Hz microphone`, () => {
|
|
const signal = Float32Array.from(
|
|
{ length: rate },
|
|
(_, index) => 0.5 * Math.sin((2 * Math.PI * 12000 * index) / rate),
|
|
);
|
|
const { pcm } = captureSignal(rate, signal);
|
|
const rms = Math.sqrt(
|
|
pcm
|
|
.slice(256)
|
|
.reduce((total, sample) => total + (sample / 32768) ** 2, 0) /
|
|
(pcm.length - 256),
|
|
);
|
|
assert.ok(rms < 0.005, `out-of-band RMS ${rms}`);
|
|
});
|
|
}
|
|
|
|
test('resampler release flushes only owned PCM and clears phase/history before a successor', () => {
|
|
const { capture, messages } = createCaptureHarness(44100);
|
|
const gate = (enabled, captureId) =>
|
|
capture.port.onmessage({ data: { enabled, captureId } });
|
|
gate(true, 10);
|
|
capture.process([[new Float32Array(442).fill(1)]]);
|
|
gate(false, 9); // A stale release cannot clear the active filter or phase.
|
|
capture.process([[new Float32Array(441).fill(1)]]);
|
|
gate(false, 10);
|
|
assert.equal(
|
|
capturedPcm(messages, 10).length,
|
|
Math.floor((883 * 16000) / 44100),
|
|
);
|
|
gate(true, 11);
|
|
gate(false, 10);
|
|
capture.process([[new Float32Array(441)]]);
|
|
gate(false, 11);
|
|
const successor = capturedPcm(messages, 11);
|
|
assert.equal(successor.length, 160);
|
|
assert.ok(successor.every((sample) => sample === 0));
|
|
assert.deepEqual(
|
|
messages.slice(-3).map((message) => [message.type, message.captureId]),
|
|
[
|
|
['start', 11],
|
|
['audio', 11],
|
|
['end', 11],
|
|
],
|
|
);
|
|
// A sub-output-sample hold has no residual PCM or phase to transfer.
|
|
gate(true, 12);
|
|
capture.process([[new Float32Array([1])]]);
|
|
gate(false, 12);
|
|
gate(true, 13);
|
|
capture.process([[new Float32Array(441)]]);
|
|
gate(false, 13);
|
|
assert.deepEqual(capturedPcm(messages, 13), successor);
|
|
assert.deepEqual(
|
|
messages
|
|
.filter((message) => message.captureId === 12)
|
|
.map((message) => message.type),
|
|
['start', 'end'],
|
|
);
|
|
});
|
|
|
|
function createContextLifecycleHarness({
|
|
resumeFailure,
|
|
moduleFailure = false,
|
|
closeFailure,
|
|
constructorFailure,
|
|
actualPlaybackRate = 24000,
|
|
} = {}) {
|
|
const contexts = [];
|
|
const worklets = [];
|
|
const events = [];
|
|
let stoppedTracks = 0;
|
|
const node = (context) => ({
|
|
context,
|
|
gain: { value: 1 },
|
|
connections: [],
|
|
connect(target) {
|
|
this.connections.push(target);
|
|
},
|
|
disconnect() {
|
|
this.disconnected = true;
|
|
},
|
|
});
|
|
class Context {
|
|
constructor(options) {
|
|
const index = contexts.length;
|
|
if (constructorFailure === index)
|
|
throw new Error('context creation failed');
|
|
this.index = index;
|
|
this.options = options;
|
|
this.sampleRate = index === 0 ? 44100 : actualPlaybackRate;
|
|
this.currentTime = index === 0 ? 99 : 7;
|
|
this.state = 'suspended';
|
|
this.destination = { context: this };
|
|
this.nodes = [];
|
|
this.closeCalls = 0;
|
|
this.audioWorklet = {
|
|
addModule: async () => {
|
|
events.push(`module:${index}`);
|
|
if (moduleFailure) throw new Error('module failed');
|
|
},
|
|
};
|
|
contexts.push(this);
|
|
}
|
|
resume() {
|
|
events.push(`resume:${this.index}`);
|
|
if (resumeFailure === this.index) throw new Error('resume failed');
|
|
this.state = 'running';
|
|
return Promise.resolve();
|
|
}
|
|
close() {
|
|
this.closeCalls++;
|
|
events.push(`close:${this.index}`);
|
|
if (closeFailure === this.index) throw new Error('close failed');
|
|
this.state = 'closed';
|
|
return Promise.resolve();
|
|
}
|
|
createGain() {
|
|
const result = node(this);
|
|
this.nodes.push(result);
|
|
return result;
|
|
}
|
|
createMediaStreamDestination() {
|
|
const result = {
|
|
...node(this),
|
|
stream: {
|
|
getTracks: () => [
|
|
{
|
|
stop: () => {
|
|
stoppedTracks++;
|
|
},
|
|
},
|
|
],
|
|
},
|
|
};
|
|
this.nodes.push(result);
|
|
return result;
|
|
}
|
|
createMediaStreamSource() {
|
|
const result = node(this);
|
|
this.nodes.push(result);
|
|
return result;
|
|
}
|
|
createBuffer(_channels, length, rate) {
|
|
return {
|
|
context: this,
|
|
rate,
|
|
duration: length / rate,
|
|
copyToChannel() {},
|
|
};
|
|
}
|
|
createBufferSource() {
|
|
const result = {
|
|
...node(this),
|
|
start(at) {
|
|
this.at = at;
|
|
},
|
|
stop() {
|
|
this.stopped = true;
|
|
},
|
|
};
|
|
this.nodes.push(result);
|
|
return result;
|
|
}
|
|
}
|
|
class Worklet {
|
|
constructor(context) {
|
|
Object.assign(this, node(context));
|
|
this.gates = [];
|
|
this.port = {
|
|
postMessage: (message) => this.gates.push(message),
|
|
close: () => {
|
|
this.portClosed = true;
|
|
},
|
|
};
|
|
worklets.push(this);
|
|
}
|
|
}
|
|
return {
|
|
contexts,
|
|
worklets,
|
|
events,
|
|
create: (callbacks = {}) =>
|
|
createGeminiAudio({
|
|
AudioContextClass: Context,
|
|
AudioWorkletNodeClass: Worklet,
|
|
...callbacks,
|
|
}),
|
|
get stoppedTracks() {
|
|
return stoppedTracks;
|
|
},
|
|
};
|
|
}
|
|
|
|
test('capture uses the device context while playback owns its clock, stream and actual rate diagnostics', async () => {
|
|
const harness = createContextLifecycleHarness({ actualPlaybackRate: 48000 });
|
|
const chunks = [];
|
|
const errors = [];
|
|
const audio = harness.create({
|
|
onAudio: (chunk) => chunks.push(chunk),
|
|
onError: (error) => errors.push(error),
|
|
});
|
|
assert.deepEqual(harness.events, ['resume:0', 'resume:1']);
|
|
assert.equal(harness.contexts[0].options, undefined);
|
|
assert.deepEqual(harness.contexts[1].options, { sampleRate: 24000 });
|
|
await audio.initialize();
|
|
assert.deepEqual(harness.events, ['resume:0', 'resume:1', 'module:0']);
|
|
audio.connectMicrophone({}, true, { captureId: 5 });
|
|
const capture = harness.worklets[0];
|
|
assert.equal(capture.context, harness.contexts[0]);
|
|
assert.deepEqual(capture.gates, [{ enabled: true, captureId: 5 }]);
|
|
const staleError = capture.onprocessorerror;
|
|
capture.port.onmessage({
|
|
data: { type: 'audio', captureId: 5, buffer: new ArrayBuffer(4) },
|
|
});
|
|
assert.equal(chunks[0].mimeType, 'audio/pcm;rate=16000');
|
|
audio.play('AAA=');
|
|
const source = harness.contexts[1].nodes.find((item) => item.buffer);
|
|
assert.equal(source.context, harness.contexts[1]);
|
|
assert.equal(source.buffer.context, harness.contexts[1]);
|
|
assert.equal(source.buffer.rate, 24000);
|
|
assert.equal(source.at, 7.02);
|
|
assert.ok(audio.diagnostics().queuedSeconds < 0.021);
|
|
assert.equal(audio.diagnostics().captureSampleRate, 44100);
|
|
assert.equal(audio.diagnostics().playbackSampleRate, 48000);
|
|
assert.equal(audio.diagnostics().inputSampleRate, 16000);
|
|
assert.equal(audio.diagnostics().contextState, 'running');
|
|
harness.contexts[0].state = 'suspended';
|
|
assert.equal(audio.diagnostics().contextState, 'suspended');
|
|
const closing = audio.close();
|
|
assert.equal(audio.close(), closing);
|
|
assert.equal(await closing, true);
|
|
assert.ok(
|
|
harness.contexts.every(
|
|
(context) => context.state === 'closed' && context.closeCalls === 1,
|
|
),
|
|
);
|
|
assert.equal(harness.stoppedTracks, 1);
|
|
assert.equal(capture.portClosed, true);
|
|
assert.equal(capture.disconnected, true);
|
|
assert.equal(capture.onprocessorerror, null);
|
|
assert.equal(source.stopped, true);
|
|
staleError();
|
|
audio.setMicrophoneEnabled(true, { captureId: 6 });
|
|
assert.equal(capture.gates.length, 1);
|
|
assert.deepEqual(errors, []);
|
|
});
|
|
|
|
for (const resumeFailure of [0, 1]) {
|
|
test(`resume failure in context ${resumeFailure} releases both contexts`, async () => {
|
|
const harness = createContextLifecycleHarness({ resumeFailure });
|
|
const audio = harness.create();
|
|
assert.deepEqual(harness.events, ['resume:0', 'resume:1']);
|
|
await assert.rejects(audio.initialize(), /resume failed/);
|
|
assert.ok(
|
|
harness.contexts.every(
|
|
(context) => context.state === 'closed' && context.closeCalls === 1,
|
|
),
|
|
);
|
|
assert.equal(harness.stoppedTracks, 1);
|
|
assert.equal(await audio.close(), true);
|
|
});
|
|
}
|
|
|
|
test('module failure and aborted initialization release both capture and playback', async () => {
|
|
for (const moduleFailure of [true, false]) {
|
|
const harness = createContextLifecycleHarness({ moduleFailure });
|
|
const audio = harness.create();
|
|
const signal = moduleFailure ? undefined : AbortSignal.abort();
|
|
await assert.rejects(
|
|
audio.initialize(signal),
|
|
moduleFailure ? /module failed/ : { name: 'AbortError' },
|
|
);
|
|
assert.ok(
|
|
harness.contexts.every(
|
|
(context) => context.state === 'closed' && context.closeCalls === 1,
|
|
),
|
|
);
|
|
assert.equal(harness.stoppedTracks, 1);
|
|
}
|
|
});
|
|
|
|
for (const closeFailure of [0, 1]) {
|
|
test(`context ${closeFailure} close failure does not prevent closing the other context`, async () => {
|
|
const harness = createContextLifecycleHarness({ closeFailure });
|
|
const audio = harness.create();
|
|
await audio.initialize();
|
|
const closing = audio.close();
|
|
assert.equal(audio.close(), closing);
|
|
assert.equal(await closing, false);
|
|
assert.ok(harness.contexts.every((context) => context.closeCalls === 1));
|
|
assert.equal(harness.contexts[1 - closeFailure].state, 'closed');
|
|
assert.equal(harness.stoppedTracks, 1);
|
|
});
|
|
}
|
|
|
|
test('partial context construction failure closes the previously acquired capture context', async () => {
|
|
const harness = createContextLifecycleHarness({ constructorFailure: 1 });
|
|
assert.throws(() => harness.create(), /context creation failed/);
|
|
assert.equal(harness.contexts.length, 1);
|
|
assert.equal(harness.contexts[0].closeCalls, 1);
|
|
assert.equal(harness.contexts[0].state, 'closed');
|
|
});
|
|
|
|
function admissionHarness({ pushToTalk = true, callbacks = {} } = {}) {
|
|
const events = [];
|
|
const worklets = [];
|
|
const node = () => ({ connect() {}, disconnect() {}, gain: { value: 1 } });
|
|
class Context {
|
|
constructor() {
|
|
this.sampleRate = 16000;
|
|
this.currentTime = 0;
|
|
this.state = 'running';
|
|
this.destination = {};
|
|
this.audioWorklet = { addModule: async () => {} };
|
|
}
|
|
async resume() {}
|
|
async close() {
|
|
this.state = 'closed';
|
|
}
|
|
createGain() {
|
|
return node();
|
|
}
|
|
createMediaStreamDestination() {
|
|
return { ...node(), stream: { getTracks: () => [] } };
|
|
}
|
|
createMediaStreamSource() {
|
|
return node();
|
|
}
|
|
createBuffer(_channels, samples, rate) {
|
|
return { duration: samples / rate, copyToChannel() {} };
|
|
}
|
|
createBufferSource() {
|
|
return { ...node(), start() {}, stop() {} };
|
|
}
|
|
}
|
|
class Worklet {
|
|
constructor() {
|
|
Object.assign(this, node());
|
|
worklets.push(this);
|
|
const { capture, messages } = createCaptureHarness();
|
|
this.processor = capture;
|
|
this.flush = () => {
|
|
for (const data of messages.splice(0)) {
|
|
// MessagePort structured cloning moves VM buffers into this realm.
|
|
if (data.buffer)
|
|
data.buffer = new Uint8Array(data.buffer).slice().buffer;
|
|
this.port.onmessage?.({ data });
|
|
}
|
|
};
|
|
this.port = {
|
|
postMessage: (data) => {
|
|
capture.port.onmessage({ data });
|
|
this.flush();
|
|
},
|
|
close() {},
|
|
};
|
|
}
|
|
feed(samples) {
|
|
this.processor.process([[samples]]);
|
|
this.flush();
|
|
}
|
|
}
|
|
const createAudio = (options) =>
|
|
createGeminiAudio({
|
|
...options,
|
|
AudioContextClass: Context,
|
|
AudioWorkletNodeClass: Worklet,
|
|
});
|
|
const audio = createAudio({
|
|
pushToTalk,
|
|
onAudioStart: (metadata) => {
|
|
events.push({ type: 'start', ...metadata });
|
|
callbacks.onAudioStart?.(metadata);
|
|
},
|
|
onAudio: (chunk, metadata) => {
|
|
events.push({ type: 'audio', ...metadata, chunk });
|
|
callbacks.onAudio?.(chunk, metadata);
|
|
},
|
|
onAudioEnd: (metadata) => {
|
|
events.push({ type: 'end', ...metadata });
|
|
callbacks.onAudioEnd?.(metadata);
|
|
},
|
|
onAudioDiscard: (metadata) => {
|
|
events.push({ type: 'discard', ...metadata });
|
|
callbacks.onAudioDiscard?.(metadata);
|
|
},
|
|
onError: (error) => {
|
|
throw error;
|
|
},
|
|
});
|
|
return { audio, events, worklets, createAudio };
|
|
}
|
|
function tone(length, amplitude = 0.04) {
|
|
return Float32Array.from(
|
|
{ length },
|
|
(_, i) => amplitude * Math.sin((i * 2 * Math.PI * 220) / 16000),
|
|
);
|
|
}
|
|
function emittedPcm(events) {
|
|
return events
|
|
.filter((e) => e.type === 'audio')
|
|
.flatMap((e) => {
|
|
const bytes = Uint8Array.from(atob(e.chunk.data), (c) => c.charCodeAt(0));
|
|
const view = new DataView(bytes.buffer);
|
|
return Array.from({ length: bytes.length / 2 }, (_, i) =>
|
|
view.getInt16(i * 2, true),
|
|
);
|
|
});
|
|
}
|
|
|
|
test('PTT admission suppresses zero and near-silent captures without protocol callbacks', async (t) => {
|
|
const h = admissionHarness();
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
h.worklets[0].feed(new Float32Array(1024));
|
|
h.worklets[0].feed(tone(320, 0.00018));
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
assert.deepEqual(h.events, [{ type: 'discard', captureId: 1 }]);
|
|
assert.equal(h.audio.diagnostics().suppressedPttCaptures, 1);
|
|
assert.equal(h.audio.diagnostics().pttBufferedBytes, 0);
|
|
});
|
|
|
|
test('PTT admission rejects isolated high-peak clicks across packet boundaries', async (t) => {
|
|
const h = admissionHarness();
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
const click = new Float32Array(1024);
|
|
click[159] = 1;
|
|
click[160] = -1;
|
|
h.worklets[0].feed(click);
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
assert.deepEqual(
|
|
h.events.map((e) => e.type),
|
|
['discard'],
|
|
);
|
|
});
|
|
|
|
test('PTT admission preserves quiet short speech and final partial release in exact order', async (t) => {
|
|
const h = admissionHarness();
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
const prefix = new Float32Array(73);
|
|
const speech = tone(640, 0.005);
|
|
const tail = new Float32Array(57);
|
|
h.worklets[0].feed(prefix);
|
|
h.worklets[0].feed(speech);
|
|
h.worklets[0].feed(tail);
|
|
assert.equal(h.events.length, 0, 'worklet still owns its incomplete packet');
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
assert.deepEqual(
|
|
h.events.map((e) => e.type),
|
|
['start', 'audio', 'end'],
|
|
);
|
|
const expected = [...prefix, ...speech, ...tail].map(
|
|
(v) => Math.round(v * (v < 0 ? 32768 : 32767)) || 0,
|
|
);
|
|
assert.deepEqual(emittedPcm(h.events), expected);
|
|
});
|
|
|
|
test('PTT admission bounds a long silent preroll and never regates voiced tails', async (t) => {
|
|
const h = admissionHarness();
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
for (let i = 0; i < 200; i++) {
|
|
h.worklets[0].feed(new Float32Array(1024));
|
|
assert.ok(h.audio.diagnostics().pttBufferedBytes <= 12800);
|
|
}
|
|
assert.equal(h.events.length, 0);
|
|
h.worklets[0].feed(tone(1024));
|
|
h.worklets[0].feed(new Float32Array(1024));
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
assert.equal(h.events[0].type, 'start');
|
|
assert.equal(h.events.at(-1).type, 'end');
|
|
assert.equal(h.events.filter((e) => e.type === 'start').length, 1);
|
|
const pcm = emittedPcm(h.events);
|
|
assert.ok(pcm.length <= 9 * 1024);
|
|
assert.deepEqual(pcm.slice(-1024), new Array(1024).fill(0));
|
|
assert.equal(h.audio.diagnostics().admittedPttCaptures, 1);
|
|
});
|
|
|
|
test('PTT admission isolates repeated holds, stale identities and close callbacks', async (t) => {
|
|
const h = admissionHarness();
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
h.worklets[0].feed(tone(160));
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
h.audio.setMicrophoneEnabled(true, { captureId: 2 });
|
|
const deliver = h.worklets[0].port.onmessage;
|
|
deliver({
|
|
data: { type: 'audio', captureId: 1, buffer: new ArrayBuffer(2048) },
|
|
});
|
|
deliver({ data: { type: 'end', captureId: 1 } });
|
|
h.worklets[0].feed(tone(160));
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 2 });
|
|
assert.deepEqual(
|
|
h.events.map((e) => e.type),
|
|
['discard', 'discard'],
|
|
);
|
|
h.audio.setMicrophoneEnabled(true, { captureId: 3 });
|
|
h.worklets[0].feed(new Float32Array(1024));
|
|
await h.audio.close();
|
|
deliver({ data: { type: 'audio', captureId: 3, buffer: tone(1024).buffer } });
|
|
deliver({ data: { type: 'end', captureId: 3 } });
|
|
assert.equal(h.events.length, 2);
|
|
assert.equal(h.audio.diagnostics().pttBufferedBytes, 0);
|
|
});
|
|
|
|
test('PTT admission leaves open-mic start, silence, PCM and end unchanged', async (t) => {
|
|
const h = admissionHarness({ pushToTalk: false });
|
|
t.after(() => h.audio.close());
|
|
await h.audio.initialize();
|
|
h.audio.connectMicrophone({}, true, { captureId: 1 });
|
|
h.worklets[0].feed(new Float32Array(1024));
|
|
h.audio.setMicrophoneEnabled(false, { captureId: 1 });
|
|
assert.deepEqual(
|
|
h.events.map((e) => e.type),
|
|
['start', 'audio', 'end'],
|
|
);
|
|
assert.deepEqual(emittedPcm(h.events), new Array(1024).fill(0));
|
|
});
|