mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask
- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
Bilawal Sidhu committed
1 parent
e7707d9a0f
commit
a19897ad15
97 files changed
+14343
-755
No files matched your search
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Gemini Live adapter — BidiGenerateContent over WebSocket.
|
||||
*
|
||||
* Same prompt as production (system instruction = realtimeInstructions(),
|
||||
* tools = GEV_REALTIME_TOOLS as functionDeclarations with the production JSON
|
||||
* schema passed verbatim via parametersJsonSchema). The native-audio Live
|
||||
* model rejects a TEXT response modality (close 1007), so the session
|
||||
* requests AUDIO with output transcription on; tool calls arrive as
|
||||
* `toolCall` messages and the transcript stands in for the final words.
|
||||
*
|
||||
* Fresh session per phrase / dialogue. Each toolCall gets stub
|
||||
* `functionResponses`; the model continues on its own. A turn ends at a
|
||||
* `turnComplete` whose generation had no toolCall, or after 5 tool rounds
|
||||
* (mirrors the Realtime adapter's continuation cap), or 12 s of silence after
|
||||
* a tool round.
|
||||
*/
|
||||
import WebSocket from 'ws';
|
||||
import { providerKey } from '../keys.mjs';
|
||||
import { realtimeInstructions, GEV_REALTIME_TOOLS } from '../productionConfig.mjs';
|
||||
import { toGeminiDeclarations } from '../toolFormats.mjs';
|
||||
import { geminiUsageCostUsd } from '../pricing.mjs';
|
||||
import { newTurnResult } from './openaiRealtime.mjs';
|
||||
|
||||
const ENDPOINT = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
|
||||
const MAX_TOOL_ROUNDS = 5;
|
||||
const IDLE_AFTER_TOOL_MS = 12_000;
|
||||
|
||||
export function createGeminiLiveAdapter({ model, schemaMode = 'jsonSchema', turnTimeoutMs = 60_000 }) {
|
||||
const instructions = realtimeInstructions();
|
||||
const functionDeclarations = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: schemaMode });
|
||||
const adapter = {
|
||||
provider: 'gemini-live',
|
||||
model,
|
||||
settings: {
|
||||
responseModalities: ['AUDIO'],
|
||||
outputAudioTranscription: true,
|
||||
schemaMode,
|
||||
temperature: 'server default (not set; production has no Gemini config)',
|
||||
thinking: 'server default',
|
||||
input: 'realtimeInput.text',
|
||||
context: 'fresh session per phrase / dialogue',
|
||||
},
|
||||
async open() {},
|
||||
async close() {},
|
||||
async runTurn({ text, onCall, context }) {
|
||||
return (await adapter.runDialogue({ turns: [text], onCall, contexts: [context] }))[0];
|
||||
},
|
||||
runDialogue({ turns, onCall, contexts = [] }) {
|
||||
return new Promise((resolveAll) => {
|
||||
const tSetup = Date.now();
|
||||
const ws = new WebSocket(`${ENDPOINT}?key=${providerKey('gemini')}`);
|
||||
const results = [];
|
||||
let setupMs = null;
|
||||
let cur = null;
|
||||
let closed = false;
|
||||
const send = (o) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(o)); };
|
||||
const shutdown = (error) => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
if (cur) {
|
||||
clearTimeout(cur.timer);
|
||||
clearTimeout(cur.idle);
|
||||
if (error) cur.r.errors.push(error);
|
||||
if (cur.r.tDoneMs === null) cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
cur = null;
|
||||
}
|
||||
while (results.length < turns.length) {
|
||||
const r = newTurnResult();
|
||||
r.errors.push(error || 'dialogue aborted');
|
||||
results.push(r);
|
||||
}
|
||||
try { ws.terminate(); } catch { /* noop */ }
|
||||
resolveAll(results);
|
||||
};
|
||||
const startTurn = (turn) => {
|
||||
const r = newTurnResult();
|
||||
r.setupMs = turn === 0 ? setupMs : 0;
|
||||
cur = { r, t0: Date.now(), turn, toolRounds: 0, toolCallThisGeneration: false, idle: null };
|
||||
cur.timer = setTimeout(() => { cur.r.timeout = true; endTurn(); }, turnTimeoutMs);
|
||||
// Live has no system-role mid-session item; a context item (pointer_context)
|
||||
// rides as a JSON line ahead of the user's words in the same text turn.
|
||||
const ctx = contexts[turn] ? `${JSON.stringify(contexts[turn])}\n` : '';
|
||||
send({ realtimeInput: { text: `${ctx}${turns[turn]}` } });
|
||||
};
|
||||
const endTurn = () => {
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.timer);
|
||||
clearTimeout(cur.idle);
|
||||
if (cur.r.tDoneMs === null) cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
const next = cur.turn + 1;
|
||||
const timedOut = cur.r.timeout;
|
||||
cur = null;
|
||||
if (next < turns.length && !timedOut) startTurn(next);
|
||||
else shutdown();
|
||||
};
|
||||
const mark = (key) => { if (cur && cur.r[key] === null) cur.r[key] = Date.now() - cur.t0; };
|
||||
|
||||
ws.on('open', () => {
|
||||
send({
|
||||
setup: {
|
||||
model: `models/${model}`,
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
systemInstruction: { parts: [{ text: instructions }] },
|
||||
tools: [{ functionDeclarations }],
|
||||
outputAudioTranscription: {},
|
||||
},
|
||||
});
|
||||
});
|
||||
ws.on('unexpected-response', (_req, res) => shutdown(`HTTP ${res.statusCode}`));
|
||||
ws.on('error', (e) => shutdown(`ws: ${e.message}`));
|
||||
ws.on('close', (code, reason) => {
|
||||
shutdown(results.length < turns.length ? `close ${code}: ${reason.toString().slice(0, 200)}` : undefined);
|
||||
});
|
||||
ws.on('message', (raw) => {
|
||||
let m;
|
||||
try { m = JSON.parse(raw.toString()); } catch { return; }
|
||||
if (m.setupComplete) {
|
||||
if (setupMs === null) {
|
||||
setupMs = Date.now() - tSetup;
|
||||
startTurn(0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.idle);
|
||||
const r = cur.r;
|
||||
const lastAt = Date.now();
|
||||
const sc = m.serverContent;
|
||||
if (sc?.modelTurn?.parts?.length) {
|
||||
mark('tFirstTokenMs');
|
||||
for (const p of sc.modelTurn.parts) if (typeof p.text === 'string' && !p.thought) r.text += p.text;
|
||||
}
|
||||
if (sc?.outputTranscription?.text) {
|
||||
mark('tFirstTokenMs');
|
||||
r.text += sc.outputTranscription.text;
|
||||
}
|
||||
if (m.toolCall?.functionCalls?.length) {
|
||||
mark('tFirstTokenMs');
|
||||
mark('tFirstCallMs');
|
||||
cur.toolCallThisGeneration = true;
|
||||
cur.toolRounds += 1;
|
||||
const functionResponses = m.toolCall.functionCalls.map((fc) => {
|
||||
const c = { name: fc.name, args: fc.args || {}, atMs: Date.now() - cur.t0 };
|
||||
r.calls.push(c);
|
||||
return { id: fc.id, name: fc.name, response: onCall(c, cur.turn) };
|
||||
});
|
||||
send({ toolResponse: { functionResponses } });
|
||||
}
|
||||
if (m.usageMetadata) {
|
||||
const u = m.usageMetadata;
|
||||
r.usage.inputTokens += u.promptTokenCount || 0;
|
||||
r.usage.cachedTokens += u.cachedContentTokenCount || 0;
|
||||
r.usage.outputTokens += u.responseTokenCount || 0;
|
||||
r.usage.reasoningTokens += u.thoughtsTokenCount || 0;
|
||||
r.costUsd += geminiUsageCostUsd(model, u);
|
||||
}
|
||||
if (m.goAway) r.errors.push('goAway');
|
||||
if (sc?.turnComplete) {
|
||||
r.responses += 1;
|
||||
if (cur.toolCallThisGeneration && cur.toolRounds < MAX_TOOL_ROUNDS) {
|
||||
cur.toolCallThisGeneration = false;
|
||||
if (r.text && !r.text.endsWith(' | ')) r.text += ' | ';
|
||||
// The follow-up generation normally starts within a second; if
|
||||
// the model has nothing more to say, end the turn at its last
|
||||
// message rather than charging the wait to its latency.
|
||||
const c = cur;
|
||||
c.idle = setTimeout(() => { if (cur === c) { c.r.tDoneMs = lastAt - c.t0; endTurn(); } }, IDLE_AFTER_TOOL_MS);
|
||||
return;
|
||||
}
|
||||
endTurn();
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Local-model adapters: stateless chat requests, so every phrase is a fresh
|
||||
* context by construction. Same system prompt (realtimeInstructions()) and
|
||||
* the production tools converted with toChatTools() (schema-normalized).
|
||||
*
|
||||
* openai-chat any OpenAI-compatible server (LM Studio :1234/v1, Ollama
|
||||
* :11434/v1). Streams SSE; timing is client-side.
|
||||
* ollama Ollama's native /api/chat. Same message and tool formats as
|
||||
* Chat Completions, but it can set num_ctx (the OpenAI shim
|
||||
* cannot — the ~15k-token prompt would be silently truncated
|
||||
* at a small default context) and turn thinking off, and it
|
||||
* reports server-side prefill/decode/load timings.
|
||||
*
|
||||
* Both mirror the Realtime loop: stub tool results, then continue until a
|
||||
* response has no tool calls, up to 4 continuations.
|
||||
*/
|
||||
import { realtimeInstructions, GEV_REALTIME_TOOLS } from '../productionConfig.mjs';
|
||||
import { toChatTools } from '../toolFormats.mjs';
|
||||
|
||||
const MAX_CONTINUATIONS = 4;
|
||||
|
||||
function emptyResult() {
|
||||
return {
|
||||
calls: [], text: '', errors: [], responses: 0,
|
||||
usage: { inputTokens: 0, cachedTokens: 0, outputTokens: 0, reasoningTokens: 0 },
|
||||
costUsd: 0, tFirstTokenMs: null, tFirstCallMs: null, tDoneMs: null, setupMs: 0, timeout: false,
|
||||
local: { prefillTokS: null, decodeTokS: null, promptEvalCount: null, evalCount: null, loadMs: null, firstResponseMs: null },
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(raw) {
|
||||
if (raw && typeof raw === 'object') return raw;
|
||||
try { return JSON.parse(raw || '{}'); } catch { return { __unparsed: String(raw).slice(0, 400) }; }
|
||||
}
|
||||
|
||||
async function* lines(body) {
|
||||
const decoder = new TextDecoder();
|
||||
let buf = '';
|
||||
for await (const chunk of body) {
|
||||
buf += decoder.decode(chunk, { stream: true });
|
||||
let i;
|
||||
while ((i = buf.indexOf('\n')) >= 0) {
|
||||
yield buf.slice(0, i);
|
||||
buf = buf.slice(i + 1);
|
||||
}
|
||||
}
|
||||
if (buf.trim()) yield buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared multi-response loop over one or more user turns in ONE growing
|
||||
* message history. `request(messages, r, t0, first, deadline)` performs one
|
||||
* streamed completion and returns `{ content, toolCalls:[{id,name,args}] }`.
|
||||
*/
|
||||
async function runChatDialogue({ turns, onCall, request, turnTimeoutMs, contexts = [] }) {
|
||||
const messages = [{ role: 'system', content: realtimeInstructions() }];
|
||||
const results = [];
|
||||
for (let turn = 0; turn < turns.length; turn += 1) {
|
||||
const r = emptyResult();
|
||||
if (contexts[turn]) messages.push({ role: 'system', content: JSON.stringify(contexts[turn]) });
|
||||
messages.push({ role: 'user', content: turns[turn] });
|
||||
const t0 = Date.now();
|
||||
const deadline = t0 + turnTimeoutMs;
|
||||
try {
|
||||
for (let round = 0; round <= MAX_CONTINUATIONS; round += 1) {
|
||||
if (Date.now() > deadline) { r.timeout = true; break; }
|
||||
const out = await request(messages, r, t0, round === 0, deadline);
|
||||
r.responses += 1;
|
||||
if (out.content) r.text += (r.text ? ' | ' : '') + out.content.trim();
|
||||
if (!out.toolCalls.length) {
|
||||
messages.push({ role: 'assistant', content: out.content || '' });
|
||||
break;
|
||||
}
|
||||
messages.push({ role: 'assistant', content: out.content || '', tool_calls: out.toolCalls.map((tc) => tc.wire) });
|
||||
for (const tc of out.toolCalls) {
|
||||
const c = { name: tc.name, args: tc.args, atMs: tc.atMs };
|
||||
r.calls.push(c);
|
||||
messages.push({ role: 'tool', tool_call_id: tc.id, tool_name: tc.name, name: tc.name, content: JSON.stringify(onCall(c, turn)) });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.name === 'TimeoutError' || e?.name === 'AbortError') r.timeout = true;
|
||||
else r.errors.push(String(e?.message || e).slice(0, 300));
|
||||
}
|
||||
r.tDoneMs = Date.now() - t0;
|
||||
results.push(r);
|
||||
if (r.timeout || r.errors.length) {
|
||||
while (results.length < turns.length) {
|
||||
const skipped = emptyResult();
|
||||
skipped.errors.push('dialogue aborted after an earlier turn failed');
|
||||
results.push(skipped);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── OpenAI-compatible (SSE) ─────────────────────────────────────────────────
|
||||
|
||||
export function createOpenAiChatAdapter({ model, baseUrl, temperature = 0.8, extraBody = {}, turnTimeoutMs = 180_000, apiKey = 'local' }) {
|
||||
const tools = toChatTools(GEV_REALTIME_TOOLS);
|
||||
const request = async (messages, r, t0, first, deadline) => {
|
||||
const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(Math.max(1000, deadline - Date.now())),
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify({
|
||||
model, messages, tools, tool_choice: 'auto', temperature, stream: true,
|
||||
stream_options: { include_usage: true }, ...extraBody,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
|
||||
const tStart = Date.now();
|
||||
let tFirst = null;
|
||||
let content = '';
|
||||
const byIndex = new Map();
|
||||
let usage = null;
|
||||
let firstCallDoneAt = null;
|
||||
for await (const line of lines(res.body)) {
|
||||
if (!line.startsWith('data:')) continue;
|
||||
const data = line.slice(5).trim();
|
||||
if (data === '[DONE]') break;
|
||||
let m;
|
||||
try { m = JSON.parse(data); } catch { continue; }
|
||||
if (m.usage) usage = m.usage;
|
||||
const choice = m.choices?.[0];
|
||||
const d = choice?.delta || {};
|
||||
const now = Date.now();
|
||||
if ((d.content || d.reasoning_content || d.reasoning || d.tool_calls?.length) && tFirst === null) {
|
||||
tFirst = now;
|
||||
if (r.tFirstTokenMs === null) r.tFirstTokenMs = now - t0;
|
||||
}
|
||||
if (d.content) content += d.content;
|
||||
for (const tc of d.tool_calls || []) {
|
||||
const idx = tc.index ?? byIndex.size;
|
||||
if (idx > 0 && byIndex.has(0) && firstCallDoneAt === null) firstCallDoneAt = now;
|
||||
const cur = byIndex.get(idx) || { id: tc.id || `call_${idx}`, name: '', argText: '' };
|
||||
if (tc.id) cur.id = tc.id;
|
||||
if (tc.function?.name) cur.name += tc.function.name;
|
||||
if (tc.function?.arguments) cur.argText += typeof tc.function.arguments === 'string' ? tc.function.arguments : JSON.stringify(tc.function.arguments);
|
||||
byIndex.set(idx, cur);
|
||||
}
|
||||
if (choice?.finish_reason && byIndex.size && firstCallDoneAt === null) firstCallDoneAt = now;
|
||||
}
|
||||
const tEnd = Date.now();
|
||||
if (byIndex.size && firstCallDoneAt === null) firstCallDoneAt = tEnd;
|
||||
if (byIndex.size && r.tFirstCallMs === null) r.tFirstCallMs = firstCallDoneAt - t0;
|
||||
if (usage) {
|
||||
r.usage.inputTokens += usage.prompt_tokens || 0;
|
||||
r.usage.cachedTokens += usage.prompt_tokens_details?.cached_tokens || 0;
|
||||
r.usage.outputTokens += usage.completion_tokens || 0;
|
||||
r.usage.reasoningTokens += usage.completion_tokens_details?.reasoning_tokens || 0;
|
||||
}
|
||||
if (first) {
|
||||
r.local.firstResponseMs = tEnd - tStart;
|
||||
if (usage && tFirst) {
|
||||
r.local.promptEvalCount = usage.prompt_tokens ?? null;
|
||||
r.local.evalCount = usage.completion_tokens ?? null;
|
||||
// Client-side approximations: prefill ≈ prompt / (request → first token);
|
||||
// decode ≈ (completion − 1) / (first token → end of stream).
|
||||
r.local.prefillTokS = usage.prompt_tokens ? usage.prompt_tokens / Math.max(0.001, (tFirst - tStart) / 1000) : null;
|
||||
r.local.decodeTokS = usage.completion_tokens > 1 && tEnd > tFirst ? (usage.completion_tokens - 1) / ((tEnd - tFirst) / 1000) : null;
|
||||
r.local.timingSource = 'client';
|
||||
}
|
||||
}
|
||||
const toolCalls = [...byIndex.values()].map((tc) => ({
|
||||
id: tc.id, name: tc.name, args: parseArgs(tc.argText), atMs: firstCallDoneAt - t0,
|
||||
wire: { id: tc.id, type: 'function', function: { name: tc.name, arguments: tc.argText || '{}' } },
|
||||
}));
|
||||
return { content, toolCalls };
|
||||
};
|
||||
return {
|
||||
provider: 'openai-chat',
|
||||
model,
|
||||
settings: { baseUrl, temperature, tool_choice: 'auto', toolSchema: 'normalized (untyped → string|number|boolean)', context: 'stateless request per phrase', ...extraBody },
|
||||
async open() {},
|
||||
async close() {},
|
||||
runDialogue: ({ turns, onCall, contexts }) => runChatDialogue({ turns, onCall, contexts, request, turnTimeoutMs }),
|
||||
runTurn: async ({ text, onCall, context }) => (await runChatDialogue({ turns: [text], onCall, contexts: [context], request, turnTimeoutMs }))[0],
|
||||
};
|
||||
}
|
||||
|
||||
// ── Ollama native (/api/chat, NDJSON) ───────────────────────────────────────
|
||||
|
||||
export function createOllamaAdapter({ model, baseUrl = 'http://localhost:11434', temperature = 0.8, numCtx = 32768, think = false, keepAlive = '30m', turnTimeoutMs = 300_000 }) {
|
||||
const tools = toChatTools(GEV_REALTIME_TOOLS);
|
||||
let thinkParam = think;
|
||||
const request = async (messages, r, t0, first, deadline) => {
|
||||
const wireMessages = messages.map((m) => {
|
||||
if (m.role === 'assistant' && m.tool_calls) {
|
||||
return { role: 'assistant', content: m.content, tool_calls: m.tool_calls.map((tc) => ({ function: { name: tc.function.name, arguments: parseArgs(tc.function.arguments) } })) };
|
||||
}
|
||||
if (m.role === 'tool') return { role: 'tool', tool_name: m.tool_name, content: m.content };
|
||||
return m;
|
||||
});
|
||||
const body = { model, messages: wireMessages, tools, stream: true, keep_alive: keepAlive, options: { temperature, num_ctx: numCtx } };
|
||||
if (thinkParam !== null) body.think = thinkParam;
|
||||
const res = await fetch(`${baseUrl}/api/chat`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(Math.max(1000, deadline - Date.now())),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const errText = await res.text();
|
||||
if (/does not support thinking/i.test(errText) && thinkParam !== null) {
|
||||
thinkParam = null;
|
||||
return request(messages, r, t0, first, deadline);
|
||||
}
|
||||
throw new Error(`HTTP ${res.status}: ${errText.slice(0, 300)}`);
|
||||
}
|
||||
let content = '';
|
||||
const toolCalls = [];
|
||||
let final = null;
|
||||
for await (const line of lines(res.body)) {
|
||||
if (!line.trim()) continue;
|
||||
let m;
|
||||
try { m = JSON.parse(line); } catch { continue; }
|
||||
if (m.error) throw new Error(m.error);
|
||||
const now = Date.now();
|
||||
const msg = m.message || {};
|
||||
if ((msg.content || msg.thinking || msg.tool_calls?.length) && r.tFirstTokenMs === null) r.tFirstTokenMs = now - t0;
|
||||
if (msg.content) content += msg.content;
|
||||
for (const tc of msg.tool_calls || []) {
|
||||
if (r.tFirstCallMs === null) r.tFirstCallMs = now - t0;
|
||||
const i = `${messages.length}_${toolCalls.length}`;
|
||||
const args = parseArgs(tc.function?.arguments);
|
||||
toolCalls.push({
|
||||
id: `call_${i}`, name: tc.function?.name, args, atMs: now - t0,
|
||||
wire: { id: `call_${i}`, type: 'function', function: { name: tc.function?.name, arguments: JSON.stringify(args) } },
|
||||
});
|
||||
}
|
||||
if (m.done) final = m;
|
||||
}
|
||||
if (final) {
|
||||
r.usage.inputTokens += final.prompt_eval_count || 0;
|
||||
r.usage.outputTokens += final.eval_count || 0;
|
||||
if (first) {
|
||||
const ns = 1e9;
|
||||
r.local.promptEvalCount = final.prompt_eval_count ?? null;
|
||||
r.local.evalCount = final.eval_count ?? null;
|
||||
r.local.prefillTokS = final.prompt_eval_duration ? final.prompt_eval_count / (final.prompt_eval_duration / ns) : null;
|
||||
r.local.decodeTokS = final.eval_duration ? final.eval_count / (final.eval_duration / ns) : null;
|
||||
r.local.loadMs = final.load_duration ? final.load_duration / 1e6 : null;
|
||||
r.local.firstResponseMs = final.total_duration ? final.total_duration / 1e6 : null;
|
||||
r.local.timingSource = 'ollama';
|
||||
}
|
||||
}
|
||||
return { content, toolCalls };
|
||||
};
|
||||
return {
|
||||
provider: 'ollama',
|
||||
model,
|
||||
get settings() {
|
||||
return { baseUrl, api: '/api/chat', temperature, num_ctx: numCtx, think: thinkParam === null ? 'unsupported by model (param omitted)' : thinkParam, keep_alive: keepAlive, toolSchema: 'normalized (untyped → string|number|boolean)', context: 'stateless request per phrase' };
|
||||
},
|
||||
async open() {},
|
||||
async close() {},
|
||||
runDialogue: ({ turns, onCall, contexts }) => runChatDialogue({ turns, onCall, contexts, request, turnTimeoutMs }),
|
||||
runTurn: async ({ text, onCall, context }) => (await runChatDialogue({ turns: [text], onCall, contexts: [context], request, turnTimeoutMs }))[0],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* OpenAI Realtime adapter — text turns over the Realtime WebSocket.
|
||||
*
|
||||
* Each dialogue (one phrase, or a multi-turn dialogue) opens a FRESH session
|
||||
* configured with production's exact session body (captured from the token
|
||||
* handler) and sends its user text items one turn at a time. Each turn
|
||||
* mirrors qa-voice-routing's loop: every function call gets a stub output, and
|
||||
* a continuation response is requested after `response.done` (never while a
|
||||
* response is active), up to 4 continuations.
|
||||
*/
|
||||
import WebSocket from 'ws';
|
||||
import { providerKey } from '../keys.mjs';
|
||||
import { captureProductionSession } from '../productionConfig.mjs';
|
||||
import { openAiUsageCostUsd } from '../pricing.mjs';
|
||||
|
||||
const MAX_CONTINUATIONS = 4;
|
||||
|
||||
export function newTurnResult() {
|
||||
return {
|
||||
calls: [], text: '', errors: [], responses: 0,
|
||||
usage: { inputTokens: 0, cachedTokens: 0, outputTokens: 0, reasoningTokens: 0 },
|
||||
costUsd: 0, tFirstTokenMs: null, tFirstCallMs: null, tDoneMs: null, setupMs: null, timeout: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function createOpenAiRealtimeAdapter({ model, turnTimeoutMs = 45_000 }) {
|
||||
let session = null;
|
||||
const adapter = {
|
||||
provider: 'openai-realtime',
|
||||
model,
|
||||
settings: {},
|
||||
async open() {
|
||||
session = await captureProductionSession(model);
|
||||
adapter.settings = {
|
||||
reasoning: session.reasoning,
|
||||
truncation: session.truncation,
|
||||
tool_choice: session.tool_choice,
|
||||
temperature: 'not settable (GA Realtime session has no temperature field)',
|
||||
output_modalities: ['text'],
|
||||
context: 'fresh session per phrase / dialogue',
|
||||
};
|
||||
},
|
||||
async close() {},
|
||||
async runTurn({ text, onCall, context }) {
|
||||
return (await adapter.runDialogue({ turns: [text], onCall, contexts: [context] }))[0];
|
||||
},
|
||||
runDialogue({ turns, onCall, contexts = [] }) {
|
||||
return new Promise((resolveAll) => {
|
||||
const tSetup = Date.now();
|
||||
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, {
|
||||
headers: { Authorization: `Bearer ${providerKey('openai')}` },
|
||||
});
|
||||
const results = [];
|
||||
let setupMs = null;
|
||||
let cur = null; // { r, t0, continuations, needContinuation, turn, timer }
|
||||
let closed = false;
|
||||
const send = (o) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(o)); };
|
||||
const shutdown = (error) => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
if (cur) {
|
||||
clearTimeout(cur.timer);
|
||||
if (error) cur.r.errors.push(error);
|
||||
cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
cur = null;
|
||||
}
|
||||
while (results.length < turns.length) {
|
||||
const r = newTurnResult();
|
||||
r.errors.push(error || 'dialogue aborted');
|
||||
results.push(r);
|
||||
}
|
||||
try { ws.terminate(); } catch { /* noop */ }
|
||||
resolveAll(results);
|
||||
};
|
||||
const startTurn = (turn) => {
|
||||
const r = newTurnResult();
|
||||
r.setupMs = turn === 0 ? setupMs : 0;
|
||||
cur = { r, t0: Date.now(), continuations: 0, needContinuation: false, turn };
|
||||
cur.timer = setTimeout(() => { cur.r.timeout = true; endTurn(); }, turnTimeoutMs);
|
||||
// Optional inert system item first (the app's pointer_context shape).
|
||||
if (contexts[turn]) send({ type: 'conversation.item.create', item: { type: 'message', role: 'system', content: [{ type: 'input_text', text: JSON.stringify(contexts[turn]) }] } });
|
||||
send({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: turns[turn] }] } });
|
||||
send({ type: 'response.create', response: { output_modalities: ['text'] } });
|
||||
};
|
||||
const endTurn = () => {
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.timer);
|
||||
cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
const next = cur.turn + 1;
|
||||
const timedOut = cur.r.timeout;
|
||||
cur = null;
|
||||
if (next < turns.length && !timedOut) startTurn(next);
|
||||
else shutdown();
|
||||
};
|
||||
const mark = (key) => { if (cur && cur.r[key] === null) cur.r[key] = Date.now() - cur.t0; };
|
||||
|
||||
ws.on('unexpected-response', (_req, res) => shutdown(`HTTP ${res.statusCode}`));
|
||||
ws.on('error', (e) => shutdown(`ws: ${e.message}`));
|
||||
ws.on('close', () => shutdown(results.length < turns.length ? 'socket closed' : undefined));
|
||||
ws.on('message', (raw) => {
|
||||
let m;
|
||||
try { m = JSON.parse(raw.toString()); } catch { return; }
|
||||
if (m.type === 'session.created') {
|
||||
send({ type: 'session.update', session });
|
||||
return;
|
||||
}
|
||||
if (m.type === 'session.updated' && setupMs === null) {
|
||||
setupMs = Date.now() - tSetup;
|
||||
startTurn(0);
|
||||
return;
|
||||
}
|
||||
if (m.type === 'error') {
|
||||
if (!cur) { shutdown(m.error?.message || 'realtime error'); return; }
|
||||
cur.r.errors.push(m.error?.message || 'realtime error');
|
||||
return;
|
||||
}
|
||||
if (!cur) return;
|
||||
const r = cur.r;
|
||||
if (
|
||||
m.type === 'response.output_text.delta' ||
|
||||
m.type === 'response.function_call_arguments.delta' ||
|
||||
m.type === 'response.output_audio_transcript.delta'
|
||||
) mark('tFirstTokenMs');
|
||||
if (m.type === 'response.output_text.delta' || m.type === 'response.output_audio_transcript.delta') {
|
||||
r.text += m.delta || '';
|
||||
}
|
||||
if (m.type === 'response.output_item.done' && m.item?.type === 'function_call') {
|
||||
mark('tFirstTokenMs');
|
||||
mark('tFirstCallMs');
|
||||
let args = {};
|
||||
try { args = JSON.parse(m.item.arguments || '{}'); } catch { args = { __unparsed: m.item.arguments }; }
|
||||
const c = { name: m.item.name, args, atMs: Date.now() - cur.t0 };
|
||||
r.calls.push(c);
|
||||
send({
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: m.item.call_id, output: JSON.stringify(onCall(c, cur.turn)) },
|
||||
});
|
||||
cur.needContinuation = true;
|
||||
}
|
||||
if (m.type === 'response.done') {
|
||||
r.responses += 1;
|
||||
const u = m.response?.usage;
|
||||
if (u) {
|
||||
r.usage.inputTokens += u.input_tokens || 0;
|
||||
r.usage.cachedTokens += u.input_token_details?.cached_tokens || 0;
|
||||
r.usage.outputTokens += u.output_tokens || 0;
|
||||
r.usage.reasoningTokens += u.output_token_details?.reasoning_tokens || 0;
|
||||
r.costUsd += openAiUsageCostUsd(model, u);
|
||||
}
|
||||
if (m.response?.status === 'failed') r.errors.push(JSON.stringify(m.response?.status_details).slice(0, 200));
|
||||
if (cur.needContinuation && cur.continuations < MAX_CONTINUATIONS) {
|
||||
cur.needContinuation = false;
|
||||
cur.continuations += 1;
|
||||
if (r.text) r.text += ' | ';
|
||||
send({ type: 'response.create', response: { output_modalities: ['text'] } });
|
||||
return;
|
||||
}
|
||||
endTurn();
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
[{"type":"function","name":"fly_to_location","description":"Fly the God's Eye View camera to a known city, geocoded country/region/city/landmark, or explicit WGS84 coordinate. Countries/cities frame the whole place; landmarks/buildings use close framing.","parameters":{"type":"object","additionalProperties":false,"properties":{"locationId":{"type":"string","enum":["austin","sf","nyc","tokyo","london","paris","dubai","dc"],"description":"Known city preset ID. Use when the requested place matches one of these cities."},"query":{"type":"string","description":"Plain place search query, e.g. \"London\", \"Eiffel Tower\", or \"Dubai Marina\"."},"latitude":{"type":"number","minimum":-90,"maximum":90},"longitude":{"type":"number","minimum":-180,"maximum":180},"viewMode":{"type":"string","enum":["close","overview"],"description":"Optional framing intent. Usually omit this; GEV infers whole-place framing for countries/cities and close framing for landmarks."},"rangeM":{"type":"number","minimum":100,"maximum":20000000,"description":"Optional camera range from the target in meters. Omit it for automatic whole-country/whole-city or close-landmark framing; provide it only when the user explicitly requests a numeric height or distance."},"waitForArrival":{"type":"boolean","description":"Set true when a later tool depends on the destination viewport. The result then waits for the camera flight and returns arrived=true; cancellation returns ok=false."}}}},{"type":"function","name":"select_nearest_aircraft","description":"Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","enum":["flights","military"],"description":"Aircraft layer to enable and search. Use flights unless the user explicitly asks for military aircraft."},"locationId":{"type":"string","enum":["austin","sf","nyc","tokyo","london","paris","dubai","dc"],"description":"Known city preset ID when the place matches one of these cities."},"locationQuery":{"type":"string","maxLength":160,"description":"Free-form destination when no locationId matches."},"latitude":{"type":"number","minimum":-90,"maximum":90},"longitude":{"type":"number","minimum":-180,"maximum":180}},"required":["layerId"]}},{"type":"function","name":"adjust_camera_zoom","description":"Move the current Cesium camera closer to or farther from what it is presently looking at. Use for relative zoom requests without changing location.","parameters":{"type":"object","additionalProperties":false,"properties":{"direction":{"type":"string","enum":["in","out"]},"amount":{"type":"string","enum":["little","medium","lot"],"description":"Use little for phrases like \"a bit\" or \"a little\", medium for ordinary zoom requests, and lot for \"way out/in\"."}},"required":["direction","amount"]}},{"type":"function","name":"zoom_to_globe","description":"Pull the camera out to an ABSOLUTE full-Earth globe view (~18,000 km altitude, the whole planet in frame), keeping the current region centered. Use for \"globe view\", \"whole earth\", \"see the planet\", \"zoom all the way out\". Never use adjust_camera_zoom for these — its relative steps cannot reach the globe.","parameters":{"type":"object","additionalProperties":false,"properties":{}}},{"type":"function","name":"set_layer_visibility","description":"Enable or disable one registered God's Eye View data layer.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","description":"Common-name mapping for the non-obvious ids: space mission(s) → rocket-launches; fires/wildfires/active fires → local-firms (NASA FIRMS); ships/vessels/boats → ais-live-vessels; undersea/submarine cables → telegeography-submarine-cables; datacenters → local-datacenters; dams → local-dams; bikes/bike share → bikeshare; street traffic/congestion → traffic; traffic cameras → cctv; internet radio/stations → radio; ALPR/license plate readers/Flock cameras → alpr-cameras; local ADS-B/my receiver/my antenna (aircraft heard by a local RTL-SDR receiver) → local-adsb.","enum":["flights","military","earthquakes","satellites","rocket-launches","traffic","cctv","radio","bikeshare","ais-live-vessels","local-datacenters","local-dams","telegeography-submarine-cables","local-firms","alpr-cameras","local-adsb"]},"enabled":{"type":"boolean"}},"required":["layerId","enabled"]}},{"type":"function","name":"show_data_layers_menu","description":"Open the data layers dropdown/menu and optionally scroll to a specific layer row without toggling it.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","enum":["flights","military","earthquakes","satellites","traffic","ccLine truncated
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* Pure grading for voice-routing runs. No I/O, no network.
|
||||
*
|
||||
* `gradeCore` reproduces the pass/fail semantics of the routing layer in
|
||||
* `scripts/qa-voice-routing.mjs` (which now calls it), plus a few extra
|
||||
* benchmark signals. `gradeCoverage` sorts coverage-probe turns into
|
||||
* correct / honest-refusal / hallucinated-capability / wrong-tool.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Spot-check `expected` against `actual` args: substring match (case
|
||||
* insensitive) for strings, exact match for everything else. An array value
|
||||
* is matched through its comma-joined string form, so `{ layers: 'earthquakes' }`
|
||||
* matches `layers: ['earthquakes']`.
|
||||
*/
|
||||
export function matchArgs(expected, actual) {
|
||||
return Object.entries(expected).every(([k, v]) => {
|
||||
const got = actual?.[k];
|
||||
if (typeof v === 'string') return String(got ?? '').toLowerCase().includes(v.toLowerCase());
|
||||
return got === v;
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a call matcher: tool name plus optional arg spot-check. */
|
||||
export function call(name, args) {
|
||||
const matcher = (c) => c?.name === name && (!args || matchArgs(args, c.args));
|
||||
matcher.toolName = name;
|
||||
return matcher;
|
||||
}
|
||||
|
||||
/** Build a call matcher from a tool name and a predicate over its args. */
|
||||
export function where(name, predicate) {
|
||||
const matcher = (c) => c?.name === name && Boolean(predicate(c.args || {}));
|
||||
matcher.toolName = name;
|
||||
return matcher;
|
||||
}
|
||||
|
||||
/** Human label for a phrase's expectation. */
|
||||
export function expectLabel(phrase) {
|
||||
if (phrase.expectNone) return '(no tool)';
|
||||
if (Array.isArray(phrase.expect)) return phrase.expect.join('+');
|
||||
if (phrase.expect?.oneOf) return phrase.expect.oneOf.join('|');
|
||||
return String(phrase.expect);
|
||||
}
|
||||
|
||||
/** Every tool name that satisfies a phrase (union for oneOf / arrays). */
|
||||
export function expectedToolNames(phrase) {
|
||||
if (phrase.expectNone) return [];
|
||||
if (Array.isArray(phrase.expect)) return [...phrase.expect];
|
||||
if (phrase.expect?.oneOf) return [...phrase.expect.oneOf];
|
||||
return [phrase.expect];
|
||||
}
|
||||
|
||||
/**
|
||||
* Grade one core-suite turn.
|
||||
*
|
||||
* `ok` is exactly the qa-voice-routing verdict: expectNone passes only with no
|
||||
* calls; an array needs every tool; oneOf needs any; a string needs that tool;
|
||||
* then `args` / `argsByTool` spot checks. The rest are benchmark extras:
|
||||
* toolOk tool-name routing alone (before the args check)
|
||||
* argsOk null when the phrase has no arg spot check
|
||||
* firstCallOk the FIRST call was an expected tool (what a user sees first)
|
||||
* falsePositive a conversational turn produced a tool call
|
||||
* extraCalls calls to tools outside the expected set
|
||||
* typedArgsOk benchmark-only typed-argument check (`typedArgs`: exact
|
||||
* enum values, layer ids, numeric thresholds, sort order);
|
||||
* null when the phrase has none. Not part of `ok`.
|
||||
* fullCallOk toolOk and every arg check (args, argsByTool, typedArgs)
|
||||
*/
|
||||
export function gradeCore(phrase, calls) {
|
||||
const names = calls.map((c) => c.name);
|
||||
if (phrase.expectNone) {
|
||||
return {
|
||||
ok: names.length === 0,
|
||||
toolOk: names.length === 0,
|
||||
fullCallOk: names.length === 0,
|
||||
argsOk: null,
|
||||
typedArgsOk: null,
|
||||
firstCallOk: names.length === 0,
|
||||
falsePositive: names.length > 0,
|
||||
extraCalls: names,
|
||||
detail: names.length ? `unexpected calls: ${names.join(',')}` : 'clean conversational turn',
|
||||
};
|
||||
}
|
||||
const expected = expectedToolNames(phrase);
|
||||
let toolOk;
|
||||
if (Array.isArray(phrase.expect)) toolOk = phrase.expect.every((n) => names.includes(n));
|
||||
else if (phrase.expect?.oneOf) toolOk = names.some((n) => phrase.expect.oneOf.includes(n));
|
||||
else toolOk = names.includes(phrase.expect);
|
||||
|
||||
let detail = `called: ${names.join(',') || '(none)'}`;
|
||||
let argsOk = null;
|
||||
if (phrase.args) {
|
||||
const target = calls.find((c) => (Array.isArray(phrase.expect) ? true : expected.includes(c.name)));
|
||||
argsOk = Boolean(target) && matchArgs(phrase.args, target.args);
|
||||
if (toolOk && !argsOk) detail += ` args mismatch: ${JSON.stringify(target?.args)}`;
|
||||
}
|
||||
if (phrase.argsByTool) {
|
||||
let allOk = true;
|
||||
for (const [toolName, expectedArgs] of Object.entries(phrase.argsByTool)) {
|
||||
const target = calls.find((c) => c.name === toolName);
|
||||
if (!target || !matchArgs(expectedArgs, target.args)) {
|
||||
allOk = false;
|
||||
if (toolOk) detail += ` ${toolName} args mismatch: ${JSON.stringify(target?.args)}`;
|
||||
}
|
||||
}
|
||||
argsOk = argsOk === null ? allOk : argsOk && allOk;
|
||||
}
|
||||
const typedArgsOk = phrase.typedArgs ? phrase.typedArgs.every((m) => calls.some((c) => m(c))) : null;
|
||||
if (toolOk && typedArgsOk === false) detail += ` typed args miss: ${JSON.stringify(calls.map((c) => c.args)).slice(0, 200)}`;
|
||||
return {
|
||||
ok: toolOk && argsOk !== false,
|
||||
toolOk,
|
||||
fullCallOk: toolOk && argsOk !== false && typedArgsOk !== false,
|
||||
argsOk,
|
||||
typedArgsOk,
|
||||
firstCallOk: names.length > 0 && expected.includes(names[0]),
|
||||
falsePositive: false,
|
||||
extraCalls: names.filter((n) => !expected.includes(n)),
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Grade a two-turn dialogue from each turn's calls. The headline `ok` is
|
||||
* turn 2's full-call accuracy (tool + typed args), since turn 2 is the one
|
||||
* that depends on context; turn 1 is reported alongside.
|
||||
*/
|
||||
export function gradeDialogue(dialogue, turnCalls) {
|
||||
const turn1 = gradeCore(dialogue.first, turnCalls[0] || []);
|
||||
const turn2 = gradeCore(dialogue.second, turnCalls[1] || []);
|
||||
return {
|
||||
ok: turn2.fullCallOk,
|
||||
toolOk: turn2.toolOk,
|
||||
fullCallOk: turn2.fullCallOk,
|
||||
turn1Ok: turn1.fullCallOk,
|
||||
turn1,
|
||||
turn2,
|
||||
detail: `t1 ${turn1.detail} | t2 ${turn2.detail}`,
|
||||
};
|
||||
}
|
||||
|
||||
// ── JSON-schema argument validation (the subset GEV tools use) ─────────────
|
||||
|
||||
const TYPE_CHECKS = {
|
||||
string: (v) => typeof v === 'string',
|
||||
number: (v) => typeof v === 'number' && Number.isFinite(v),
|
||||
integer: (v) => Number.isInteger(v),
|
||||
boolean: (v) => typeof v === 'boolean',
|
||||
object: (v) => v !== null && typeof v === 'object' && !Array.isArray(v),
|
||||
array: (v) => Array.isArray(v),
|
||||
null: (v) => v === null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Validate `value` against a JSON schema (type / enum / required /
|
||||
* properties / additionalProperties:false / items / min/max bounds).
|
||||
* Returns a list of human-readable violations; empty means valid.
|
||||
*/
|
||||
export function validateArgs(schema, value, where = 'args') {
|
||||
const errors = [];
|
||||
if (!schema || typeof schema !== 'object') return errors;
|
||||
if (schema.type) {
|
||||
const types = Array.isArray(schema.type) ? schema.type : [schema.type];
|
||||
if (!types.some((t) => TYPE_CHECKS[t]?.(value))) {
|
||||
errors.push(`${where} should be ${types.join('|')}`);
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
if (schema.enum && !schema.enum.includes(value)) {
|
||||
errors.push(`${where}=${JSON.stringify(value)} is not one of ${schema.enum.join(', ')}`);
|
||||
}
|
||||
if (typeof value === 'number') {
|
||||
if (typeof schema.minimum === 'number' && value < schema.minimum) errors.push(`${where} below ${schema.minimum}`);
|
||||
if (typeof schema.maximum === 'number' && value > schema.maximum) errors.push(`${where} above ${schema.maximum}`);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (typeof schema.minItems === 'number' && value.length < schema.minItems) errors.push(`${where} needs ≥${schema.minItems} items`);
|
||||
if (schema.items) value.forEach((item, i) => errors.push(...validateArgs(schema.items, item, `${where}[${i}]`)));
|
||||
}
|
||||
if (TYPE_CHECKS.object(value) && (schema.properties || schema.required)) {
|
||||
for (const key of schema.required || []) {
|
||||
if (!(key in value)) errors.push(`${where}.${key} is required`);
|
||||
}
|
||||
for (const [key, v] of Object.entries(value)) {
|
||||
const sub = schema.properties?.[key];
|
||||
if (!sub) {
|
||||
if (schema.additionalProperties === false) errors.push(`${where}.${key} is not a known argument`);
|
||||
continue;
|
||||
}
|
||||
errors.push(...validateArgs(sub, v, `${where}.${key}`));
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/** Validate a tool call against the Realtime tool list. */
|
||||
export function validateCall(tools, c) {
|
||||
const tool = tools.find((t) => t.name === c.name);
|
||||
if (!tool) return [`unknown tool ${c.name}`];
|
||||
return validateArgs(tool.parameters, c.args ?? {});
|
||||
}
|
||||
|
||||
// ── Coverage probe grading ─────────────────────────────────────────────────
|
||||
|
||||
/** Tools that only read state; they never count as acting on the request. */
|
||||
export const LOOKUP_TOOLS = Object.freeze([
|
||||
'get_entity_context',
|
||||
'get_current_view_state',
|
||||
'show_data_layers_menu',
|
||||
'set_panel_open',
|
||||
]);
|
||||
|
||||
export const REFUSAL_RE =
|
||||
/\b(can(?:no|')t|cannot|unable|not able|don'?t have|do not have|doesn'?t have|does not have|isn'?t (?:available|supported|something)|aren'?t (?:available|supported)|not (?:available|supported|possible|yet)|no (?:access|way|tool|data|layer|feed|live|transit|weather|wind|cyclone|radar|lightning|receiver)|n'?t (?:connected|see any|find)|(?:isn'?t|is no|are no) (?:an? )?(?:\w+ ){0,2}(?:layer|feed|tool|data source|integration)s?|outside what i can|beyond what i can|only (?:have|support|show|cover)|unfortunately|sorry)\b/i;
|
||||
|
||||
/**
|
||||
* Grade one coverage-probe turn.
|
||||
*
|
||||
* Order matters:
|
||||
* 1. a call pattern in `correct` → correct
|
||||
* 2. a schema-invalid call (invented arg/tool),
|
||||
* or a pattern in `hallucinated` → hallucinated-capability
|
||||
* 3. any other acting call (not lookup/neutral) → wrong-tool
|
||||
* 4. no acting call: text matches `claimText`
|
||||
* and no refusal wording → hallucinated-capability
|
||||
* 5. no acting call: refusal wording or a question → honest-refusal
|
||||
* 6. no acting call, no refusal → hallucinated-capability
|
||||
* (it answered as if it had the data)
|
||||
*
|
||||
* `acceptable` = correct, or honest-refusal on an unsupported item.
|
||||
*/
|
||||
export function gradeCoverage(item, calls, text, tools = []) {
|
||||
// Models write typographic apostrophes ("can’t"); fold them before matching.
|
||||
const finalText = String(text || '').replace(/[\u2018\u2019\u02bc]/g, "'");
|
||||
const verdict = (v, reason) => ({
|
||||
verdict: v,
|
||||
reason,
|
||||
acceptable: v === 'correct' || (v === 'honest-refusal' && !item.supported),
|
||||
});
|
||||
for (const alternative of item.correct || []) {
|
||||
if (alternative.every((m) => calls.some((c) => m(c)))) {
|
||||
return verdict('correct', `matched ${alternative.map((m) => m.toolName || 'pattern').join('+')}`);
|
||||
}
|
||||
}
|
||||
if (tools.length) {
|
||||
for (const c of calls) {
|
||||
const errors = validateCall(tools, c);
|
||||
if (errors.length) return verdict('hallucinated-capability', `invalid ${c.name}: ${errors[0]}`);
|
||||
}
|
||||
}
|
||||
for (const m of item.hallucinated || []) {
|
||||
const hit = calls.find((c) => m(c));
|
||||
if (hit) return verdict('hallucinated-capability', `faked capability via ${hit.name}(${JSON.stringify(hit.args).slice(0, 80)})`);
|
||||
}
|
||||
const neutral = new Set([...LOOKUP_TOOLS, ...(item.neutral || [])]);
|
||||
const acting = calls.filter((c) => !neutral.has(c.name));
|
||||
if (acting.length) return verdict('wrong-tool', `called ${acting.map((c) => c.name).join(',')}`);
|
||||
const refused = REFUSAL_RE.test(finalText);
|
||||
if (item.claimText && item.claimText.test(finalText) && !refused) {
|
||||
return verdict('hallucinated-capability', 'asserted data it cannot have');
|
||||
}
|
||||
if (refused || /\?\s*$/.test(finalText.trim())) return verdict('honest-refusal', 'refused or asked');
|
||||
if (!finalText.trim().replace(/\|/g, '').trim()) {
|
||||
return verdict('wrong-tool', calls.length ? `only lookups/neutral calls (${calls.map((c) => c.name).join(',')}) and no answer` : 'no tool call and no answer');
|
||||
}
|
||||
return verdict('hallucinated-capability', 'answered without a tool or a refusal');
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Runtime key lookup for the voice benchmark. Keys are read from the
|
||||
* environment, then the macOS Keychain, at call time. They are never written
|
||||
* to results, logs, or files.
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const KEYCHAIN = {
|
||||
openai: { env: 'OPENAI_API_KEY', service: 'openai-api', account: 'api-key' },
|
||||
gemini: { env: 'GEMINI_API_KEY', service: 'gemini-api', account: 'api-key' },
|
||||
};
|
||||
|
||||
const cache = new Map();
|
||||
|
||||
/** Resolve a provider key from the environment, then the Keychain. */
|
||||
export function providerKey(provider) {
|
||||
if (cache.has(provider)) return cache.get(provider);
|
||||
const spec = KEYCHAIN[provider];
|
||||
if (!spec) throw new Error(`unknown key provider: ${provider}`);
|
||||
let value = process.env[spec.env] || '';
|
||||
if (!value && process.platform === 'darwin') {
|
||||
try {
|
||||
value = execFileSync(
|
||||
'security',
|
||||
['find-generic-password', '-s', spec.service, '-a', spec.account, '-w'],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
).trim();
|
||||
} catch {
|
||||
value = '';
|
||||
}
|
||||
}
|
||||
if (!value) throw new Error(`${spec.env} is not available (env or Keychain)`);
|
||||
cache.set(provider, value);
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
/**
|
||||
* Shared voice-routing phrase tables.
|
||||
*
|
||||
* CORE_PHRASES is the routing table used by `scripts/qa-voice-routing.mjs`
|
||||
* (layer 1) and the core suite of `scripts/qa-voice-bench.mjs`.
|
||||
*
|
||||
* `expect` tool name the model must call (string), every tool in an
|
||||
* array, or `{ oneOf: [...] }` for documented acceptable variance.
|
||||
* `expectNone` conversational turns that must NOT call tools.
|
||||
* `args` spot-check subset matched against the first matching call's
|
||||
* args (substring match for strings, exact for booleans/numbers).
|
||||
* `argsByTool` the same spot check, keyed by tool, for multi-intent phrases.
|
||||
* `typedArgs` benchmark-only matchers for typed arguments (exact enum
|
||||
* values, layer ids, thresholds, sort direction). They feed the
|
||||
* benchmark's full-call accuracy and never change the
|
||||
* qa-voice-routing verdict.
|
||||
*
|
||||
* COVERAGE_PROBE is the newer-layers coverage probe: utterances about layers
|
||||
* and questions that today's tool surface only partly supports. Each item is
|
||||
* graded into correct / honest-refusal / hallucinated-capability / wrong-tool
|
||||
* by `gradeCoverage` in ./grade.mjs.
|
||||
*/
|
||||
|
||||
import { call, where } from './grade.mjs';
|
||||
|
||||
export const CORE_PHRASES = Object.freeze([
|
||||
// — navigation & framing —
|
||||
{ phrase: 'Take me to Tokyo', expect: 'fly_to_location', typedArgs: [where('fly_to_location', (a) => a.locationId === 'tokyo' || /tokyo/i.test(a.query || ''))] },
|
||||
{ phrase: 'Fly to the Golden Gate Bridge', expect: 'fly_to_location' },
|
||||
{ phrase: 'Go to Sixth Street in Austin', expect: 'fly_to_location' },
|
||||
{ phrase: 'Show me the Alps from above', expect: { oneOf: ['fly_to_location', 'frame_overhead'] } },
|
||||
{ phrase: 'Zoom in a bit', expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'in' && a.amount === 'little')] },
|
||||
{ phrase: 'Zoom out a little', expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'out' && a.amount === 'little')] },
|
||||
{ phrase: 'Zoom out to a globe view', expect: 'zoom_to_globe' },
|
||||
{ phrase: 'Show me the whole earth', expect: 'zoom_to_globe' },
|
||||
{ phrase: 'Frame the aircraft near us from overhead', expect: 'frame_overhead', typedArgs: [where('frame_overhead', (a) => a.target === 'flights')] },
|
||||
|
||||
// — the satellites trap: data layer, never basemap —
|
||||
{ phrase: 'Show me the satellites', expect: { oneOf: ['set_layer_visibility', 'frame_overhead'] } },
|
||||
{ phrase: 'Turn off the satellites', expect: 'set_layer_visibility', args: { layerId: 'satellites' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'satellites' && a.enabled === false)] },
|
||||
{ phrase: 'Switch to Bing aerial', expect: 'set_map_stack', typedArgs: [where('set_map_stack', (a) => a.stack === 'bing-aerial')] },
|
||||
{ phrase: 'Switch the basemap to OSM', expect: 'set_map_stack', typedArgs: [where('set_map_stack', (a) => a.stack === 'osm')] },
|
||||
|
||||
// — layers —
|
||||
{ phrase: 'Turn on the flights layer', expect: 'set_layer_visibility', args: { layerId: 'flights' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'flights' && a.enabled === true)] },
|
||||
{ phrase: 'Show me live vessels', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'ais-live-vessels' && a.enabled === true)] },
|
||||
{ phrase: 'Turn on the fires layer', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'local-firms' && a.enabled === true)] },
|
||||
{ phrase: 'Turn on street traffic', expect: 'set_layer_visibility', args: { layerId: 'traffic' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'traffic' && a.enabled === true)] },
|
||||
{ phrase: 'Open the data layers menu', expect: 'show_data_layers_menu' },
|
||||
{ phrase: 'Show me the datacenter layers', expect: 'show_data_layers_menu', typedArgs: [where('show_data_layers_menu', (a) => a.layerId === 'local-datacenters')] },
|
||||
{ phrase: 'Turn on the datacenters layer', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'local-datacenters' && a.enabled === true)] },
|
||||
|
||||
// — visual styles & post-fx —
|
||||
{ phrase: 'Give me night vision', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'surveillance')] },
|
||||
{ phrase: 'Switch to thermal view', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'thermal')] },
|
||||
{ phrase: 'Back to the normal look', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'normal')] },
|
||||
{ phrase: 'Turn on bloom', expect: 'set_post_processing', typedArgs: [where('set_post_processing', (a) => a.bloom?.enabled === true)] },
|
||||
{ phrase: 'Sharpen the image a touch', expect: 'set_post_processing', typedArgs: [where('set_post_processing', (a) => a.sharpen && a.sharpen.enabled !== false)] },
|
||||
|
||||
// — HUD / detection / panels —
|
||||
{ phrase: 'Turn the HUD off', expect: 'set_hud', typedArgs: [where('set_hud', (a) => a.visible === 'off')] },
|
||||
{ phrase: 'Switch to the tactical layout', expect: 'set_hud', typedArgs: [where('set_hud', (a) => a.layout === 'tactical')] },
|
||||
{ phrase: 'Turn on detection', expect: 'set_detection', typedArgs: [where('set_detection', (a) => a.enabled !== false)] },
|
||||
{ phrase: 'Set detection density to fifty percent', expect: 'set_detection', typedArgs: [where('set_detection', (a) => a.densityPct === 50 || a.mode === 'balanced')] },
|
||||
|
||||
// — context questions —
|
||||
{ phrase: 'What am I looking at right now?', expect: 'get_entity_context' },
|
||||
{ phrase: 'What city is this below us?', expect: 'get_entity_context' },
|
||||
{ phrase: 'Is there anything interesting in view?', expect: 'get_entity_context' },
|
||||
|
||||
// — tracking —
|
||||
// Context-free text turns may reasonably look before tracking; either
|
||||
// routing is correct (production sessions always carry screen context).
|
||||
{ phrase: 'Track that plane', expect: { oneOf: ['track_entity', 'get_entity_context'] } },
|
||||
// The session instructions route "nearest" through analyst_query first
|
||||
// (sortBy distance, limit 1); stubbed results make that first hop the whole
|
||||
// observable turn.
|
||||
{ phrase: 'Follow the nearest aircraft', expect: { oneOf: ['track_entity', 'get_entity_context', 'analyst_query'] } },
|
||||
{ phrase: 'Stop tracking', expect: 'stop_tracking' },
|
||||
|
||||
// — CCTV / scenes / ISS —
|
||||
{ phrase: 'Show me the nearest traffic camera', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'nearest')] },
|
||||
{ phrase: 'Turn on the camera viewsheds', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'viewshed' && a.enabled !== false)] },
|
||||
{ phrase: 'Play a news radio station near Austin', expect: 'control_radio', args: { action: 'select', category: 'news', locationId: 'austin' }, typedArgs: [where('control_radio', (a) => a.action === 'select' && a.category === 'news' && a.locationId === 'austin')] },
|
||||
{ phrase: 'Turn on the radio', expect: 'control_radio', args: { action: 'play' }, typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
{ phrase: 'Set the radio volume to thirty percent', expect: 'control_radio', args: { action: 'volume', volumePct: 30 } },
|
||||
{ phrase: 'Pause the radio', expect: 'control_radio', args: { action: 'pause' }, typedArgs: [where('control_radio', (a) => a.action === 'pause')] },
|
||||
{ phrase: 'Stop the radio', expect: 'control_radio', args: { action: 'stop' }, typedArgs: [where('control_radio', (a) => a.action === 'stop')] },
|
||||
{ phrase: 'When does the ISS pass over next?', expect: 'next_iss_pass' },
|
||||
|
||||
// — annotations —
|
||||
{ phrase: 'Annotate the Texas State Capitol and its grounds', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Outline the state of Texas', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Outline Lady Bird Lake', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Draw the walking route from the Capitol to Zilker Park', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'route' && (x.mode ?? 'walking') === 'walking'))] },
|
||||
{ phrase: 'How far is the Eiffel Tower from the Louvre?', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'arrow'))] },
|
||||
{ phrase: 'Clear the map', expect: 'clear_annotations' },
|
||||
|
||||
// — multi-intent (assert ALL tools fire before speech) —
|
||||
{
|
||||
phrase: 'Switch to night vision and turn on the flights layer',
|
||||
expect: ['set_visual_style', 'set_layer_visibility'],
|
||||
typedArgs: [where('set_visual_style', (a) => a.style === 'surveillance'), where('set_layer_visibility', (a) => a.layerId === 'flights' && a.enabled === true)],
|
||||
},
|
||||
{
|
||||
phrase: 'Turn off the HUD and take me to Paris',
|
||||
expect: ['set_hud', 'fly_to_location'],
|
||||
typedArgs: [where('set_hud', (a) => a.visible === 'off'), where('fly_to_location', (a) => a.locationId === 'paris' || /paris/i.test(a.query || ''))],
|
||||
},
|
||||
{
|
||||
phrase: 'Go to full planet view and then turn on the radio',
|
||||
expect: ['zoom_to_globe', 'control_radio'],
|
||||
argsByTool: { control_radio: { action: 'play' } },
|
||||
},
|
||||
|
||||
// — camera verbs (tools #23/#24) —
|
||||
{ phrase: 'Orbit around this area slowly', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'orbit' && a.speed === 'slow')] },
|
||||
{ phrase: 'Pan left a bit', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'pan' && a.direction === 'left')] },
|
||||
{ phrase: 'Stop moving the camera', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'stop')] },
|
||||
{ phrase: 'Fly the route we just drew', expect: 'fly_route' },
|
||||
|
||||
// — analyst queries (tool #22) —
|
||||
{ phrase: 'How many flights are over Texas right now?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('flights') && a.scope?.kind === 'region' && /texas/i.test(a.scope?.name || ''))] },
|
||||
{ phrase: 'Which ships are headed to Oakland?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('ais-live-vessels') && (a.filters || []).some((f) => f.field === 'destination' && /oakland/i.test(String(f.value))))] },
|
||||
{ phrase: 'What is the biggest fire near Los Angeles?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('local-firms') && ['frp', 'acres'].includes(a.sortBy) && a.sortDir !== 'asc')] },
|
||||
{ phrase: 'Is anything flying above forty thousand feet?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).some((l) => l === 'flights' || l === 'military') && (a.filters || []).some((f) => f.field === 'altitudeM' && (f.op === 'gt' || f.op === 'gte') && Math.abs(Number(f.value) - 12192) <= 50))] },
|
||||
|
||||
// — negative controls: conversation must NOT tool-call —
|
||||
{ phrase: 'How is your evening going?', expectNone: true },
|
||||
{ phrase: 'Tell me a fun fact about maps', expectNone: true },
|
||||
]);
|
||||
|
||||
// ── Coverage probe ──────────────────────────────────────────────────────────
|
||||
//
|
||||
// Text-mode port of the voice-coverage routing probe. The live probe ran each
|
||||
// utterance against a real app page (setup steps, real tool results); this
|
||||
// benchmark has no app, so every item states the scene it assumes in `scene`
|
||||
// and the model sees only stub tool results. Grading therefore judges the
|
||||
// ROUTING DECISION and the honesty of the final words, not app behaviour.
|
||||
//
|
||||
// supported true when today's tools can satisfy the ask (a refusal is a
|
||||
// miss); false when the best possible outcome is an honest
|
||||
// refusal, optionally after a neutral helper call.
|
||||
// correct call patterns that satisfy the ask (any alternative).
|
||||
// neutral extra tools that neither help nor hurt (navigation to the
|
||||
// named place, enabling a related layer that exists).
|
||||
// hallucinated call patterns that fake the capability (e.g. toggling the
|
||||
// satellites DATA layer for "recent satellite imagery").
|
||||
// claimText final-text pattern that asserts data the model cannot have.
|
||||
//
|
||||
// Any call whose arguments violate the tool schema (an invented layer id) is
|
||||
// graded hallucinated-capability regardless of the item.
|
||||
//
|
||||
// Two rubrics, so a before/after comparison stays honest:
|
||||
// v1 the original rubric, graded against the tool schema of 4b56d0e9
|
||||
// (baseline-tools-4b56d0e9.json) — "by the old yardstick".
|
||||
// v2 `item.v2` overrides (supported / correct / hallucinated / neutral),
|
||||
// graded against the CURRENT production tools after the voice layer
|
||||
// manifest and point-and-ask landed. Use
|
||||
// coverageRubric(item, 'v2') to get the merged item.
|
||||
|
||||
const hasRoute = (c) =>
|
||||
c.name === 'annotate_map' &&
|
||||
Array.isArray(c.args?.annotations) &&
|
||||
c.args.annotations.some((a) => a?.type === 'route');
|
||||
|
||||
/** The coverage item as graded under rubric 'v1' (original) or 'v2' (current tools). */
|
||||
export function coverageRubric(item, version = 'v1') {
|
||||
if (version !== 'v2' || !item.v2) return item;
|
||||
return { ...item, ...item.v2 };
|
||||
}
|
||||
|
||||
export const COVERAGE_PROBE = Object.freeze([
|
||||
{
|
||||
id: 'quakes-m5-week',
|
||||
v2: { supported: true },
|
||||
utterance: 'Show me earthquakes above magnitude 5 this week',
|
||||
scene: 'globe, no layers on',
|
||||
expect: 'enable earthquakes + analyst_query magnitude>=5; must say feed covers last 24h only (no week)',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('analyst_query', { layers: 'earthquakes' })],
|
||||
[call('set_layer_visibility', { layerId: 'earthquakes', enabled: true })],
|
||||
],
|
||||
neutral: ['zoom_to_globe'],
|
||||
},
|
||||
{
|
||||
id: 'trains-london',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'transit' })], [call('set_layer_visibility', { layerId: 'transit', enabled: true })]], hallucinated: [] },
|
||||
utterance: 'How many trains are moving in London right now?',
|
||||
scene: 'London',
|
||||
expect: 'honest "no transit data for London / transit not queryable" — no invented count',
|
||||
supported: false,
|
||||
neutral: ['fly_to_location'],
|
||||
hallucinated: [call('analyst_query')],
|
||||
claimText: /\b\d+\s+(trains?|services)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'transit-on',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'transit', enabled: true })]] },
|
||||
utterance: 'Turn on transit',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility transit (no transit id in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('set_layer_visibility', { layerId: 'traffic', enabled: true })],
|
||||
},
|
||||
{
|
||||
id: 'ship-fastest-singapore',
|
||||
v2: { supported: true },
|
||||
utterance: 'Which ship near Singapore is fastest?',
|
||||
scene: 'Singapore Strait, vessels layer on',
|
||||
expect: 'analyst_query ais-live-vessels near Singapore sortBy speedKts desc',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'ais-live-vessels' })]],
|
||||
neutral: ['fly_to_location', 'set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'recent-imagery',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'recent-imagery', enabled: true })]] },
|
||||
utterance: 'Show recent satellite imagery of this area',
|
||||
scene: 'Austin',
|
||||
expect: 'enable recent-imagery for current view (no tool today) — must NOT toggle satellites data layer or silently switch basemap',
|
||||
supported: false,
|
||||
hallucinated: [
|
||||
call('set_layer_visibility', { layerId: 'satellites' }),
|
||||
call('set_map_stack'),
|
||||
call('frame_overhead', { target: 'satellites' }),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'wind-here',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'wind', enabled: true })]] },
|
||||
utterance: "What's the wind doing here?",
|
||||
scene: 'Austin',
|
||||
expect: 'enable/inspect wind layer at view centre (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b\d+\s*(mph|km\/?h|knots?|kts|m\/s|meters per second|miles per hour)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'cyclones-active',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'weather-cyclones' })], [call('set_layer_visibility', { layerId: 'weather-cyclones', enabled: true })]] },
|
||||
utterance: 'Are there any cyclones active right now?',
|
||||
scene: 'globe',
|
||||
expect: 'enable cyclones + list active storms (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b(there (are|is)|currently)\b[^.?!]*\b(cyclones?|storms?|hurricanes?|typhoons?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'fires-in-drawn-area',
|
||||
// There is no drawn-area scope yet; the prompt says to say so. Any count
|
||||
// presented as inside the drawn area is a faked capability.
|
||||
v2: {
|
||||
supported: false,
|
||||
correct: [],
|
||||
hallucinated: [where('analyst_query', (a) => (a.layers || []).includes('local-firms'))],
|
||||
},
|
||||
utterance: 'Count the fires inside the area I drew',
|
||||
scene: 'Los Angeles, fires layer on, an area drawn around Angeles National Forest',
|
||||
expect: 'analyst_query local-firms scoped to the drawn polygon (no polygon scope today)',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'local-firms' })]],
|
||||
},
|
||||
{
|
||||
id: 'local-adsb-on',
|
||||
v2: { supported: true },
|
||||
utterance: 'Turn on the local ADS-B layer',
|
||||
scene: 'globe',
|
||||
expect: 'set_layer_visibility local-adsb; narrate receiver state honestly',
|
||||
supported: true,
|
||||
correct: [[call('set_layer_visibility', { layerId: 'local-adsb', enabled: true })]],
|
||||
},
|
||||
{
|
||||
id: 'bikes-near-me',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'bikeshare' })]] },
|
||||
utterance: 'How many bikes are available near me?',
|
||||
scene: 'New York, bikeshare layer on',
|
||||
expect: 'count available bikes at nearby stations (bikeshare not in analyst today) — honest otherwise',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
claimText: /\b\d+\s+(available\s+)?(bikes?|bicycles?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'rocket-launch-track',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'rocket-launches', enabled: true })], [call('set_context_mode', { mode: 'space-missions' })], [call('set_context_mode', { mode: 'missions' })], [call('analyst_query', { layers: 'rocket-launches' })]] },
|
||||
utterance: 'Track the next rocket launch',
|
||||
scene: 'globe',
|
||||
expect: 'enable rocket-launches / space-missions and select next launch (track_entity cannot) — honest otherwise',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('set_layer_visibility', { layerId: 'rocket-launches', enabled: true })],
|
||||
[call('set_context_mode', { mode: 'space-missions' })],
|
||||
[call('set_context_mode', { mode: 'missions' })],
|
||||
],
|
||||
hallucinated: [call('track_entity')],
|
||||
},
|
||||
{
|
||||
id: 'weather-radar-on',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-radar', enabled: true })]] },
|
||||
utterance: 'Turn on the weather radar',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility weather-radar (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('set_map_stack'), call('set_visual_style')],
|
||||
},
|
||||
{
|
||||
id: 'lightning',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-lightning', enabled: true })]] },
|
||||
utterance: 'Show me lightning strikes',
|
||||
scene: 'globe',
|
||||
expect: 'set_layer_visibility weather-lightning (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
},
|
||||
{
|
||||
id: 'military-bases',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'military-installations', enabled: true })], [call('analyst_query', { layers: 'military-installations' })]], hallucinated: [] },
|
||||
utterance: 'Show military bases near here',
|
||||
scene: 'San Diego',
|
||||
expect: 'set_layer_visibility military-installations (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('annotate_map')],
|
||||
},
|
||||
{
|
||||
id: 'driving-directions',
|
||||
v2: { supported: true, correct: [[hasRoute], [call('set_layer_visibility', { layerId: 'directions', enabled: true })]] },
|
||||
utterance: 'Give me driving directions from the Texas Capitol to the Austin airport',
|
||||
scene: 'Austin',
|
||||
expect: 'annotate_map route driving (or directions layer)',
|
||||
supported: true,
|
||||
correct: [[hasRoute]],
|
||||
neutral: ['fly_to_location'],
|
||||
},
|
||||
{
|
||||
id: 'satellites-overhead-count',
|
||||
v2: { supported: true },
|
||||
utterance: 'How many satellites are overhead right now?',
|
||||
scene: 'globe, satellites layer on',
|
||||
expect: 'analyst_query satellites (control: supported)',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('analyst_query', { layers: 'satellites' })],
|
||||
[call('frame_overhead', { target: 'satellites' })],
|
||||
],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'biggest-quake-today',
|
||||
v2: { supported: true },
|
||||
utterance: "What's the biggest earthquake today?",
|
||||
scene: 'globe, earthquakes layer on',
|
||||
expect: 'analyst_query earthquakes sortBy magnitude desc (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'earthquakes' })]],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'flock-on',
|
||||
v2: { supported: true },
|
||||
utterance: 'Turn on the Flock cameras',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility alpr-cameras (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('set_layer_visibility', { layerId: 'alpr-cameras', enabled: true })]],
|
||||
hallucinated: [call('control_cctv')],
|
||||
},
|
||||
{
|
||||
id: 'alpr-count',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'alpr-cameras' })]] },
|
||||
utterance: 'How many license plate cameras are around here?',
|
||||
scene: 'Austin, ALPR layer on',
|
||||
expect: 'count alpr-cameras in view (not in analyst today) — honest otherwise',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
claimText: /\b\d+\s+(license|plate|alpr|flock|cameras?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'nearest-bus',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'transit' })]] },
|
||||
utterance: 'Which bus is closest to the center of the screen?',
|
||||
scene: 'Austin, transit layer on',
|
||||
expect: 'nearest transit vehicle (no transit query/context today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b(route|bus|line)\s+#?\d+\b/i,
|
||||
},
|
||||
{
|
||||
id: 'weather-replay',
|
||||
v2: { supported: false, hallucinated: [call('control_scene')] },
|
||||
utterance: 'Replay the last hour of weather radar',
|
||||
scene: 'globe',
|
||||
expect: 'weather clock replay (no time tool today) — honest refusal',
|
||||
supported: false,
|
||||
},
|
||||
{
|
||||
id: 'receiver-count',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'local-adsb' })]], hallucinated: [call('frame_overhead')] },
|
||||
utterance: 'How many aircraft is my receiver picking up?',
|
||||
scene: 'globe, no receiver connected',
|
||||
expect: 'count local-adsb (not in analyst) — honest: no receiver connected',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
hallucinated: [call('analyst_query'), call('frame_overhead')],
|
||||
claimText: /\b\d+\s+(aircraft|planes?|contacts?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'nearest-traffic-camera',
|
||||
v2: { supported: true },
|
||||
utterance: 'Show me the nearest traffic camera',
|
||||
scene: 'Austin',
|
||||
expect: 'control_cctv nearest (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('control_cctv', { action: 'nearest' })]],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'hurricane-strongest',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-cyclones', enabled: true }), call('analyst_query', { layers: 'weather-cyclones' })], [call('analyst_query', { layers: 'weather-cyclones' })]], hallucinated: [] },
|
||||
utterance: 'Turn on the hurricane tracks and fly me to the strongest storm',
|
||||
scene: 'globe',
|
||||
expect: 'enable cyclones + fly to strongest (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('fly_to_location')],
|
||||
},
|
||||
]);
|
||||
|
||||
// ── Multi-turn dialogues ───────────────────────────────────────────────────
|
||||
//
|
||||
// Two-turn dialogues where turn 2 only makes sense given turn 1. Both turns
|
||||
// run in ONE session/context. Turn 1's tools get the plausible canned results
|
||||
// in `results` (anything not listed gets the neutral stub), so turn 2 has
|
||||
// something real to refer back to. `first` and `second` use the CORE_PHRASES
|
||||
// grading shape (expect / oneOf / expectNone / typedArgs); the headline score
|
||||
// is turn 2's full-call accuracy.
|
||||
|
||||
const flightsTexas = {
|
||||
ok: true,
|
||||
count: 212,
|
||||
scopeLabel: '212 flights over Texas',
|
||||
feedProvenance: { overall: 'nominal' },
|
||||
examples: [
|
||||
{ callsign: 'AAL1402', operator: 'American Airlines', altitudeM: 10972, military: false },
|
||||
{ callsign: 'SWA2210', operator: 'Southwest Airlines', altitudeM: 11278, military: false },
|
||||
{ callsign: 'RCH451', operator: 'US Air Force', altitudeM: 9144, military: true },
|
||||
],
|
||||
coverage: 'Counts cover data loaded by enabled layers.',
|
||||
};
|
||||
const shipsOakland = {
|
||||
ok: true,
|
||||
count: 4,
|
||||
scopeLabel: '4 ships headed to Oakland',
|
||||
feedProvenance: { overall: 'nominal' },
|
||||
examples: [
|
||||
{ name: 'MAERSK KENSINGTON', destination: 'OAKLAND', speedKts: 14.2, distanceKm: 180 },
|
||||
{ name: 'EVER LOVELY', destination: 'OAKLAND', speedKts: 16.8, distanceKm: 410 },
|
||||
{ name: 'APL SINGAPURA', destination: 'OAKLAND', speedKts: 12.1, distanceKm: 62 },
|
||||
],
|
||||
};
|
||||
|
||||
export const DIALOGUES = Object.freeze([
|
||||
{
|
||||
id: 'flights-then-military',
|
||||
turns: ['How many flights are over Texas right now?', 'Now just the military ones'],
|
||||
results: { analyst_query: [flightsTexas, { ...flightsTexas, count: 9, scopeLabel: '9 military aircraft over Texas', examples: [flightsTexas.examples[2]] }] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: {
|
||||
expect: 'analyst_query',
|
||||
typedArgs: [where('analyst_query', (a) => a.followUp === true || (a.layers || []).includes('military') || (a.filters || []).some((f) => f.field === 'military'))],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'quakes-then-biggest',
|
||||
turns: ['Turn on the earthquakes layer', 'Take me to the biggest one'],
|
||||
results: {
|
||||
set_layer_visibility: [{ ok: true, layerId: 'earthquakes', enabled: true, feedState: 'nominal', count: 143 }],
|
||||
analyst_query: [null, { ok: true, count: 143, results: [{ id: 'us7000abcd', place: 'Kermadec Islands', magnitude: 6.4, latitude: -29.9, longitude: -177.8 }] }],
|
||||
},
|
||||
first: { expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'earthquakes' && a.enabled === true)] },
|
||||
second: {
|
||||
expect: { oneOf: ['track_entity', 'analyst_query'] },
|
||||
typedArgs: [(c) =>
|
||||
(c.name === 'track_entity' && /quake|biggest|largest|strongest|kermadec/i.test(c.args?.query || '')) ||
|
||||
(c.name === 'analyst_query' && (c.args?.layers || []).includes('earthquakes') && c.args?.sortBy === 'magnitude' && c.args?.sortDir !== 'asc')],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'track-then-stop',
|
||||
turns: ['Track UAL428', 'Stop'],
|
||||
results: {
|
||||
track_entity: [{ ok: true, tracking: 'UAL428', layerId: 'flights' }],
|
||||
move_camera: [null, { ok: true, stopped: false, tracking: true, note: 'No camera motion was active; an entity is being tracked.' }],
|
||||
},
|
||||
first: { expect: 'track_entity', typedArgs: [where('track_entity', (a) => /UAL\s*428/i.test(a.query || ''))] },
|
||||
second: { expect: 'stop_tracking' },
|
||||
},
|
||||
{
|
||||
id: 'paris-then-closer',
|
||||
turns: ['Take me to Paris', 'Get a little closer'],
|
||||
results: { fly_to_location: [{ ok: true, arrived: true, place: 'Paris, France' }] },
|
||||
first: { expect: 'fly_to_location' },
|
||||
second: { expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'in')] },
|
||||
},
|
||||
{
|
||||
id: 'thermal-then-back',
|
||||
turns: ['Switch to thermal view', 'Actually, put it back the way it was'],
|
||||
results: { set_visual_style: [{ ok: true, style: 'thermal', previousStyle: 'normal' }] },
|
||||
first: { expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'thermal')] },
|
||||
second: { expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'normal')] },
|
||||
},
|
||||
{
|
||||
id: 'radio-then-volume',
|
||||
turns: ['Turn on the radio', 'Turn it down to twenty percent'],
|
||||
results: { control_radio: [{ ok: true, prepared: true, action: 'play', station: 'KUT 90.5 Austin' }] },
|
||||
first: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
second: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'volume' && a.volumePct === 20)] },
|
||||
},
|
||||
{
|
||||
id: 'mark-then-route',
|
||||
turns: ['Mark the Texas Capitol on the map', 'Now draw the walking route from there to Zilker Park'],
|
||||
results: { annotate_map: [{ ok: true, placed: [{ label: 'Texas State Capitol', type: 'pin' }] }] },
|
||||
first: { expect: 'annotate_map' },
|
||||
second: {
|
||||
expect: 'annotate_map',
|
||||
typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) =>
|
||||
x.type === 'route' && (x.mode ?? 'walking') === 'walking' && /capitol/i.test(x.points?.[0]?.target || '')))],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'ships-then-closest',
|
||||
turns: ['Which ships are headed to Oakland?', 'Which of those is closest?'],
|
||||
results: { analyst_query: [shipsOakland, { ...shipsOakland, count: 1, examples: [shipsOakland.examples[2]] }] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: { expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => a.followUp === true && a.sortBy === 'distance')] },
|
||||
},
|
||||
{
|
||||
id: 'flights-then-thanks',
|
||||
turns: ['Turn on the flights layer', "Thanks, that's perfect"],
|
||||
results: { set_layer_visibility: [{ ok: true, layerId: 'flights', enabled: true, feedState: 'nominal', count: 1840 }] },
|
||||
first: { expect: 'set_layer_visibility' },
|
||||
second: { expectNone: true },
|
||||
},
|
||||
]);
|
||||
|
||||
/** Canned result for a dialogue tool call, or null for the neutral stub. */
|
||||
export function dialogueResult(dialogue, toolName, turn) {
|
||||
const byTurn = dialogue.results?.[toolName];
|
||||
if (!byTurn) return null;
|
||||
return byTurn[turn] ?? null;
|
||||
}
|
||||
|
||||
// ── New capabilities (point-and-ask, layer reach) ─────────────────────────
|
||||
//
|
||||
// ROUTING_PHRASES is the qa-voice-routing table, so that harness runs exactly
|
||||
// its own table. The benchmark runs CAPABILITY_EXTRAS as the separate
|
||||
// `capability` suite so the core suite stays comparable across tool versions.
|
||||
|
||||
/** The full qa-voice-routing table. */
|
||||
export const ROUTING_PHRASES = CORE_PHRASES;
|
||||
|
||||
// Pointer context items in the app's own format (src/voice/pointerContext.js
|
||||
// output), sent as an inert system item just before the user's words.
|
||||
const AIRCRAFT_POINTER = {
|
||||
type: 'pointer_context', at: 'speech_start',
|
||||
target: { kind: 'aircraft', layer: 'flights', label: 'UPS793' },
|
||||
ground: { lat: 30.3121, lon: -97.6013 }, screen: { x: 0.608, y: 0.408 },
|
||||
};
|
||||
const GROUND_POINTER = {
|
||||
type: 'pointer_context', at: 'speech_start',
|
||||
ground: { lat: 30.2849, lon: -97.7341 }, screen: { x: 0.427, y: 0.547 },
|
||||
};
|
||||
const isPointer = (v) => String(v || '').toLowerCase() === 'pointer';
|
||||
const flightsInView = {
|
||||
ok: true,
|
||||
count: 3,
|
||||
scopeLabel: 'in view',
|
||||
items: [
|
||||
{ n: 1, id: 'a1b2c3', callsign: 'UAL428', layerKey: 'flights', lat: 30.2, lon: -97.7 },
|
||||
{ n: 2, id: 'a4b5c6', callsign: 'SWA1201', layerKey: 'flights', lat: 30.3, lon: -97.6 },
|
||||
{ n: 3, id: 'a7b8c9', callsign: 'AAL88', layerKey: 'flights', lat: 30.1, lon: -97.8 },
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* Extra single-turn phrases (with an optional pointer `context`) and two-turn
|
||||
* dialogues for the new capabilities. Items with `turns` are graded like
|
||||
* DIALOGUES (turn 2 full-call); the rest like CORE_PHRASES.
|
||||
*/
|
||||
export const CAPABILITY_EXTRAS = Object.freeze([
|
||||
{ phrase: 'Tell me about this', context: AIRCRAFT_POINTER, expect: 'get_entity_context', typedArgs: [where('get_entity_context', (a) => isPointer(a.scope))] },
|
||||
{ phrase: 'How many flights are around here?', context: GROUND_POINTER, expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => isPointer(a.scope?.kind))] },
|
||||
{ phrase: 'Take me there', context: GROUND_POINTER, expect: 'fly_to_location', typedArgs: [where('fly_to_location', (a) => isPointer(a.query))] },
|
||||
{ phrase: 'Track that one', context: AIRCRAFT_POINTER, expect: 'track_entity', typedArgs: [where('track_entity', (a) => isPointer(a.query))] },
|
||||
{ phrase: 'Turn on the weather radar', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'weather-radar' && a.enabled === true)] },
|
||||
{ phrase: 'Are there any hurricanes right now?', expect: { oneOf: ['analyst_query', 'set_layer_visibility'] }, typedArgs: [(c) => (c.args?.layers || []).includes('weather-cyclones') || c.args?.layerId === 'weather-cyclones'] },
|
||||
{ phrase: 'Which bus is closest to the middle of the screen?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('transit'))] },
|
||||
{
|
||||
id: 'flights-then-second',
|
||||
turns: ['How many flights are in view?', 'Track the second one'],
|
||||
results: { analyst_query: [flightsInView] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: { expect: 'track_entity', typedArgs: [where('track_entity', (a) => Number(a.referent) === 2 || /SWA\s*1201/i.test(a.query || ''))] },
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Cost estimation for benchmark turns (USD). Pure.
|
||||
*
|
||||
* OpenAI Realtime: the app's own rate table (`src/voice/voiceCost.js`).
|
||||
* `gpt-realtime-2.1` has no registry entry, so it resolves to the priciest
|
||||
* known table (gpt-realtime-2) — which matches OpenAI's published 2.1 rates
|
||||
* as read on 2026-09-23 ($4 / $0.40 cached / $24 per 1M text tokens).
|
||||
*
|
||||
* Gemini Live: https://ai.google.dev/gemini-api/docs/pricing, read 2026-09-23,
|
||||
* gemini-3.8-live paid tier, USD per 1M tokens. Thinking tokens bill as text
|
||||
* output. The native-audio Live model rejects TEXT output, so the benchmark
|
||||
* requests AUDIO and the spoken confirmation bills at the audio-output rate.
|
||||
*/
|
||||
import { estimateUsageCostUsd, resolveVoiceModelById } from '../../src/voice/voiceCost.js';
|
||||
|
||||
export const GEMINI_RATES = Object.freeze({
|
||||
'gemini-3.8-live': Object.freeze({
|
||||
textInput: 0.75,
|
||||
audioInput: 3.0,
|
||||
textOutput: 4.5,
|
||||
audioOutput: 12.0,
|
||||
verifiedOn: '2026-09-23',
|
||||
}),
|
||||
});
|
||||
|
||||
/** Cost of one Realtime `response.usage` for `model`. */
|
||||
export function openAiUsageCostUsd(model, usage) {
|
||||
return estimateUsageCostUsd(usage, resolveVoiceModelById(model).rates);
|
||||
}
|
||||
|
||||
const modalityCount = (details, modality) =>
|
||||
(Array.isArray(details) ? details : [])
|
||||
.filter((d) => String(d?.modality || '').toUpperCase() === modality)
|
||||
.reduce((sum, d) => sum + (Number(d?.tokenCount) || 0), 0);
|
||||
|
||||
/**
|
||||
* Cost of one Gemini Live `usageMetadata`. Unattributed prompt tokens bill
|
||||
* as audio input and unattributed response tokens as audio output — the
|
||||
* conservative direction for a spend cap.
|
||||
*/
|
||||
export function geminiUsageCostUsd(model, usage) {
|
||||
const rates = GEMINI_RATES[model] || GEMINI_RATES['gemini-3.8-live'];
|
||||
if (!usage) return 0;
|
||||
const prompt = Number(usage.promptTokenCount) || 0;
|
||||
const response = Number(usage.responseTokenCount) || 0;
|
||||
const thoughts = Number(usage.thoughtsTokenCount) || 0;
|
||||
const textIn = modalityCount(usage.promptTokensDetails, 'TEXT');
|
||||
const audioIn = Math.max(0, prompt - textIn);
|
||||
const textOut = modalityCount(usage.responseTokensDetails, 'TEXT');
|
||||
const audioOut = Math.max(0, response - textOut);
|
||||
const usd =
|
||||
(textIn * rates.textInput +
|
||||
audioIn * rates.audioInput +
|
||||
(textOut + thoughts) * rates.textOutput +
|
||||
audioOut * rates.audioOutput) /
|
||||
1_000_000;
|
||||
return Number.isFinite(usd) && usd > 0 ? usd : 0;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Capture the exact Realtime session config production mints.
|
||||
*
|
||||
* Rather than re-typing the session fields, this drives the production token
|
||||
* handler (`server/providers/openai/realtime.js`) with a fake upstream fetch
|
||||
* and returns the body it would have POSTed to OpenAI. Instructions, tools,
|
||||
* tool_choice, reasoning, and truncation therefore cannot drift from what a
|
||||
* real GEV mic session receives.
|
||||
*/
|
||||
import { createRealtimeTokenHandler } from '../../server/providers/openai/realtime.js';
|
||||
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
|
||||
|
||||
export { realtimeInstructions, GEV_REALTIME_TOOLS };
|
||||
|
||||
/** Return production's `{ session: {...} }` client-secret body for `model`. */
|
||||
export async function captureProductionSession(model) {
|
||||
let captured = null;
|
||||
const handler = createRealtimeTokenHandler({
|
||||
resolveApiKey: () => 'bench-placeholder',
|
||||
models: { standard: model, mini: model },
|
||||
fetchImpl: async (_url, init) => {
|
||||
captured = JSON.parse(init.body);
|
||||
return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
},
|
||||
});
|
||||
const res = {
|
||||
statusCode: 0,
|
||||
headers: {},
|
||||
setHeader(k, v) { this.headers[k] = v; },
|
||||
end() {},
|
||||
};
|
||||
await handler({ method: 'POST', url: '/api/realtime/token', headers: {}, socket: {} }, res);
|
||||
if (!captured?.session) throw new Error('production token handler did not produce a session config');
|
||||
return captured.session;
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
/**
|
||||
* Pure aggregation of benchmark rows into summary.json / summary.md.
|
||||
* Rows: `{ type:'run', ... }` run metadata and `{ type:'turn', ... }` turns.
|
||||
*/
|
||||
|
||||
export function percentile(values, p) {
|
||||
const xs = values.filter((v) => typeof v === 'number' && Number.isFinite(v)).sort((a, b) => a - b);
|
||||
if (!xs.length) return null;
|
||||
const idx = Math.min(xs.length - 1, Math.max(0, Math.ceil((p / 100) * xs.length) - 1));
|
||||
return xs[idx];
|
||||
}
|
||||
|
||||
const pct = (n, d) => (d ? Math.round((1000 * n) / d) / 10 : null);
|
||||
const sum = (xs) => xs.reduce((a, b) => a + (Number(b) || 0), 0);
|
||||
const mean = (xs) => {
|
||||
const v = xs.filter((x) => typeof x === 'number' && Number.isFinite(x));
|
||||
return v.length ? sum(v) / v.length : null;
|
||||
};
|
||||
const round = (x, d = 0) => (x === null || x === undefined ? null : Math.round(x * 10 ** d) / 10 ** d);
|
||||
|
||||
export const COVERAGE_VERDICTS = Object.freeze(['correct', 'honest-refusal', 'hallucinated-capability', 'wrong-tool']);
|
||||
|
||||
export function modelKey(row) {
|
||||
return `${row.provider}:${row.model}${row.variant ? `#${row.variant}` : ''}`;
|
||||
}
|
||||
|
||||
/** Aggregate turn rows into one summary entry per provider×model(×variant). */
|
||||
export function summarize(rows) {
|
||||
const runs = rows.filter((r) => r.type === 'run');
|
||||
const turns = rows.filter((r) => r.type === 'turn');
|
||||
const keys = [...new Set(turns.map(modelKey))];
|
||||
const models = keys.map((key) => {
|
||||
const mine = turns.filter((t) => modelKey(t) === key);
|
||||
const core = mine.filter((t) => t.suite === 'core');
|
||||
const cov = mine.filter((t) => t.suite === 'coverage');
|
||||
const dia = mine.filter((t) => t.suite === 'dialogue');
|
||||
const capAll = mine.filter((t) => t.suite === 'capability');
|
||||
// Latency percentiles stay on the core + coverage phrase sets so runs of
|
||||
// different builds remain comparable.
|
||||
const single = mine.filter((t) => t.suite === 'core' || t.suite === 'coverage');
|
||||
const covV2 = cov.filter((t) => t.gradeV2);
|
||||
const reps = [...new Set(core.map((t) => t.rep))].sort((a, b) => a - b);
|
||||
const withCall = single.filter((t) => t.tFirstCallMs !== null && t.tFirstCallMs !== undefined);
|
||||
const typed = core.filter((t) => t.grade?.typedArgsOk !== null && t.grade?.typedArgsOk !== undefined);
|
||||
const diaFailures = dia.filter((t) => !t.grade?.ok).map((t) => ({
|
||||
id: t.phraseId, rep: t.rep, expect: t.expect,
|
||||
turn1: (t.turn1?.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 80)})`).join(', ') || '(none)',
|
||||
turn2: (t.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 80)})`).join(', ') || '(none)',
|
||||
}));
|
||||
const coverageV2Counts = Object.fromEntries(COVERAGE_VERDICTS.map((v) => [v, covV2.filter((t) => t.gradeV2?.verdict === v).length]));
|
||||
const capFailures = {};
|
||||
for (const t of capAll.filter((x) => !x.grade?.fullCallOk)) {
|
||||
const k = t.phraseId;
|
||||
capFailures[k] = capFailures[k] || { id: k, expect: t.expect, fails: 0, of: capAll.filter((x) => x.phraseId === k).length, called: [] };
|
||||
capFailures[k].fails += 1;
|
||||
capFailures[k].called.push((t.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 70)})`).join(', ') || '(none)');
|
||||
}
|
||||
const coverageCounts = Object.fromEntries(COVERAGE_VERDICTS.map((v) => [v, cov.filter((t) => t.grade?.verdict === v).length]));
|
||||
const coreFailures = {};
|
||||
for (const t of core.filter((x) => !x.grade?.ok)) {
|
||||
const k = t.phrase;
|
||||
coreFailures[k] = coreFailures[k] || { phrase: k, expect: t.expect, fails: 0, of: core.filter((x) => x.phrase === k).length, called: [] };
|
||||
coreFailures[k].fails += 1;
|
||||
coreFailures[k].called.push((t.calls || []).map((c) => c.name).join(',') || '(none)');
|
||||
}
|
||||
const covByItem = {};
|
||||
for (const t of cov) {
|
||||
covByItem[t.phraseId] = covByItem[t.phraseId] || { id: t.phraseId, utterance: t.phrase, supported: t.supported, verdicts: [] };
|
||||
covByItem[t.phraseId].verdicts.push(t.grade?.verdict);
|
||||
}
|
||||
const run = runs.filter((r) => modelKey(r) === key).at(-1) || {};
|
||||
const local = mine.filter((t) => t.local);
|
||||
return {
|
||||
key,
|
||||
provider: mine[0].provider,
|
||||
model: mine[0].model,
|
||||
variant: mine[0].variant || null,
|
||||
settings: run.settings || null,
|
||||
machine: run.machine || null,
|
||||
warmup: run.warmup || null,
|
||||
turns: mine.length,
|
||||
core: {
|
||||
n: core.length,
|
||||
reps: reps.length,
|
||||
accuracyPct: pct(core.filter((t) => t.grade?.ok).length, core.length),
|
||||
toolAccuracyPct: pct(core.filter((t) => t.grade?.toolOk).length, core.length),
|
||||
fullCallAccuracyPct: pct(core.filter((t) => t.grade?.fullCallOk).length, core.length),
|
||||
typedArgsAccuracyPct: pct(typed.filter((t) => t.grade?.typedArgsOk).length, typed.length),
|
||||
typedArgsN: typed.length,
|
||||
argsAccuracyPct: pct(core.filter((t) => t.grade?.argsOk === true).length, core.filter((t) => t.grade?.argsOk !== null && t.grade?.argsOk !== undefined).length),
|
||||
firstCallAccuracyPct: pct(core.filter((t) => t.grade?.firstCallOk).length, core.length),
|
||||
falsePositives: core.filter((t) => t.grade?.falsePositive).length,
|
||||
negativeTurns: core.filter((t) => t.expectNone).length,
|
||||
perRepAccuracyPct: reps.map((rep) => {
|
||||
const r = core.filter((t) => t.rep === rep);
|
||||
return pct(r.filter((t) => t.grade?.ok).length, r.length);
|
||||
}),
|
||||
failures: Object.values(coreFailures).sort((a, b) => b.fails - a.fails),
|
||||
},
|
||||
coverage: {
|
||||
n: cov.length,
|
||||
counts: coverageCounts,
|
||||
acceptablePct: pct(cov.filter((t) => t.grade?.acceptable).length, cov.length),
|
||||
supportedCorrectPct: pct(cov.filter((t) => t.supported && t.grade?.verdict === 'correct').length, cov.filter((t) => t.supported).length),
|
||||
unsupportedHonestPct: pct(cov.filter((t) => !t.supported && t.grade?.acceptable).length, cov.filter((t) => !t.supported).length),
|
||||
items: Object.values(covByItem),
|
||||
},
|
||||
coverageV2: {
|
||||
n: covV2.length,
|
||||
counts: coverageV2Counts,
|
||||
acceptablePct: pct(covV2.filter((t) => t.gradeV2?.acceptable).length, covV2.length),
|
||||
},
|
||||
capability: {
|
||||
n: capAll.length,
|
||||
toolPct: pct(capAll.filter((t) => t.grade?.toolOk).length, capAll.length),
|
||||
fullPct: pct(capAll.filter((t) => t.grade?.fullCallOk).length, capAll.length),
|
||||
failures: Object.values(capFailures).sort((a, b) => b.fails - a.fails),
|
||||
},
|
||||
prompt: run.prompt || null,
|
||||
dialogue: {
|
||||
n: dia.length,
|
||||
turn2FullPct: pct(dia.filter((t) => t.grade?.fullCallOk).length, dia.length),
|
||||
turn2ToolPct: pct(dia.filter((t) => t.grade?.toolOk).length, dia.length),
|
||||
turn1FullPct: pct(dia.filter((t) => t.grade?.turn1Ok).length, dia.length),
|
||||
turn2ToolCallP50: percentile(dia.map((t) => t.tFirstCallMs), 50),
|
||||
failures: diaFailures,
|
||||
},
|
||||
latencyMs: {
|
||||
scope: 'single-turn phrases (core + coverage); send → event, text mode',
|
||||
firstTokenP50: percentile(single.map((t) => t.tFirstTokenMs), 50),
|
||||
firstTokenP90: percentile(single.map((t) => t.tFirstTokenMs), 90),
|
||||
toolCallP50: percentile(withCall.map((t) => t.tFirstCallMs), 50),
|
||||
toolCallP90: percentile(withCall.map((t) => t.tFirstCallMs), 90),
|
||||
turnDoneP50: percentile(single.map((t) => t.tDoneMs), 50),
|
||||
turnDoneP90: percentile(single.map((t) => t.tDoneMs), 90),
|
||||
setupP50: percentile(single.map((t) => t.setupMs), 50),
|
||||
},
|
||||
tokens: {
|
||||
input: sum(mine.map((t) => t.usage?.inputTokens)),
|
||||
cached: sum(mine.map((t) => t.usage?.cachedTokens)),
|
||||
output: sum(mine.map((t) => t.usage?.outputTokens)),
|
||||
reasoning: sum(mine.map((t) => t.usage?.reasoningTokens)),
|
||||
inputPerTurnMean: round(mean(mine.map((t) => t.usage?.inputTokens))),
|
||||
inputPerRequestMean: round(mean(single.filter((t) => t.responses > 0 && t.usage?.inputTokens).map((t) => t.usage.inputTokens / t.responses))),
|
||||
},
|
||||
costUsd: round(sum(mine.map((t) => t.costUsd)), 4),
|
||||
costPerTurnUsd: mine.length ? round(sum(mine.map((t) => t.costUsd)) / mine.length, 5) : null,
|
||||
errors: mine.filter((t) => t.errors?.length).length,
|
||||
timeouts: mine.filter((t) => t.timeout).length,
|
||||
local: local.length
|
||||
? {
|
||||
prefillTokSMean: round(mean(local.map((t) => t.local.prefillTokS)), 1),
|
||||
prefillTokSP50: round(percentile(local.map((t) => t.local.prefillTokS), 50), 0),
|
||||
decodeTokSMean: round(mean(local.map((t) => t.local.decodeTokS)), 1),
|
||||
decodeTokSP50: round(percentile(local.map((t) => t.local.decodeTokS), 50), 1),
|
||||
decodeTokSP90: round(percentile(local.map((t) => t.local.decodeTokS), 90), 1),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
});
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
cloudSpendUsd: round(sum(turns.filter((t) => t.cloud).map((t) => t.costUsd)), 4),
|
||||
models,
|
||||
};
|
||||
}
|
||||
|
||||
const machineBrief = (machine) => {
|
||||
if (!machine) return '—';
|
||||
const load1 = String(machine.loadavg || '').split(' ')[0] || '?';
|
||||
const busy = (machine.busyProcesses || []).filter((p) => Number(p.match(/([\d.]+)%cpu/)?.[1]) >= 100)
|
||||
.map((p) => p.replace(/ Helper \(GPU\)/, ' GPU').replace(/Google Chrome for Testing/, 'Chrome-for-Testing'));
|
||||
return `load ${load1}${busy.length ? `; ${busy.join(', ')}` : ''}`;
|
||||
};
|
||||
const fmtCounts = (block) => (block?.n
|
||||
? `${block.counts.correct} / ${block.counts['honest-refusal']} / ${block.counts['hallucinated-capability']} / ${block.counts['wrong-tool']} (n=${block.n})`
|
||||
: '—');
|
||||
const fmtMs = (v) => (v === null || v === undefined ? '—' : v >= 10_000 ? `${(v / 1000).toFixed(1)}s` : `${Math.round(v)}ms`);
|
||||
const fmtPct = (v) => (v === null || v === undefined ? '—' : `${v}%`);
|
||||
|
||||
/** Render summary.md from a summarize() result plus free-form notes. */
|
||||
export function renderMarkdown(summary, { title = 'GEV voice-routing benchmark', notes = [], payload = [] } = {}) {
|
||||
const out = [`# ${title}`, '', `Generated ${summary.generatedAt}. Cloud spend so far: **$${summary.cloudSpendUsd?.toFixed(2)}**.`, ''];
|
||||
out.push('> **What this measures.** Tool CHOICE for text turns sent over each provider\'s API with production\'s instructions and tools, with stubbed (always-successful) tool results — the same method as `qa-voice-routing.mjs` layer 1. It is **not** end-to-end spoken accuracy (no speech recognition, no audio, no screenshot/scene context) and **not** spoken latency: every latency below is "send text → event", before any audio would be synthesized or played.', '');
|
||||
out.push('## Routing accuracy', '');
|
||||
out.push('| model | core tool-name | core full-call (tool + all arg checks) | typed args (n) | harness verdict (legacy) | first call right | false +ve | per-rep harness verdict | dialogue turn-2 full-call | dialogue turn-1 | coverage v1 C / HR / HC / WT | coverage v1 acceptable | coverage v2 C / HR / HC / WT | coverage v2 acceptable | capability tool / full-call |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of summary.models) {
|
||||
const c = m.coverage.counts;
|
||||
out.push(`| ${m.key} | ${fmtPct(m.core.toolAccuracyPct)} | ${fmtPct(m.core.fullCallAccuracyPct)} | ${fmtPct(m.core.typedArgsAccuracyPct)} (${m.core.typedArgsN}) | ${fmtPct(m.core.accuracyPct)} (n=${m.core.n}) | ${fmtPct(m.core.firstCallAccuracyPct)} | ${m.core.falsePositives}/${m.core.negativeTurns} | ${m.core.perRepAccuracyPct.map(fmtPct).join(', ')} | ${fmtPct(m.dialogue.turn2FullPct)} (n=${m.dialogue.n}) | ${fmtPct(m.dialogue.turn1FullPct)} | ${c.correct} / ${c['honest-refusal']} / ${c['hallucinated-capability']} / ${c['wrong-tool']} (n=${m.coverage.n}) | ${fmtPct(m.coverage.acceptablePct)} | ${fmtCounts(m.coverageV2)} | ${fmtPct(m.coverageV2.acceptablePct)} | ${m.capability.n ? `${fmtPct(m.capability.toolPct)} / ${fmtPct(m.capability.fullPct)} (n=${m.capability.n})` : '—'} |`);
|
||||
}
|
||||
out.push('', 'Core tool-name = right tool(s) called. Full-call = right tool(s) AND every argument check (the legacy `args` spot checks plus the benchmark\'s typed checks: exact layer ids, enum values, numeric thresholds, sort direction). Legacy harness verdict = exactly what `qa-voice-routing.mjs` would report. Coverage: C = correct, HR = honest refusal, HC = hallucinated capability, WT = wrong tool; "acceptable" = correct, or an honest refusal where today\'s tools cannot do it.', '');
|
||||
out.push('## Latency and cost', '');
|
||||
out.push('| model | p50 send → first token (text) | p50 send → tool call (text) | p90 send → tool call (text) | p50 send → turn done (text) | dialogue turn-2 p50 → tool call | session setup p50 | input tok/turn | $ total | $/turn |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of summary.models) {
|
||||
out.push(`| ${m.key} | ${fmtMs(m.latencyMs.firstTokenP50)} | ${fmtMs(m.latencyMs.toolCallP50)} | ${fmtMs(m.latencyMs.toolCallP90)} | ${fmtMs(m.latencyMs.turnDoneP50)} | ${fmtMs(m.dialogue.turn2ToolCallP50)} | ${fmtMs(m.latencyMs.setupP50)} | ${m.tokens.inputPerTurnMean ?? '—'} | ${m.costUsd?.toFixed(3)} | ${m.costPerTurnUsd ?? '—'} |`);
|
||||
}
|
||||
out.push('', 'Latencies are over single-turn phrases (core + coverage). "Turn done" includes the continuation response after the stubbed tool result. Cloud session setup (WebSocket + session config) is excluded from every latency and shown separately.', '');
|
||||
const locals = summary.models.filter((m) => m.local);
|
||||
if (locals.length) {
|
||||
out.push('## Local runtime (cold vs warm)', '');
|
||||
out.push('| model | load (wall) | COLD: first request after load → tool call | cold prefill tok/s (full prompt) | prompt tokens | WARM p50 → tool call | WARM p90 → tool call | warm per-turn prefill tok/s p50 | decode tok/s p50 / p90 | machine at start |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of locals) {
|
||||
const w = m.warmup || {};
|
||||
out.push(`| ${m.key} | ${fmtMs(w.loadWallMs ?? w.loadMs)} | ${fmtMs(w.coldToolCallMs ?? w.warmupTurnMs)}${w.coldToolCallMs == null ? ' (turn, no tool)' : ''} | ${w.prefillTokS ? Math.round(w.prefillTokS) : '—'} | ${w.promptTokens ?? '—'} | ${fmtMs(m.latencyMs.toolCallP50)} | ${fmtMs(m.latencyMs.toolCallP90)} | ${m.local.prefillTokSP50 ?? '—'} | ${m.local.decodeTokSP50 ?? '—'} / ${m.local.decodeTokSP90 ?? '—'} | ${machineBrief(m.machine)} |`);
|
||||
}
|
||||
out.push('', 'COLD = the first request after the model is loaded (pays the full ~11k-token prompt prefill). WARM = every later turn. "Warm per-turn prefill tok/s" is prompt tokens ÷ prompt-eval time as the runtime reports it: a value in the tens of thousands means the cached prompt prefix was reused and only the new user text was prefilled; a value near the cold rate means the WHOLE prompt was re-prefilled on every turn. A real voice session pays the cold cost once per model load. "Machine at start" = 1-minute load average and processes above 100% CPU when the run began (other GPU/CPU users on the machine inflate local timings).', '');
|
||||
}
|
||||
if (payload.length) {
|
||||
out.push('## Payload changes per provider', '');
|
||||
for (const p of payload) out.push(`- ${p}`);
|
||||
out.push('');
|
||||
}
|
||||
out.push('## Failures by model', '');
|
||||
for (const m of summary.models) {
|
||||
out.push(`### ${m.key}`, '');
|
||||
out.push(`Settings: \`${JSON.stringify(m.settings)}\``, '');
|
||||
if (m.errors || m.timeouts) out.push(`Errors: ${m.errors} turns, timeouts: ${m.timeouts}.`, '');
|
||||
if (m.core.failures.length) {
|
||||
out.push('Core misses (legacy verdict):', '');
|
||||
for (const f of m.core.failures) out.push(`- "${f.phrase}" (expect ${f.expect}) — failed ${f.fails}/${f.of}; called: ${f.called.join(' ; ')}`);
|
||||
out.push('');
|
||||
}
|
||||
if (m.capability.failures.length) {
|
||||
out.push('Capability misses (full call):', '');
|
||||
for (const f of m.capability.failures) out.push(`- "${f.id}" (expect ${f.expect}) — failed ${f.fails}/${f.of}; called: ${f.called.join(' ; ').slice(0, 400)}`);
|
||||
out.push('');
|
||||
}
|
||||
if (m.dialogue.failures.length) {
|
||||
out.push('Dialogue misses (turn 2):', '');
|
||||
for (const f of m.dialogue.failures) out.push(`- ${f.id} rep ${f.rep} (expect ${f.expect}) — t1: ${f.turn1} ⇒ t2: ${f.turn2}`);
|
||||
out.push('');
|
||||
}
|
||||
const badCov = m.coverage.items.filter((i) => i.verdicts.some((v) => v === 'hallucinated-capability' || v === 'wrong-tool' || (i.supported && v !== 'correct')));
|
||||
if (badCov.length) {
|
||||
out.push('Coverage misses:', '');
|
||||
for (const i of badCov) out.push(`- "${i.utterance}" (${i.supported ? 'supported' : 'unsupported'}) — ${i.verdicts.join(', ')}`);
|
||||
out.push('');
|
||||
}
|
||||
}
|
||||
if (notes.length) {
|
||||
out.push('## Notes', '');
|
||||
for (const n of notes) out.push(`- ${n}`);
|
||||
out.push('');
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Before → after table for two summarize() results (matched by model key).
|
||||
* Coverage "before" is the v1 rubric (what the old tools could do); "after"
|
||||
* shows both v1 (same yardstick) and v2 (the current tool surface).
|
||||
*/
|
||||
export function renderComparison(before, after, { beforeLabel = 'before', afterLabel = 'after' } = {}) {
|
||||
const out = ['## Before → after', '', `Before = ${beforeLabel}; after = ${afterLabel}. Core accuracy = tool + all argument checks (full call). Coverage OK % = correct, or an honest refusal where the tools of that build cannot do it. HC = hallucinated-capability count. p50 = send → tool call (text). Prompt tokens = mean input tokens per model request (cloud: as billed, before cache discounts; local: the cold full-prompt count).`, ''];
|
||||
out.push('| model | core full-call | coverage OK % (v1 → v1 / v2) | HC (v1 → v1 / v2) | dialogue turn-2 | capability full-call (after) | p50 → tool call | prompt tokens |');
|
||||
out.push('|---|---|---|---|---|---|---|---|');
|
||||
const hc = (block) => (block?.n ? block.counts['hallucinated-capability'] : null);
|
||||
const promptTok = (m) => (m?.warmup?.promptTokens ?? m?.tokens?.inputPerRequestMean ?? null);
|
||||
for (const a of after.models) {
|
||||
const b = before.models.find((x) => x.key === a.key);
|
||||
const arrow = (x, y, f) => `${f(x)} → ${f(y)}`;
|
||||
const pctF = (v) => (v === null || v === undefined ? '—' : `${v}%`);
|
||||
const msF = (v) => (v === null || v === undefined ? '—' : `${Math.round(v)}ms`);
|
||||
const num = (v) => (v === null || v === undefined ? '—' : String(v));
|
||||
out.push(`| ${a.key} | ${arrow(b?.core.fullCallAccuracyPct, a.core.fullCallAccuracyPct, pctF)} | ${pctF(b?.coverage.acceptablePct)} → ${pctF(a.coverage.acceptablePct)} / ${pctF(a.coverageV2.acceptablePct)} | ${num(hc(b?.coverage))} → ${num(hc(a.coverage))} / ${num(hc(a.coverageV2))} | ${arrow(b?.dialogue.turn2FullPct, a.dialogue.turn2FullPct, pctF)} | ${pctF(a.capability.fullPct)} | ${arrow(b?.latencyMs.toolCallP50, a.latencyMs.toolCallP50, msF)} | ${arrow(promptTok(b), promptTok(a), num)} |`);
|
||||
}
|
||||
out.push('');
|
||||
return out.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Stub tool results fed back to the model after each call. Pure.
|
||||
*
|
||||
* core suite: the neutral `{ ok:true }` stub qa-voice-routing has always
|
||||
* used, so continuations behave exactly as in that harness.
|
||||
* coverage suite: the same stub, except a call whose arguments violate the
|
||||
* tool schema gets `{ ok:false, error }` — what the app's runner returns
|
||||
* for an unknown layer id — so the model's follow-up words can be judged
|
||||
* for honesty.
|
||||
*/
|
||||
import { validateCall } from './grade.mjs';
|
||||
|
||||
export const STUB_OK = Object.freeze({ ok: true, note: 'qa-harness stub result' });
|
||||
|
||||
export function stubResult(suite, c, tools) {
|
||||
if (suite === 'coverage') {
|
||||
const errors = validateCall(tools, c);
|
||||
if (errors.length) return { ok: false, error: `Invalid arguments: ${errors.join('; ')}` };
|
||||
}
|
||||
return { ...STUB_OK };
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Pure converters from the production Realtime tool list
|
||||
* (`{ type:'function', name, description, parameters }`) to other providers'
|
||||
* tool formats. Descriptions and schemas are carried over unchanged except
|
||||
* where a target cannot represent them; every such change is listed below.
|
||||
*/
|
||||
|
||||
const PERMISSIVE_SCALAR = Object.freeze(['string', 'number', 'boolean']);
|
||||
|
||||
const clone = (value) => JSON.parse(JSON.stringify(value));
|
||||
|
||||
/**
|
||||
* Normalize a JSON schema for OpenAI-compatible local servers.
|
||||
*
|
||||
* Only one change: a property schema with no `type` (and no enum / anyOf /
|
||||
* oneOf / $ref / const that would imply one) gets a permissive scalar union.
|
||||
* `analyst_query.filters[].value` is `{}` in production — valid JSON Schema,
|
||||
* but chat templates that print `type` for every property (Gemma's among
|
||||
* them) crash on it.
|
||||
*/
|
||||
export function normalizeSchemaForChat(schema) {
|
||||
const out = clone(schema);
|
||||
const visit = (node, isRoot) => {
|
||||
if (!node || typeof node !== 'object' || Array.isArray(node)) return;
|
||||
const implied = ['type', 'enum', 'anyOf', 'oneOf', 'allOf', '$ref', 'const'].some((k) => k in node);
|
||||
if (!isRoot && !implied) node.type = [...PERMISSIVE_SCALAR];
|
||||
if (node.properties) for (const sub of Object.values(node.properties)) visit(sub, false);
|
||||
if (node.items) visit(node.items, false);
|
||||
for (const key of ['anyOf', 'oneOf', 'allOf']) if (Array.isArray(node[key])) node[key].forEach((s) => visit(s, false));
|
||||
};
|
||||
visit(out, true);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every schema location normalizeSchemaForChat changes, as
|
||||
* `{ tool, path, from, to }` — logged in summary.md so the per-provider
|
||||
* payload difference is explicit rather than implied.
|
||||
*/
|
||||
export function chatNormalizationDiff(realtimeTools) {
|
||||
const changes = [];
|
||||
const walk = (before, after, tool, where) => {
|
||||
if (!before || typeof before !== 'object' || Array.isArray(before)) return;
|
||||
if (JSON.stringify(before.type) !== JSON.stringify(after.type)) {
|
||||
changes.push({ tool, path: where, from: before.type === undefined ? '(untyped)' : before.type, to: after.type });
|
||||
}
|
||||
for (const [k, v] of Object.entries(before.properties || {})) walk(v, after.properties[k], tool, `${where}.${k}`);
|
||||
if (before.items) walk(before.items, after.items, tool, `${where}[]`);
|
||||
};
|
||||
for (const tool of realtimeTools) walk(tool.parameters, normalizeSchemaForChat(tool.parameters), tool.name, tool.name);
|
||||
return changes;
|
||||
}
|
||||
|
||||
/** Realtime tools → Chat Completions tools (`{type:'function', function:{...}}`). */
|
||||
export function toChatTools(realtimeTools, { normalize = true } = {}) {
|
||||
return realtimeTools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: normalize ? normalizeSchemaForChat(tool.parameters) : clone(tool.parameters),
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
// Keywords the Gemini OpenAPI-subset `Schema` does not accept.
|
||||
const GEMINI_DROP = new Set(['additionalProperties', '$schema', 'default', 'examples', '$id', 'const']);
|
||||
|
||||
/**
|
||||
* Convert a JSON schema to the Gemini OpenAPI-subset `Schema`.
|
||||
* Drops unsupported keywords (additionalProperties, $schema, default, ...);
|
||||
* an untyped property becomes `anyOf` string|number|boolean; a type array
|
||||
* becomes `anyOf` of single types.
|
||||
*/
|
||||
export function toGeminiSchema(schema) {
|
||||
if (!schema || typeof schema !== 'object' || Array.isArray(schema)) return schema;
|
||||
const out = {};
|
||||
for (const [key, value] of Object.entries(schema)) {
|
||||
if (GEMINI_DROP.has(key)) continue;
|
||||
if (key === 'properties') {
|
||||
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
|
||||
} else if (key === 'items') {
|
||||
out.items = toGeminiSchema(value);
|
||||
} else if (key === 'anyOf' || key === 'oneOf') {
|
||||
out.anyOf = value.map((s) => toGeminiSchema(s));
|
||||
} else {
|
||||
out[key] = clone(value);
|
||||
}
|
||||
}
|
||||
if (Array.isArray(out.type)) {
|
||||
const types = out.type;
|
||||
delete out.type;
|
||||
out.anyOf = types.map((t) => ({ type: t }));
|
||||
}
|
||||
const implied = ['type', 'enum', 'anyOf'].some((k) => k in out);
|
||||
if (!implied) out.anyOf = PERMISSIVE_SCALAR.map((t) => ({ type: t }));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Realtime tools → Gemini `functionDeclarations`.
|
||||
* mode 'jsonSchema' (default): pass the production schema verbatim through
|
||||
* `parametersJsonSchema` — full JSON Schema, closest to prompt parity.
|
||||
* mode 'openapi': convert to the OpenAPI-subset `parameters` field.
|
||||
* A tool with no properties gets no parameters field in openapi mode (Gemini
|
||||
* rejects an empty OBJECT schema there).
|
||||
*/
|
||||
export function toGeminiDeclarations(realtimeTools, { mode = 'jsonSchema' } = {}) {
|
||||
return realtimeTools.map((tool) => {
|
||||
const decl = { name: tool.name, description: tool.description };
|
||||
const params = tool.parameters;
|
||||
if (mode === 'jsonSchema') {
|
||||
decl.parametersJsonSchema = clone(params);
|
||||
} else if (params && Object.keys(params.properties || {}).length) {
|
||||
decl.parameters = toGeminiSchema(params);
|
||||
}
|
||||
return decl;
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user