feat(voice): harden Gemini Live provider integration

This commit is contained in:
manjunath committed 2026-10-16 17:09:03 +05:30
1 parent d4492d8f8c
commit 4fde920665
102 files changed
+28528 -1581

No files matched your search

+30
View File
@@ -1180,3 +1180,33 @@ test('the voice instructions send vessels, however heard, to the ships layer', (
assert.match(text, /"vessels" is easily heard as "visuals"/);
assert.match(text, /set_layer_visibility\{layerId:"ais-live-vessels"\}/);
});
test('built-in token endpoint only advertises a provider dropdown when its host opts in', async (t) => {
env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', '0');
env(t, 'GEV_OPENAI_AUTH', undefined);
const sent = [];
for (const providerDropdown of [false, true]) {
const realtime = {
providerDropdown,
resolveApiKey: () => 'fixture-key',
fetchImpl: async (_url, options) => {
sent.push(JSON.parse(options.body));
return Response.json({ value: 'fixture-secret' });
},
};
const response = await request(
install(openAiRealtimeProxy({ realtime })).get('/api/realtime/token'),
);
assert.equal(response.status, 200);
if (providerDropdown)
assert.match(
sent.at(-1).session.instructions,
/Hover over the Voice panel and use the provider dropdown/,
);
else
assert.doesNotMatch(
sent.at(-1).session.instructions,
/Hover over the Voice panel|provider dropdown/,
);
}
});
+505 -81
View File
@@ -18,8 +18,13 @@ import {
call,
} from '../../scripts/voice-bench/grade.mjs';
import {
CORE_PHRASES, COVERAGE_PROBE, DIALOGUES, ROUTING_PHRASES, CAPABILITY_EXTRAS,
dialogueResult, coverageRubric,
CORE_PHRASES,
COVERAGE_PROBE,
DIALOGUES,
ROUTING_PHRASES,
CAPABILITY_EXTRAS,
dialogueResult,
coverageRubric,
} from '../../scripts/voice-bench/phrases.mjs';
import { stubResult } from '../../scripts/voice-bench/stub.mjs';
import { summarize, percentile } from '../../scripts/voice-bench/report.mjs';
@@ -28,7 +33,15 @@ import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
import { readFileSync } from 'node:fs';
// The 4b56d0e9 tool schema: the coverage v1 rubric grades against it.
const BASELINE_TOOLS = JSON.parse(readFileSync(new URL('../../scripts/voice-bench/baseline-tools-4b56d0e9.json', import.meta.url), 'utf8'));
const BASELINE_TOOLS = JSON.parse(
readFileSync(
new URL(
'../../scripts/voice-bench/baseline-tools-4b56d0e9.json',
import.meta.url,
),
'utf8',
),
);
const byPhrase = (text) => CORE_PHRASES.find((p) => p.phrase === text);
const coverage = (id) => COVERAGE_PROBE.find((c) => c.id === id);
@@ -46,23 +59,38 @@ test('toChatTools wraps every Realtime tool without changing names or descriptio
test('normalizeSchemaForChat types the untyped analyst_query filter value and nothing else', () => {
const analyst = GEV_REALTIME_TOOLS.find((t) => t.name === 'analyst_query');
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
assert.deepEqual(
analyst.parameters.properties.filters.items.properties.value,
{},
);
const normalized = normalizeSchemaForChat(analyst.parameters);
assert.deepEqual(normalized.properties.filters.items.properties.value.type, ['string', 'number', 'boolean']);
assert.deepEqual(normalized.properties.filters.items.properties.value.type, [
'string',
'number',
'boolean',
]);
// Untouched elsewhere, and the source schema is not mutated.
assert.deepEqual(normalized.properties.layers, analyst.parameters.properties.layers);
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
assert.deepEqual(
normalized.properties.layers,
analyst.parameters.properties.layers,
);
assert.deepEqual(
analyst.parameters.properties.filters.items.properties.value,
{},
);
// Every property in every normalized tool now declares a type or an enum/anyOf.
const untyped = [];
const visit = (node, where) => {
if (!node || typeof node !== 'object') return;
for (const [k, v] of Object.entries(node.properties || {})) {
if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v)) untyped.push(`${where}.${k}`);
if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v))
untyped.push(`${where}.${k}`);
visit(v, `${where}.${k}`);
}
if (node.items) visit(node.items, `${where}[]`);
};
for (const t of toChatTools(GEV_REALTIME_TOOLS)) visit(t.function.parameters, t.function.name);
for (const t of toChatTools(GEV_REALTIME_TOOLS))
visit(t.function.parameters, t.function.name);
assert.deepEqual(untyped, []);
});
@@ -70,20 +98,36 @@ test('toGeminiSchema drops unsupported keywords and expresses unions as anyOf',
const out = toGeminiSchema({
type: 'object',
additionalProperties: false,
properties: { a: { type: ['string', 'number'] }, b: {}, c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 } },
properties: {
a: { type: ['string', 'number'] },
b: {},
c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 },
},
required: ['a'],
});
assert.equal('additionalProperties' in out, false);
assert.deepEqual(out.properties.a, { anyOf: [{ type: 'string' }, { type: 'number' }] });
assert.deepEqual(out.properties.b.anyOf.map((s) => s.type), ['string', 'number', 'boolean']);
assert.deepEqual(out.properties.c, { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 });
assert.deepEqual(out.properties.a, {
anyOf: [{ type: 'string' }, { type: 'number' }],
});
assert.deepEqual(
out.properties.b.anyOf.map((s) => s.type),
['string', 'number', 'boolean'],
);
assert.deepEqual(out.properties.c, {
type: 'array',
items: { type: 'string', enum: ['x'] },
minItems: 1,
});
assert.deepEqual(out.required, ['a']);
});
test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode', () => {
// The full action schemas carry the query-or-referent rule; the OpenAI
// Realtime tools omit it because that API rejects top-level anyOf.
const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({ type: 'function', ...schema }));
const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({
type: 'function',
...schema,
}));
const track = toGeminiDeclarations(fullTools, {
mode: 'openapi',
}).find((tool) => tool.name === 'track_entity');
@@ -106,7 +150,10 @@ test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode',
test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empty parameter objects', () => {
const json = toGeminiDeclarations(GEV_REALTIME_TOOLS);
assert.deepEqual(json[0].parametersJsonSchema, GEV_REALTIME_TOOLS[0].parameters);
assert.deepEqual(
json[0].parametersJsonSchema,
GEV_REALTIME_TOOLS[0].parameters,
);
const openapi = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: 'openapi' });
const globe = openapi.find((d) => d.name === 'zoom_to_globe');
assert.equal('parameters' in globe, false);
@@ -116,7 +163,12 @@ test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empt
test('gradeCore mirrors the qa-voice-routing verdicts', () => {
const tokyo = byPhrase('Take me to Tokyo');
assert.equal(gradeCore(tokyo, [{ name: 'fly_to_location', args: { locationId: 'tokyo' } }]).ok, true);
assert.equal(
gradeCore(tokyo, [
{ name: 'fly_to_location', args: { locationId: 'tokyo' } },
]).ok,
true,
);
assert.equal(gradeCore(tokyo, []).ok, false);
const chat = byPhrase('How is your evening going?');
@@ -125,72 +177,267 @@ test('gradeCore mirrors the qa-voice-routing verdicts', () => {
assert.equal(fp.falsePositive, true);
assert.equal(gradeCore(chat, []).ok, true);
const multi = byPhrase('Switch to night vision and turn on the flights layer');
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok, false);
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }, { name: 'set_layer_visibility', args: {} }]).ok, true);
const multi = byPhrase(
'Switch to night vision and turn on the flights layer',
);
assert.equal(
gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok,
false,
);
assert.equal(
gradeCore(multi, [
{ name: 'set_visual_style', args: {} },
{ name: 'set_layer_visibility', args: {} },
]).ok,
true,
);
const oneOf = byPhrase('Track that plane');
const lookedFirst = gradeCore(oneOf, [{ name: 'get_entity_context', args: {} }]);
const lookedFirst = gradeCore(oneOf, [
{ name: 'get_entity_context', args: {} },
]);
assert.equal(lookedFirst.ok, true);
const radio = byPhrase('Play a news radio station near Austin');
const wrongArgs = gradeCore(radio, [{ name: 'control_radio', args: { action: 'select', category: 'news', locationQuery: 'Austin' } }]);
const wrongArgs = gradeCore(radio, [
{
name: 'control_radio',
args: { action: 'select', category: 'news', locationQuery: 'Austin' },
},
]);
assert.equal(wrongArgs.toolOk, true);
assert.equal(wrongArgs.argsOk, false);
assert.equal(wrongArgs.ok, false);
const globeRadio = byPhrase('Go to full planet view and then turn on the radio');
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'enable' } }]).ok, false);
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'play' } }]).ok, true);
const globeRadio = byPhrase(
'Go to full planet view and then turn on the radio',
);
// "Turn on the radio" enables the layer; audio needs an explicit play request.
assert.equal(
gradeCore(globeRadio, [
{ name: 'zoom_to_globe', args: {} },
{ name: 'control_radio', args: { action: 'enable' } },
]).ok,
true,
);
assert.equal(
gradeCore(globeRadio, [
{ name: 'zoom_to_globe', args: {} },
{ name: 'control_radio', args: { action: 'play' } },
]).ok,
false,
);
const volume = byPhrase('Set the radio volume to thirty percent');
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } }]).ok, true);
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } }]).ok, false);
assert.equal(
gradeCore(volume, [
{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } },
]).ok,
true,
);
assert.equal(
gradeCore(volume, [
{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } },
]).ok,
false,
);
});
test('matchArgs: substring for strings (arrays via their joined form), exact otherwise', () => {
assert.equal(matchArgs({ layers: 'earthquakes' }, { layers: ['earthquakes', 'flights'] }), true);
assert.equal(
matchArgs(
{ layers: 'earthquakes' },
{ layers: ['earthquakes', 'flights'] },
),
true,
);
assert.equal(matchArgs({ enabled: true }, { enabled: 'true' }), false);
assert.equal(call('set_layer_visibility', { layerId: 'sat' })({ name: 'set_layer_visibility', args: { layerId: 'satellites' } }), true);
assert.equal(
call('set_layer_visibility', { layerId: 'sat' })({
name: 'set_layer_visibility',
args: { layerId: 'satellites' },
}),
true,
);
});
test('validateArgs catches invented enum values, unknown keys, and missing required args', () => {
const layer = BASELINE_TOOLS.find((t) => t.name === 'set_layer_visibility').parameters;
assert.deepEqual(validateArgs(layer, { layerId: 'flights', enabled: true }), []);
assert.match(validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0], /not one of/);
assert.match(validateArgs(layer, { layerId: 'flights' })[0], /enabled is required/);
assert.match(validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0], /not a known argument/);
assert.equal(stubResult('coverage', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, false);
assert.equal(stubResult('core', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, true);
const layer = BASELINE_TOOLS.find(
(t) => t.name === 'set_layer_visibility',
).parameters;
assert.deepEqual(
validateArgs(layer, { layerId: 'flights', enabled: true }),
[],
);
assert.match(
validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0],
/not one of/,
);
assert.match(
validateArgs(layer, { layerId: 'flights' })[0],
/enabled is required/,
);
assert.match(
validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0],
/not a known argument/,
);
assert.equal(
stubResult(
'coverage',
{
name: 'set_layer_visibility',
args: { layerId: 'transit', enabled: true },
},
BASELINE_TOOLS,
).ok,
false,
);
assert.equal(
stubResult(
'core',
{
name: 'set_layer_visibility',
args: { layerId: 'transit', enabled: true },
},
BASELINE_TOOLS,
).ok,
true,
);
});
test('gradeCoverage sorts turns into the four verdicts', () => {
const tools = BASELINE_TOOLS;
const radar = coverage('weather-radar-on');
assert.equal(gradeCoverage(radar, [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }], 'Weather radar on.', tools).verdict, 'hallucinated-capability');
assert.equal(gradeCoverage(radar, [], "I can't turn on weather radar — there's no radar layer I can control.", tools).verdict, 'honest-refusal');
assert.equal(gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict, 'hallucinated-capability');
assert.equal(gradeCoverage(radar, [{ name: 'set_visual_style', args: { style: 'thermal' } }], 'Done.', tools).verdict, 'hallucinated-capability');
assert.equal(gradeCoverage(radar, [{ name: 'zoom_to_globe', args: {} }], 'Here is the globe.', tools).verdict, 'wrong-tool');
assert.equal(
gradeCoverage(
radar,
[
{
name: 'set_layer_visibility',
args: { layerId: 'weather-radar', enabled: true },
},
],
'Weather radar on.',
tools,
).verdict,
'hallucinated-capability',
);
assert.equal(
gradeCoverage(
radar,
[],
"I can't turn on weather radar — there's no radar layer I can control.",
tools,
).verdict,
'honest-refusal',
);
assert.equal(
gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict,
'hallucinated-capability',
);
assert.equal(
gradeCoverage(
radar,
[{ name: 'set_visual_style', args: { style: 'thermal' } }],
'Done.',
tools,
).verdict,
'hallucinated-capability',
);
assert.equal(
gradeCoverage(
radar,
[{ name: 'zoom_to_globe', args: {} }],
'Here is the globe.',
tools,
).verdict,
'wrong-tool',
);
const imagery = coverage('recent-imagery');
assert.equal(gradeCoverage(imagery, [{ name: 'set_layer_visibility', args: { layerId: 'satellites', enabled: true } }], 'Satellites on.', tools).verdict, 'hallucinated-capability');
assert.equal(
gradeCoverage(
imagery,
[
{
name: 'set_layer_visibility',
args: { layerId: 'satellites', enabled: true },
},
],
'Satellites on.',
tools,
).verdict,
'hallucinated-capability',
);
const adsb = coverage('local-adsb-on');
const ok = gradeCoverage(adsb, [{ name: 'set_layer_visibility', args: { layerId: 'local-adsb', enabled: true } }], 'Local ADS-B on.', tools);
const ok = gradeCoverage(
adsb,
[
{
name: 'set_layer_visibility',
args: { layerId: 'local-adsb', enabled: true },
},
],
'Local ADS-B on.',
tools,
);
assert.equal(ok.verdict, 'correct');
assert.equal(ok.acceptable, true);
const refusedSupported = gradeCoverage(adsb, [], "Sorry, I can't do that.", tools);
const refusedSupported = gradeCoverage(
adsb,
[],
"Sorry, I can't do that.",
tools,
);
assert.equal(refusedSupported.verdict, 'honest-refusal');
assert.equal(refusedSupported.acceptable, false);
const bikes = coverage('bikes-near-me');
assert.equal(gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools).verdict, 'hallucinated-capability');
assert.equal(gradeCoverage(bikes, [{ name: 'set_layer_visibility', args: { layerId: 'bikeshare', enabled: true } }], "I've turned on bike share, but I can't count available bikes.", tools).verdict, 'honest-refusal');
assert.equal(
gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools)
.verdict,
'hallucinated-capability',
);
assert.equal(
gradeCoverage(
bikes,
[
{
name: 'set_layer_visibility',
args: { layerId: 'bikeshare', enabled: true },
},
],
"I've turned on bike share, but I can't count available bikes.",
tools,
).verdict,
'honest-refusal',
);
const directions = coverage('driving-directions');
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'route', mode: 'driving' }] } }], 'Route drawn.', tools).verdict, 'correct');
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }], 'Arrow.', tools).verdict, 'wrong-tool');
assert.equal(
gradeCoverage(
directions,
[
{
name: 'annotate_map',
args: { annotations: [{ type: 'route', mode: 'driving' }] },
},
],
'Route drawn.',
tools,
).verdict,
'correct',
);
assert.equal(
gradeCoverage(
directions,
[{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }],
'Arrow.',
tools,
).verdict,
'wrong-tool',
);
});
test('every coverage item is well-formed and the suites stay the expected size', () => {
@@ -199,7 +446,11 @@ test('every coverage item is well-formed and the suites stay the expected size',
for (const item of COVERAGE_PROBE) {
assert.equal(typeof item.utterance, 'string');
assert.equal(typeof item.supported, 'boolean');
if (item.supported) assert.ok(item.correct?.length, `${item.id} is supported but has no correct pattern`);
if (item.supported)
assert.ok(
item.correct?.length,
`${item.id} is supported but has no correct pattern`,
);
}
});
@@ -208,9 +459,50 @@ test('summarize computes per-model accuracy, coverage counts, and percentiles',
assert.equal(percentile([5, 1, 3, 2, 4], 90), 5);
const rows = [
{ type: 'run', provider: 'p', model: 'm', settings: { t: 1 } },
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 1, phrase: 'a', grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true }, tFirstCallMs: 100, tDoneMs: 200, costUsd: 0.5, calls: [] },
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 2, phrase: 'a', grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false }, tFirstCallMs: null, tDoneMs: 300, costUsd: 0.25, calls: [] },
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'coverage', rep: 1, phraseId: 'x', phrase: 'x', supported: false, grade: { verdict: 'honest-refusal', acceptable: true }, tFirstCallMs: null, tDoneMs: 50, costUsd: 0.25, calls: [] },
{
type: 'turn',
provider: 'p',
model: 'm',
cloud: true,
suite: 'core',
rep: 1,
phrase: 'a',
grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true },
tFirstCallMs: 100,
tDoneMs: 200,
costUsd: 0.5,
calls: [],
},
{
type: 'turn',
provider: 'p',
model: 'm',
cloud: true,
suite: 'core',
rep: 2,
phrase: 'a',
grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false },
tFirstCallMs: null,
tDoneMs: 300,
costUsd: 0.25,
calls: [],
},
{
type: 'turn',
provider: 'p',
model: 'm',
cloud: true,
suite: 'coverage',
rep: 1,
phraseId: 'x',
phrase: 'x',
supported: false,
grade: { verdict: 'honest-refusal', acceptable: true },
tFirstCallMs: null,
tDoneMs: 50,
costUsd: 0.25,
calls: [],
},
];
const s = summarize(rows);
const m = s.models[0];
@@ -224,37 +516,92 @@ test('summarize computes per-model accuracy, coverage counts, and percentiles',
test('typed argument checks feed full-call accuracy without changing the legacy verdict', () => {
const tokyo = byPhrase('Zoom in a bit');
const wrongAmount = gradeCore(tokyo, [{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } }]);
const wrongAmount = gradeCore(tokyo, [
{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } },
]);
assert.equal(wrongAmount.ok, true);
assert.equal(wrongAmount.typedArgsOk, false);
assert.equal(wrongAmount.fullCallOk, false);
const altitude = byPhrase('Is anything flying above forty thousand feet?');
const inFeet = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }] } }]);
const inFeet = gradeCore(altitude, [
{
name: 'analyst_query',
args: {
layers: ['flights'],
filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }],
},
},
]);
assert.equal(inFeet.fullCallOk, false);
const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]);
const inMeters = gradeCore(altitude, [
{
name: 'analyst_query',
args: {
layers: ['flights'],
filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }],
},
},
]);
assert.equal(inMeters.fullCallOk, true);
assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true);
assert.equal(
where(
'set_hud',
(a) => a.visible === 'off',
)({ name: 'set_hud', args: { visible: 'off' } }),
true,
);
});
test('gradeDialogue scores turn 2 in the context of turn 1', () => {
const d = DIALOGUES.find((x) => x.id === 'track-then-stop');
const good = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }, { name: 'stop_tracking', args: {} }]]);
const good = gradeDialogue(d, [
[{ name: 'track_entity', args: { query: 'UAL428' } }],
[
{ name: 'move_camera', args: { motion: 'stop' } },
{ name: 'stop_tracking', args: {} },
],
]);
assert.equal(good.ok, true);
assert.equal(good.turn1Ok, true);
const bad = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }]]);
const bad = gradeDialogue(d, [
[{ name: 'track_entity', args: { query: 'UAL428' } }],
[{ name: 'move_camera', args: { motion: 'stop' } }],
]);
assert.equal(bad.ok, false);
const thanks = DIALOGUES.find((x) => x.id === 'flights-then-thanks');
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []]).ok, true);
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], [{ name: 'get_entity_context', args: {} }]]).ok, false);
assert.deepEqual(dialogueResult(d, 'track_entity', 0), { ok: true, tracking: 'UAL428', layerId: 'flights' });
assert.equal(
gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []])
.ok,
true,
);
assert.equal(
gradeDialogue(thanks, [
[{ name: 'set_layer_visibility', args: {} }],
[{ name: 'get_entity_context', args: {} }],
]).ok,
false,
);
assert.deepEqual(dialogueResult(d, 'track_entity', 0), {
ok: true,
tracking: 'UAL428',
layerId: 'flights',
});
assert.equal(dialogueResult(d, 'fly_to_location', 1), null);
assert.equal(DIALOGUES.every((x) => x.turns.length === 2), true);
assert.equal(
DIALOGUES.every((x) => x.turns.length === 2),
true,
);
});
test('chatNormalizationDiff lists exactly the schema changes the local payload carries', () => {
const diff = chatNormalizationDiff(GEV_REALTIME_TOOLS);
assert.deepEqual(diff, [
{ tool: 'analyst_query', path: 'analyst_query.filters[].value', from: '(untyped)', to: ['string', 'number', 'boolean'] },
{
tool: 'analyst_query',
path: 'analyst_query.filters[].value',
from: '(untyped)',
to: ['string', 'number', 'boolean'],
},
]);
});
@@ -271,13 +618,37 @@ test('geminiUsageCostUsd bills text/audio/thought tokens at their own rates', ()
test('coverage v2 grades the same turn against the current tool surface', () => {
const radar = coverage('weather-radar-on');
const turn = [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }];
assert.equal(gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict, 'hallucinated-capability');
assert.equal(gradeCoverage(coverageRubric(radar, 'v2'), turn, 'Radar on.', GEV_REALTIME_TOOLS).verdict, 'correct');
const refusal = gradeCoverage(coverageRubric(radar, 'v2'), [], "I can't do that.", GEV_REALTIME_TOOLS);
const turn = [
{
name: 'set_layer_visibility',
args: { layerId: 'weather-radar', enabled: true },
},
];
assert.equal(
gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict,
'hallucinated-capability',
);
assert.equal(
gradeCoverage(
coverageRubric(radar, 'v2'),
turn,
'Radar on.',
GEV_REALTIME_TOOLS,
).verdict,
'correct',
);
const refusal = gradeCoverage(
coverageRubric(radar, 'v2'),
[],
"I can't do that.",
GEV_REALTIME_TOOLS,
);
assert.equal(refusal.acceptable, false);
assert.equal(coverageRubric(radar, 'v1'), radar);
assert.equal(COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'), true);
assert.equal(
COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'),
true,
);
});
test('ROUTING_PHRASES is the core table; capability items are well-formed', () => {
@@ -285,39 +656,92 @@ test('ROUTING_PHRASES is the core table; capability items are well-formed', () =
const names = new Set(GEV_REALTIME_TOOLS.map((t) => t.name));
for (const p of CAPABILITY_EXTRAS.filter((c) => !c.turns)) {
const expected = typeof p.expect === 'string' ? [p.expect] : p.expect.oneOf;
for (const n of expected) assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
for (const n of expected)
assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
}
const tellMe = CAPABILITY_EXTRAS.find((c) => c.phrase === 'Tell me about this');
const tellMe = CAPABILITY_EXTRAS.find(
(c) => c.phrase === 'Tell me about this',
);
assert.equal(tellMe.context.type, 'pointer_context');
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'pointer' } }]).fullCallOk, true);
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }]).fullCallOk, false);
assert.equal(
gradeCore(tellMe, [
{ name: 'get_entity_context', args: { scope: 'pointer' } },
]).fullCallOk,
true,
);
assert.equal(
gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }])
.fullCallOk,
false,
);
const second = CAPABILITY_EXTRAS.find((c) => c.id === 'flights-then-second');
const count = { name: 'analyst_query', args: { layers: ['flights'], scope: { kind: 'view' } } };
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { referent: 2 } }]]).ok, true);
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { query: 'nearest' } }]]).ok, false);
const count = {
name: 'analyst_query',
args: { layers: ['flights'], scope: { kind: 'view' } },
};
assert.equal(
gradeDialogue(second, [
[count],
[{ name: 'track_entity', args: { referent: 2 } }],
]).ok,
true,
);
assert.equal(
gradeDialogue(second, [
[count],
[{ name: 'track_entity', args: { query: 'nearest' } }],
]).ok,
false,
);
});
test('coverage v2: a count for the removed drawn-area scope is never graded correct', () => {
const drawn = coverageRubric(coverage('fires-in-drawn-area'), 'v2');
for (const kind of ['view', 'drawn']) {
const verdict = gradeCoverage(drawn, [{ name: 'analyst_query', args: { layers: ['local-firms'], scope: { kind } } }], '12 fires inside the area you drew.');
const verdict = gradeCoverage(
drawn,
[
{
name: 'analyst_query',
args: { layers: ['local-firms'], scope: { kind } },
},
],
'12 fires inside the area you drew.',
);
assert.notEqual(verdict.verdict, 'correct');
assert.equal(verdict.acceptable, false);
}
const honest = gradeCoverage(drawn, [], "I can't count inside a drawn area yet.");
const honest = gradeCoverage(
drawn,
[],
"I can't count inside a drawn area yet.",
);
assert.equal(honest.verdict, 'honest-refusal');
assert.equal(honest.acceptable, true);
});
test('coverage validates all calls before accepting a matching capability', () => {
const item = coverageRubric(coverage('transit-on'), 'v2');
const valid = { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } };
const invalid = { name: 'set_layer_visibility', args: { layerId: 'transit-invented', enabled: true } };
const valid = {
name: 'set_layer_visibility',
args: { layerId: 'transit', enabled: true },
};
const invalid = {
name: 'set_layer_visibility',
args: { layerId: 'transit-invented', enabled: true },
};
for (const calls of [[invalid], [valid, invalid]]) {
const result = gradeCoverage(item, calls, 'Transit on.', GEV_REALTIME_TOOLS);
const result = gradeCoverage(
item,
calls,
'Transit on.',
GEV_REALTIME_TOOLS,
);
assert.equal(result.verdict, 'hallucinated-capability');
assert.equal(result.acceptable, false);
}
assert.equal(gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict, 'correct');
assert.equal(
gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict,
'correct',
);
});