Merge remote-tracking branch 'origin/main' into german/street-level-slim

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/tooling/localServices.test.mjs

@@ -21,6 +21,9 @@ import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai';
 import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup';
 import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
 import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
+import { createDebugLogHandler } from '../../server/providers/openai/debug-log.js';
+import { attachVoiceResult } from '../voice/speech.js';
+import { sanitizeDebugValue } from '../voice/realtimeDiagnostics.js';
 import { standaloneVoiceTools } from '../../server/standalone/voiceTools.js';
 
 function install(plugin, preview = false) {
@@ -196,6 +199,30 @@ test('Realtime handler preserves tools and default instructions, isolates suppli
   assert.equal(sent[0].session.instructions, sent[2].session.instructions);
 });
 
+test('caption transcription is on by default, reported to the client meter, and can be turned off', async (t) => {
+  env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
+  env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
+  const sent = [];
+  t.mock.method(globalThis, 'fetch', async (url, options) => {
+    sent.push(JSON.parse(options.body));
+    return Response.json({ value: 'fixture-ephemeral' });
+  });
+  for (const [setting, expected] of [
+    [undefined, 'gpt-4o-mini-transcribe'],
+    ['off', 'off'],
+  ]) {
+    env(t, 'OPENAI_REALTIME_TRANSCRIBE_MODEL', setting);
+    const response = await request(
+      install(openAiRealtimeProxy()).get('/api/realtime/token'),
+    );
+    assert.equal(response.headers['x-gev-voice-transcribe-model'], expected);
+    assert.equal(
+      sent.at(-1).session.audio.input.transcription?.model ?? 'off',
+      expected,
+    );
+  }
+});
+
 test('Realtime sessions carry supplied tools, and the standalone voice adds the catalog queries', async (t) => {
   env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
   env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
@@ -374,6 +401,269 @@ test('OpenAI routes answer generically when the upstream or the request fails',
   assert.equal(token.body.includes('fixture-upstream-secret'), false);
 });
 
+test('the debug-log sink omits what people said and tool arguments unless content logging is enabled', async (t) => {
+  const nestObject = (value, levels) => {
+    let nested = value;
+    for (let index = 0; index < levels; index += 1) nested = { nested };
+    return nested;
+  };
+  const nestArray = (value, levels) => {
+    let nested = value;
+    for (let index = 0; index < levels; index += 1) nested = [nested];
+    return nested;
+  };
+  const records = [
+    {
+      timestamp: '2026-10-05T00:00:00.000Z',
+      sessionId: 'gev-session-1',
+      event: 'tool.call',
+      status: 'active',
+      payload: {
+        name: 'analyst_query',
+        callId: 'call-1',
+        arguments: { location: '12 Elm Street', note: 'meet Alice there' },
+      },
+    },
+    {
+      timestamp: '2026-10-05T00:00:01.000Z',
+      sessionId: 'gev-session-1',
+      event: 'server.event',
+      status: 'active',
+      payload: {
+        type: 'response.function_call_arguments.done',
+        eventId: 'event-1',
+        responseId: 'response-1',
+        payload: {
+          type: 'response.function_call_arguments.done',
+          event_id: 'event-1',
+          response_id: 'response-1',
+          item_id: 'item-1',
+          output_index: 0,
+          call_id: 'call-1',
+          name: 'analyst_query',
+          arguments: '{"query":"Tokyo Station"}',
+        },
+      },
+    },
+    {
+      event: 'server.event',
+      status: 'active',
+      payload: {
+        type: 'response.audio_transcript.done',
+        transcript: 'my home address is 12 Elm Street',
+        text: 'call my sister',
+        delta: 'Flying to Tokyo.',
+        usage: { input_tokens: 40 },
+      },
+    },
+    {
+      event: 'voice.cost',
+      status: 'active',
+      payload: { delta: 17, total: 41 },
+    },
+    {
+      event: 'tool.call',
+      status: 'active',
+      payload: { name: 'adjust_camera_zoom', callId: 'call-2', arguments: 2 },
+    },
+    {
+      event: 'deep.object',
+      status: 'active',
+      payload: nestObject(
+        {
+          transcript: 'deep-object-transcript-secret',
+          arguments: { query: 'deep-object-argument-secret' },
+        },
+        14,
+      ),
+    },
+    {
+      event: 'deep.array',
+      status: 'active',
+      payload: nestArray(
+        {
+          transcript: 'deep-array-transcript-secret',
+          arguments: { query: 'deep-array-argument-secret' },
+        },
+        14,
+      ),
+    },
+  ];
+  for (const includeContent of [false, true]) {
+    const sourceRoot = root(t);
+    const handler = createDebugLogHandler({ sourceRoot, includeContent });
+    for (const record of records) {
+      const response = await request(handler, {
+        method: 'POST',
+        body: JSON.stringify(record),
+      });
+      assert.equal(response.status, 204);
+    }
+    const logged = readFileSync(
+      path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'),
+      'utf8',
+    );
+    assert.equal(logged.includes('Elm Street'), includeContent);
+    assert.equal(logged.includes('call my sister'), includeContent);
+    assert.equal(logged.includes('meet Alice there'), includeContent);
+    assert.equal(logged.includes('Tokyo Station'), includeContent);
+    assert.equal(logged.includes('Flying to Tokyo'), includeContent);
+    assert.equal(
+      logged.includes('deep-object-transcript-secret'),
+      includeContent,
+    );
+    assert.equal(
+      logged.includes('deep-object-argument-secret'),
+      includeContent,
+    );
+    assert.equal(
+      logged.includes('deep-array-transcript-secret'),
+      includeContent,
+    );
+    assert.equal(logged.includes('deep-array-argument-secret'), includeContent);
+    const [
+      toolCall,
+      serverEvent,
+      transcriptEvent,
+      numericDelta,
+      numericArguments,
+      deepObject,
+      deepArray,
+    ] = logged
+      .split('\n')
+      .filter(Boolean)
+      .map((line) => JSON.parse(line));
+    assert.equal(toolCall.timestamp, records[0].timestamp);
+    assert.equal(toolCall.sessionId, 'gev-session-1');
+    assert.equal(toolCall.event, 'tool.call');
+    assert.equal(toolCall.status, 'active');
+    assert.equal(toolCall.payload.name, 'analyst_query');
+    assert.equal(toolCall.payload.callId, 'call-1');
+    assert.equal(serverEvent.payload.eventId, 'event-1');
+    assert.equal(serverEvent.payload.responseId, 'response-1');
+    assert.equal(serverEvent.payload.payload.event_id, 'event-1');
+    assert.equal(serverEvent.payload.payload.response_id, 'response-1');
+    assert.equal(serverEvent.payload.payload.item_id, 'item-1');
+    assert.equal(serverEvent.payload.payload.output_index, 0);
+    assert.equal(serverEvent.payload.payload.call_id, 'call-1');
+    assert.equal(serverEvent.payload.payload.name, 'analyst_query');
+    assert.equal(transcriptEvent.payload.usage.input_tokens, 40);
+    assert.equal(numericDelta.payload.delta, 17);
+    assert.equal(numericDelta.payload.total, 41);
+    if (!includeContent) {
+      assert.equal(toolCall.payload.arguments, '[omitted]');
+      assert.match(
+        serverEvent.payload.payload.arguments,
+        /^\[omitted \d+ chars\]$/,
+      );
+      assert.equal(transcriptEvent.payload.transcript, '[omitted 32 chars]');
+      assert.equal(transcriptEvent.payload.text, '[omitted 14 chars]');
+      assert.equal(transcriptEvent.payload.delta, '[omitted 16 chars]');
+      assert.equal(numericArguments.payload.arguments, '[omitted]');
+      assert.equal(
+        JSON.stringify(deepObject.payload).includes('[omitted: max depth]'),
+        true,
+      );
+      assert.equal(
+        JSON.stringify(deepArray.payload).includes('[omitted: max depth]'),
+        true,
+      );
+    } else {
+      assert.deepEqual(
+        toolCall.payload.arguments,
+        records[0].payload.arguments,
+      );
+      assert.equal(
+        serverEvent.payload.payload.arguments,
+        records[1].payload.payload.arguments,
+      );
+      assert.equal(numericArguments.payload.arguments, 2);
+    }
+  }
+});
+
+test('debug logs omit structured and serialized tool results unless content logging is enabled', async (t) => {
+  const secret = 'private-address-fixture';
+  const result = attachVoiceResult('fly_to_location', {
+    ok: true,
+    query: secret,
+    label: secret,
+    arrived: true,
+    // New tool-specific fields must not need their own redaction rule.
+    futureField: { nested: [secret] },
+  });
+  const records = [
+    {
+      event: 'tool.result',
+      payload: { name: 'fly_to_location', callId: 'call-1', result },
+    },
+    {
+      event: 'client.function_call_output',
+      payload: {
+        message: {
+          type: 'conversation.item.create',
+          item: {
+            type: 'function_call_output',
+            call_id: 'call-1',
+            output: JSON.stringify(result),
+          },
+        },
+      },
+    },
+    {
+      event: 'server.event',
+      payload: {
+        response: {
+          id: 'response-1',
+          output: [
+            {
+              type: 'message',
+              content: [{ type: 'output_text', text: secret }],
+            },
+          ],
+          usage: { input_tokens: 40, output_tokens: 10 },
+        },
+      },
+    },
+  ].map((record) => sanitizeDebugValue(record));
+  for (const includeContent of [false, true]) {
+    const sourceRoot = root(t);
+    const handler = createDebugLogHandler({ sourceRoot, includeContent });
+    for (const record of records) {
+      const response = await request(handler, {
+        method: 'POST',
+        body: JSON.stringify(record),
+      });
+      assert.equal(response.status, 204);
+    }
+    const logged = readFileSync(
+      path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'),
+      'utf8',
+    );
+    assert.equal(logged.includes(secret), includeContent);
+    const [tool, client, server] = logged.trim().split('\n').map(JSON.parse);
+    assert.equal(tool.payload.name, 'fly_to_location');
+    assert.equal(tool.payload.callId, 'call-1');
+    assert.equal(client.payload.message.item.call_id, 'call-1');
+    assert.equal(server.payload.response.id, 'response-1');
+    assert.equal(server.payload.response.output[0].type, 'message');
+    assert.deepEqual(
+      server.payload.response.usage,
+      records[2].payload.response.usage,
+    );
+    if (includeContent) {
+      assert.deepEqual(tool.payload.result, result);
+      assert.equal(client.payload.message.item.output, JSON.stringify(result));
+    } else {
+      assert.equal(tool.payload.result, '[omitted]');
+      assert.match(
+        client.payload.message.item.output,
+        /^\[omitted \d+ chars\]$/,
+      );
+    }
+  }
+});
+
 test('the debug-log sink stays bounded, rate limited, and quiet about failures', async (t) => {
   const sourceRoot = root(t);
   const handler = install(openAiRealtimeProxy({ sourceRoot })).get(

src/tooling/voiceBench.test.mjs

@@ -0,0 +1,323 @@
+import { GEV_ACTION_SCHEMAS } from '../voice/actionSchemas.js';
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  normalizeSchemaForChat,
+  toChatTools,
+  toGeminiDeclarations,
+  toGeminiSchema,
+  chatNormalizationDiff,
+} from '../../scripts/voice-bench/toolFormats.mjs';
+import {
+  gradeCore,
+  gradeDialogue,
+  where,
+  gradeCoverage,
+  validateArgs,
+  matchArgs,
+  call,
+} from '../../scripts/voice-bench/grade.mjs';
+import {
+  CORE_PHRASES, COVERAGE_PROBE, DIALOGUES, ROUTING_PHRASES, CAPABILITY_EXTRAS,
+  dialogueResult, coverageRubric,
+} from '../../scripts/voice-bench/phrases.mjs';
+import { stubResult } from '../../scripts/voice-bench/stub.mjs';
+import { summarize, percentile } from '../../scripts/voice-bench/report.mjs';
+import { geminiUsageCostUsd } from '../../scripts/voice-bench/pricing.mjs';
+import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
+import { readFileSync } from 'node:fs';
+
+// The 4b56d0e9 tool schema: the coverage v1 rubric grades against it.
+const BASELINE_TOOLS = JSON.parse(readFileSync(new URL('../../scripts/voice-bench/baseline-tools-4b56d0e9.json', import.meta.url), 'utf8'));
+
+const byPhrase = (text) => CORE_PHRASES.find((p) => p.phrase === text);
+const coverage = (id) => COVERAGE_PROBE.find((c) => c.id === id);
+
+test('toChatTools wraps every Realtime tool without changing names or descriptions', () => {
+  const chat = toChatTools(GEV_REALTIME_TOOLS);
+  assert.equal(chat.length, GEV_REALTIME_TOOLS.length);
+  chat.forEach((t, i) => {
+    assert.equal(t.type, 'function');
+    assert.equal(t.function.name, GEV_REALTIME_TOOLS[i].name);
+    assert.equal(t.function.description, GEV_REALTIME_TOOLS[i].description);
+    assert.equal(t.function.parameters.type, 'object');
+  });
+});
+
+test('normalizeSchemaForChat types the untyped analyst_query filter value and nothing else', () => {
+  const analyst = GEV_REALTIME_TOOLS.find((t) => t.name === 'analyst_query');
+  assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
+  const normalized = normalizeSchemaForChat(analyst.parameters);
+  assert.deepEqual(normalized.properties.filters.items.properties.value.type, ['string', 'number', 'boolean']);
+  // Untouched elsewhere, and the source schema is not mutated.
+  assert.deepEqual(normalized.properties.layers, analyst.parameters.properties.layers);
+  assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
+  // Every property in every normalized tool now declares a type or an enum/anyOf.
+  const untyped = [];
+  const visit = (node, where) => {
+    if (!node || typeof node !== 'object') return;
+    for (const [k, v] of Object.entries(node.properties || {})) {
+      if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v)) untyped.push(`${where}.${k}`);
+      visit(v, `${where}.${k}`);
+    }
+    if (node.items) visit(node.items, `${where}[]`);
+  };
+  for (const t of toChatTools(GEV_REALTIME_TOOLS)) visit(t.function.parameters, t.function.name);
+  assert.deepEqual(untyped, []);
+});
+
+test('toGeminiSchema drops unsupported keywords and expresses unions as anyOf', () => {
+  const out = toGeminiSchema({
+    type: 'object',
+    additionalProperties: false,
+    properties: { a: { type: ['string', 'number'] }, b: {}, c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 } },
+    required: ['a'],
+  });
+  assert.equal('additionalProperties' in out, false);
+  assert.deepEqual(out.properties.a, { anyOf: [{ type: 'string' }, { type: 'number' }] });
+  assert.deepEqual(out.properties.b.anyOf.map((s) => s.type), ['string', 'number', 'boolean']);
+  assert.deepEqual(out.properties.c, { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 });
+  assert.deepEqual(out.required, ['a']);
+});
+
+test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode', () => {
+  // The full action schemas carry the query-or-referent rule; the OpenAI
+  // Realtime tools omit it because that API rejects top-level anyOf.
+  const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({ type: 'function', ...schema }));
+  const track = toGeminiDeclarations(fullTools, {
+    mode: 'openapi',
+  }).find((tool) => tool.name === 'track_entity');
+  assert.deepEqual(
+    track.parameters.anyOf.map((branch) => ({
+      type: branch.type,
+      required: branch.required,
+    })),
+    [
+      { type: 'object', required: ['query'] },
+      { type: 'object', required: ['referent'] },
+    ],
+  );
+  assert.equal(track.parameters.properties.query.pattern, '\\S');
+  assert.deepEqual(
+    track.parameters.properties.referent.enum,
+    [-1, 1, 2, 3, 4, 5],
+  );
+});
+
+test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empty parameter objects', () => {
+  const json = toGeminiDeclarations(GEV_REALTIME_TOOLS);
+  assert.deepEqual(json[0].parametersJsonSchema, GEV_REALTIME_TOOLS[0].parameters);
+  const openapi = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: 'openapi' });
+  const globe = openapi.find((d) => d.name === 'zoom_to_globe');
+  assert.equal('parameters' in globe, false);
+  const serialized = JSON.stringify(openapi);
+  assert.equal(serialized.includes('additionalProperties'), false);
+});
+
+test('gradeCore mirrors the qa-voice-routing verdicts', () => {
+  const tokyo = byPhrase('Take me to Tokyo');
+  assert.equal(gradeCore(tokyo, [{ name: 'fly_to_location', args: { locationId: 'tokyo' } }]).ok, true);
+  assert.equal(gradeCore(tokyo, []).ok, false);
+
+  const chat = byPhrase('How is your evening going?');
+  const fp = gradeCore(chat, [{ name: 'get_entity_context', args: {} }]);
+  assert.equal(fp.ok, false);
+  assert.equal(fp.falsePositive, true);
+  assert.equal(gradeCore(chat, []).ok, true);
+
+  const multi = byPhrase('Switch to night vision and turn on the flights layer');
+  assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok, false);
+  assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }, { name: 'set_layer_visibility', args: {} }]).ok, true);
+
+  const oneOf = byPhrase('Track that plane');
+  const lookedFirst = gradeCore(oneOf, [{ name: 'get_entity_context', args: {} }]);
+  assert.equal(lookedFirst.ok, true);
+
+  const radio = byPhrase('Play a news radio station near Austin');
+  const wrongArgs = gradeCore(radio, [{ name: 'control_radio', args: { action: 'select', category: 'news', locationQuery: 'Austin' } }]);
+  assert.equal(wrongArgs.toolOk, true);
+  assert.equal(wrongArgs.argsOk, false);
+  assert.equal(wrongArgs.ok, false);
+
+  const globeRadio = byPhrase('Go to full planet view and then turn on the radio');
+  assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'enable' } }]).ok, false);
+  assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'play' } }]).ok, true);
+
+  const volume = byPhrase('Set the radio volume to thirty percent');
+  assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } }]).ok, true);
+  assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } }]).ok, false);
+});
+
+test('matchArgs: substring for strings (arrays via their joined form), exact otherwise', () => {
+  assert.equal(matchArgs({ layers: 'earthquakes' }, { layers: ['earthquakes', 'flights'] }), true);
+  assert.equal(matchArgs({ enabled: true }, { enabled: 'true' }), false);
+  assert.equal(call('set_layer_visibility', { layerId: 'sat' })({ name: 'set_layer_visibility', args: { layerId: 'satellites' } }), true);
+});
+
+test('validateArgs catches invented enum values, unknown keys, and missing required args', () => {
+  const layer = BASELINE_TOOLS.find((t) => t.name === 'set_layer_visibility').parameters;
+  assert.deepEqual(validateArgs(layer, { layerId: 'flights', enabled: true }), []);
+  assert.match(validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0], /not one of/);
+  assert.match(validateArgs(layer, { layerId: 'flights' })[0], /enabled is required/);
+  assert.match(validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0], /not a known argument/);
+  assert.equal(stubResult('coverage', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, false);
+  assert.equal(stubResult('core', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, true);
+});
+
+test('gradeCoverage sorts turns into the four verdicts', () => {
+  const tools = BASELINE_TOOLS;
+  const radar = coverage('weather-radar-on');
+  assert.equal(gradeCoverage(radar, [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }], 'Weather radar on.', tools).verdict, 'hallucinated-capability');
+  assert.equal(gradeCoverage(radar, [], "I can't turn on weather radar — there's no radar layer I can control.", tools).verdict, 'honest-refusal');
+  assert.equal(gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict, 'hallucinated-capability');
+  assert.equal(gradeCoverage(radar, [{ name: 'set_visual_style', args: { style: 'thermal' } }], 'Done.', tools).verdict, 'hallucinated-capability');
+  assert.equal(gradeCoverage(radar, [{ name: 'zoom_to_globe', args: {} }], 'Here is the globe.', tools).verdict, 'wrong-tool');
+
+  const imagery = coverage('recent-imagery');
+  assert.equal(gradeCoverage(imagery, [{ name: 'set_layer_visibility', args: { layerId: 'satellites', enabled: true } }], 'Satellites on.', tools).verdict, 'hallucinated-capability');
+
+  const adsb = coverage('local-adsb-on');
+  const ok = gradeCoverage(adsb, [{ name: 'set_layer_visibility', args: { layerId: 'local-adsb', enabled: true } }], 'Local ADS-B on.', tools);
+  assert.equal(ok.verdict, 'correct');
+  assert.equal(ok.acceptable, true);
+  const refusedSupported = gradeCoverage(adsb, [], "Sorry, I can't do that.", tools);
+  assert.equal(refusedSupported.verdict, 'honest-refusal');
+  assert.equal(refusedSupported.acceptable, false);
+
+  const bikes = coverage('bikes-near-me');
+  assert.equal(gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools).verdict, 'hallucinated-capability');
+  assert.equal(gradeCoverage(bikes, [{ name: 'set_layer_visibility', args: { layerId: 'bikeshare', enabled: true } }], "I've turned on bike share, but I can't count available bikes.", tools).verdict, 'honest-refusal');
+
+  const directions = coverage('driving-directions');
+  assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'route', mode: 'driving' }] } }], 'Route drawn.', tools).verdict, 'correct');
+  assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }], 'Arrow.', tools).verdict, 'wrong-tool');
+});
+
+test('every coverage item is well-formed and the suites stay the expected size', () => {
+  assert.equal(COVERAGE_PROBE.length, 24);
+  assert.ok(CORE_PHRASES.length >= 60);
+  for (const item of COVERAGE_PROBE) {
+    assert.equal(typeof item.utterance, 'string');
+    assert.equal(typeof item.supported, 'boolean');
+    if (item.supported) assert.ok(item.correct?.length, `${item.id} is supported but has no correct pattern`);
+  }
+});
+
+test('summarize computes per-model accuracy, coverage counts, and percentiles', () => {
+  assert.equal(percentile([5, 1, 3, 2, 4], 50), 3);
+  assert.equal(percentile([5, 1, 3, 2, 4], 90), 5);
+  const rows = [
+    { type: 'run', provider: 'p', model: 'm', settings: { t: 1 } },
+    { type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 1, phrase: 'a', grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true }, tFirstCallMs: 100, tDoneMs: 200, costUsd: 0.5, calls: [] },
+    { type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 2, phrase: 'a', grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false }, tFirstCallMs: null, tDoneMs: 300, costUsd: 0.25, calls: [] },
+    { type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'coverage', rep: 1, phraseId: 'x', phrase: 'x', supported: false, grade: { verdict: 'honest-refusal', acceptable: true }, tFirstCallMs: null, tDoneMs: 50, costUsd: 0.25, calls: [] },
+  ];
+  const s = summarize(rows);
+  const m = s.models[0];
+  assert.equal(m.core.accuracyPct, 50);
+  assert.deepEqual(m.core.perRepAccuracyPct, [100, 0]);
+  assert.equal(m.coverage.counts['honest-refusal'], 1);
+  assert.equal(m.coverage.acceptablePct, 100);
+  assert.equal(m.latencyMs.toolCallP50, 100);
+  assert.equal(s.cloudSpendUsd, 1);
+});
+
+test('typed argument checks feed full-call accuracy without changing the legacy verdict', () => {
+  const tokyo = byPhrase('Zoom in a bit');
+  const wrongAmount = gradeCore(tokyo, [{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } }]);
+  assert.equal(wrongAmount.ok, true);
+  assert.equal(wrongAmount.typedArgsOk, false);
+  assert.equal(wrongAmount.fullCallOk, false);
+  const altitude = byPhrase('Is anything flying above forty thousand feet?');
+  const inFeet = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }] } }]);
+  assert.equal(inFeet.fullCallOk, false);
+  const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]);
+  assert.equal(inMeters.fullCallOk, true);
+  assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true);
+});
+
+test('gradeDialogue scores turn 2 in the context of turn 1', () => {
+  const d = DIALOGUES.find((x) => x.id === 'track-then-stop');
+  const good = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }, { name: 'stop_tracking', args: {} }]]);
+  assert.equal(good.ok, true);
+  assert.equal(good.turn1Ok, true);
+  const bad = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }]]);
+  assert.equal(bad.ok, false);
+  const thanks = DIALOGUES.find((x) => x.id === 'flights-then-thanks');
+  assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []]).ok, true);
+  assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], [{ name: 'get_entity_context', args: {} }]]).ok, false);
+  assert.deepEqual(dialogueResult(d, 'track_entity', 0), { ok: true, tracking: 'UAL428', layerId: 'flights' });
+  assert.equal(dialogueResult(d, 'fly_to_location', 1), null);
+  assert.equal(DIALOGUES.every((x) => x.turns.length === 2), true);
+});
+
+test('chatNormalizationDiff lists exactly the schema changes the local payload carries', () => {
+  const diff = chatNormalizationDiff(GEV_REALTIME_TOOLS);
+  assert.deepEqual(diff, [
+    { tool: 'analyst_query', path: 'analyst_query.filters[].value', from: '(untyped)', to: ['string', 'number', 'boolean'] },
+  ]);
+});
+
+test('geminiUsageCostUsd bills text/audio/thought tokens at their own rates', () => {
+  const usd = geminiUsageCostUsd('gemini-3.8-live', {
+    promptTokenCount: 1_000_000,
+    promptTokensDetails: [{ modality: 'TEXT', tokenCount: 1_000_000 }],
+    responseTokenCount: 1_000_000,
+    responseTokensDetails: [{ modality: 'AUDIO', tokenCount: 1_000_000 }],
+    thoughtsTokenCount: 1_000_000,
+  });
+  assert.equal(usd, 0.75 + 12 + 4.5);
+});
+
+test('coverage v2 grades the same turn against the current tool surface', () => {
+  const radar = coverage('weather-radar-on');
+  const turn = [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }];
+  assert.equal(gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict, 'hallucinated-capability');
+  assert.equal(gradeCoverage(coverageRubric(radar, 'v2'), turn, 'Radar on.', GEV_REALTIME_TOOLS).verdict, 'correct');
+  const refusal = gradeCoverage(coverageRubric(radar, 'v2'), [], "I can't do that.", GEV_REALTIME_TOOLS);
+  assert.equal(refusal.acceptable, false);
+  assert.equal(coverageRubric(radar, 'v1'), radar);
+  assert.equal(COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'), true);
+});
+
+test('ROUTING_PHRASES is the core table; capability items are well-formed', () => {
+  assert.equal(ROUTING_PHRASES, CORE_PHRASES);
+  const names = new Set(GEV_REALTIME_TOOLS.map((t) => t.name));
+  for (const p of CAPABILITY_EXTRAS.filter((c) => !c.turns)) {
+    const expected = typeof p.expect === 'string' ? [p.expect] : p.expect.oneOf;
+    for (const n of expected) assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
+  }
+  const tellMe = CAPABILITY_EXTRAS.find((c) => c.phrase === 'Tell me about this');
+  assert.equal(tellMe.context.type, 'pointer_context');
+  assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'pointer' } }]).fullCallOk, true);
+  assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }]).fullCallOk, false);
+  const second = CAPABILITY_EXTRAS.find((c) => c.id === 'flights-then-second');
+  const count = { name: 'analyst_query', args: { layers: ['flights'], scope: { kind: 'view' } } };
+  assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { referent: 2 } }]]).ok, true);
+  assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { query: 'nearest' } }]]).ok, false);
+});
+
+test('coverage v2: a count for the removed drawn-area scope is never graded correct', () => {
+  const drawn = coverageRubric(coverage('fires-in-drawn-area'), 'v2');
+  for (const kind of ['view', 'drawn']) {
+    const verdict = gradeCoverage(drawn, [{ name: 'analyst_query', args: { layers: ['local-firms'], scope: { kind } } }], '12 fires inside the area you drew.');
+    assert.notEqual(verdict.verdict, 'correct');
+    assert.equal(verdict.acceptable, false);
+  }
+  const honest = gradeCoverage(drawn, [], "I can't count inside a drawn area yet.");
+  assert.equal(honest.verdict, 'honest-refusal');
+  assert.equal(honest.acceptable, true);
+});
+
+
+test('coverage validates all calls before accepting a matching capability', () => {
+  const item = coverageRubric(coverage('transit-on'), 'v2');
+  const valid = { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } };
+  const invalid = { name: 'set_layer_visibility', args: { layerId: 'transit-invented', enabled: true } };
+  for (const calls of [[invalid], [valid, invalid]]) {
+    const result = gradeCoverage(item, calls, 'Transit on.', GEV_REALTIME_TOOLS);
+    assert.equal(result.verdict, 'hallucinated-capability');
+    assert.equal(result.acceptable, false);
+  }
+  assert.equal(gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict, 'correct');
+});