mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
Merge main into PR #621 (ChatGPT OAuth for Realtime voice)
Resolve conflicts with the voice-core captions, panel resize and MCP panel key changes by keeping both sides.
This commit is contained in:
Sameh Khamis committed
commit
fb1596d2f1
244 files changed
+40846
-1558
No files matched your search
@@ -21,6 +21,9 @@ import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai';
|
||||
import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup';
|
||||
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
|
||||
import { createDebugLogHandler } from '../../server/providers/openai/debug-log.js';
|
||||
import { attachVoiceResult } from '../voice/speech.js';
|
||||
import { sanitizeDebugValue } from '../voice/realtimeDiagnostics.js';
|
||||
import { standaloneVoiceTools } from '../../server/standalone/voiceTools.js';
|
||||
|
||||
function install(plugin, preview = false) {
|
||||
@@ -577,6 +580,30 @@ test('OAuth login endpoint selects an existing local sign-in without launching C
|
||||
assert.equal(response.body.includes('fixture-existing-secret'), false);
|
||||
});
|
||||
|
||||
test('caption transcription is on by default, reported to the client meter, and can be turned off', async (t) => {
|
||||
env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
|
||||
env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
|
||||
const sent = [];
|
||||
t.mock.method(globalThis, 'fetch', async (url, options) => {
|
||||
sent.push(JSON.parse(options.body));
|
||||
return Response.json({ value: 'fixture-ephemeral' });
|
||||
});
|
||||
for (const [setting, expected] of [
|
||||
[undefined, 'gpt-4o-mini-transcribe'],
|
||||
['off', 'off'],
|
||||
]) {
|
||||
env(t, 'OPENAI_REALTIME_TRANSCRIBE_MODEL', setting);
|
||||
const response = await request(
|
||||
install(openAiRealtimeProxy()).get('/api/realtime/token'),
|
||||
);
|
||||
assert.equal(response.headers['x-gev-voice-transcribe-model'], expected);
|
||||
assert.equal(
|
||||
sent.at(-1).session.audio.input.transcription?.model ?? 'off',
|
||||
expected,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('Realtime sessions carry supplied tools, and the standalone voice adds the catalog queries', async (t) => {
|
||||
env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
|
||||
env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
|
||||
@@ -755,6 +782,269 @@ test('OpenAI routes answer generically when the upstream or the request fails',
|
||||
assert.equal(token.body.includes('fixture-upstream-secret'), false);
|
||||
});
|
||||
|
||||
test('the debug-log sink omits what people said and tool arguments unless content logging is enabled', async (t) => {
|
||||
const nestObject = (value, levels) => {
|
||||
let nested = value;
|
||||
for (let index = 0; index < levels; index += 1) nested = { nested };
|
||||
return nested;
|
||||
};
|
||||
const nestArray = (value, levels) => {
|
||||
let nested = value;
|
||||
for (let index = 0; index < levels; index += 1) nested = [nested];
|
||||
return nested;
|
||||
};
|
||||
const records = [
|
||||
{
|
||||
timestamp: '2026-10-05T00:00:00.000Z',
|
||||
sessionId: 'gev-session-1',
|
||||
event: 'tool.call',
|
||||
status: 'active',
|
||||
payload: {
|
||||
name: 'analyst_query',
|
||||
callId: 'call-1',
|
||||
arguments: { location: '12 Elm Street', note: 'meet Alice there' },
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamp: '2026-10-05T00:00:01.000Z',
|
||||
sessionId: 'gev-session-1',
|
||||
event: 'server.event',
|
||||
status: 'active',
|
||||
payload: {
|
||||
type: 'response.function_call_arguments.done',
|
||||
eventId: 'event-1',
|
||||
responseId: 'response-1',
|
||||
payload: {
|
||||
type: 'response.function_call_arguments.done',
|
||||
event_id: 'event-1',
|
||||
response_id: 'response-1',
|
||||
item_id: 'item-1',
|
||||
output_index: 0,
|
||||
call_id: 'call-1',
|
||||
name: 'analyst_query',
|
||||
arguments: '{"query":"Tokyo Station"}',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
event: 'server.event',
|
||||
status: 'active',
|
||||
payload: {
|
||||
type: 'response.audio_transcript.done',
|
||||
transcript: 'my home address is 12 Elm Street',
|
||||
text: 'call my sister',
|
||||
delta: 'Flying to Tokyo.',
|
||||
usage: { input_tokens: 40 },
|
||||
},
|
||||
},
|
||||
{
|
||||
event: 'voice.cost',
|
||||
status: 'active',
|
||||
payload: { delta: 17, total: 41 },
|
||||
},
|
||||
{
|
||||
event: 'tool.call',
|
||||
status: 'active',
|
||||
payload: { name: 'adjust_camera_zoom', callId: 'call-2', arguments: 2 },
|
||||
},
|
||||
{
|
||||
event: 'deep.object',
|
||||
status: 'active',
|
||||
payload: nestObject(
|
||||
{
|
||||
transcript: 'deep-object-transcript-secret',
|
||||
arguments: { query: 'deep-object-argument-secret' },
|
||||
},
|
||||
14,
|
||||
),
|
||||
},
|
||||
{
|
||||
event: 'deep.array',
|
||||
status: 'active',
|
||||
payload: nestArray(
|
||||
{
|
||||
transcript: 'deep-array-transcript-secret',
|
||||
arguments: { query: 'deep-array-argument-secret' },
|
||||
},
|
||||
14,
|
||||
),
|
||||
},
|
||||
];
|
||||
for (const includeContent of [false, true]) {
|
||||
const sourceRoot = root(t);
|
||||
const handler = createDebugLogHandler({ sourceRoot, includeContent });
|
||||
for (const record of records) {
|
||||
const response = await request(handler, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(record),
|
||||
});
|
||||
assert.equal(response.status, 204);
|
||||
}
|
||||
const logged = readFileSync(
|
||||
path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'),
|
||||
'utf8',
|
||||
);
|
||||
assert.equal(logged.includes('Elm Street'), includeContent);
|
||||
assert.equal(logged.includes('call my sister'), includeContent);
|
||||
assert.equal(logged.includes('meet Alice there'), includeContent);
|
||||
assert.equal(logged.includes('Tokyo Station'), includeContent);
|
||||
assert.equal(logged.includes('Flying to Tokyo'), includeContent);
|
||||
assert.equal(
|
||||
logged.includes('deep-object-transcript-secret'),
|
||||
includeContent,
|
||||
);
|
||||
assert.equal(
|
||||
logged.includes('deep-object-argument-secret'),
|
||||
includeContent,
|
||||
);
|
||||
assert.equal(
|
||||
logged.includes('deep-array-transcript-secret'),
|
||||
includeContent,
|
||||
);
|
||||
assert.equal(logged.includes('deep-array-argument-secret'), includeContent);
|
||||
const [
|
||||
toolCall,
|
||||
serverEvent,
|
||||
transcriptEvent,
|
||||
numericDelta,
|
||||
numericArguments,
|
||||
deepObject,
|
||||
deepArray,
|
||||
] = logged
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.equal(toolCall.timestamp, records[0].timestamp);
|
||||
assert.equal(toolCall.sessionId, 'gev-session-1');
|
||||
assert.equal(toolCall.event, 'tool.call');
|
||||
assert.equal(toolCall.status, 'active');
|
||||
assert.equal(toolCall.payload.name, 'analyst_query');
|
||||
assert.equal(toolCall.payload.callId, 'call-1');
|
||||
assert.equal(serverEvent.payload.eventId, 'event-1');
|
||||
assert.equal(serverEvent.payload.responseId, 'response-1');
|
||||
assert.equal(serverEvent.payload.payload.event_id, 'event-1');
|
||||
assert.equal(serverEvent.payload.payload.response_id, 'response-1');
|
||||
assert.equal(serverEvent.payload.payload.item_id, 'item-1');
|
||||
assert.equal(serverEvent.payload.payload.output_index, 0);
|
||||
assert.equal(serverEvent.payload.payload.call_id, 'call-1');
|
||||
assert.equal(serverEvent.payload.payload.name, 'analyst_query');
|
||||
assert.equal(transcriptEvent.payload.usage.input_tokens, 40);
|
||||
assert.equal(numericDelta.payload.delta, 17);
|
||||
assert.equal(numericDelta.payload.total, 41);
|
||||
if (!includeContent) {
|
||||
assert.equal(toolCall.payload.arguments, '[omitted]');
|
||||
assert.match(
|
||||
serverEvent.payload.payload.arguments,
|
||||
/^\[omitted \d+ chars\]$/,
|
||||
);
|
||||
assert.equal(transcriptEvent.payload.transcript, '[omitted 32 chars]');
|
||||
assert.equal(transcriptEvent.payload.text, '[omitted 14 chars]');
|
||||
assert.equal(transcriptEvent.payload.delta, '[omitted 16 chars]');
|
||||
assert.equal(numericArguments.payload.arguments, '[omitted]');
|
||||
assert.equal(
|
||||
JSON.stringify(deepObject.payload).includes('[omitted: max depth]'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
JSON.stringify(deepArray.payload).includes('[omitted: max depth]'),
|
||||
true,
|
||||
);
|
||||
} else {
|
||||
assert.deepEqual(
|
||||
toolCall.payload.arguments,
|
||||
records[0].payload.arguments,
|
||||
);
|
||||
assert.equal(
|
||||
serverEvent.payload.payload.arguments,
|
||||
records[1].payload.payload.arguments,
|
||||
);
|
||||
assert.equal(numericArguments.payload.arguments, 2);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('debug logs omit structured and serialized tool results unless content logging is enabled', async (t) => {
|
||||
const secret = 'private-address-fixture';
|
||||
const result = attachVoiceResult('fly_to_location', {
|
||||
ok: true,
|
||||
query: secret,
|
||||
label: secret,
|
||||
arrived: true,
|
||||
// New tool-specific fields must not need their own redaction rule.
|
||||
futureField: { nested: [secret] },
|
||||
});
|
||||
const records = [
|
||||
{
|
||||
event: 'tool.result',
|
||||
payload: { name: 'fly_to_location', callId: 'call-1', result },
|
||||
},
|
||||
{
|
||||
event: 'client.function_call_output',
|
||||
payload: {
|
||||
message: {
|
||||
type: 'conversation.item.create',
|
||||
item: {
|
||||
type: 'function_call_output',
|
||||
call_id: 'call-1',
|
||||
output: JSON.stringify(result),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
event: 'server.event',
|
||||
payload: {
|
||||
response: {
|
||||
id: 'response-1',
|
||||
output: [
|
||||
{
|
||||
type: 'message',
|
||||
content: [{ type: 'output_text', text: secret }],
|
||||
},
|
||||
],
|
||||
usage: { input_tokens: 40, output_tokens: 10 },
|
||||
},
|
||||
},
|
||||
},
|
||||
].map((record) => sanitizeDebugValue(record));
|
||||
for (const includeContent of [false, true]) {
|
||||
const sourceRoot = root(t);
|
||||
const handler = createDebugLogHandler({ sourceRoot, includeContent });
|
||||
for (const record of records) {
|
||||
const response = await request(handler, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(record),
|
||||
});
|
||||
assert.equal(response.status, 204);
|
||||
}
|
||||
const logged = readFileSync(
|
||||
path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'),
|
||||
'utf8',
|
||||
);
|
||||
assert.equal(logged.includes(secret), includeContent);
|
||||
const [tool, client, server] = logged.trim().split('\n').map(JSON.parse);
|
||||
assert.equal(tool.payload.name, 'fly_to_location');
|
||||
assert.equal(tool.payload.callId, 'call-1');
|
||||
assert.equal(client.payload.message.item.call_id, 'call-1');
|
||||
assert.equal(server.payload.response.id, 'response-1');
|
||||
assert.equal(server.payload.response.output[0].type, 'message');
|
||||
assert.deepEqual(
|
||||
server.payload.response.usage,
|
||||
records[2].payload.response.usage,
|
||||
);
|
||||
if (includeContent) {
|
||||
assert.deepEqual(tool.payload.result, result);
|
||||
assert.equal(client.payload.message.item.output, JSON.stringify(result));
|
||||
} else {
|
||||
assert.equal(tool.payload.result, '[omitted]');
|
||||
assert.match(
|
||||
client.payload.message.item.output,
|
||||
/^\[omitted \d+ chars\]$/,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('the debug-log sink stays bounded, rate limited, and quiet about failures', async (t) => {
|
||||
const sourceRoot = root(t);
|
||||
const handler = install(openAiRealtimeProxy({ sourceRoot })).get(
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,57 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
driftedEdges,
|
||||
growthDelta,
|
||||
handlePoint,
|
||||
MIN_SIZE,
|
||||
pinnedEdges,
|
||||
STORAGE_KEY,
|
||||
} from '../../scripts/qa-panel-resize.mjs';
|
||||
import { RESIZE_DIRECTIONS } from '../ui/panelResize.js';
|
||||
|
||||
test('the harness targets the key and minimums the controls use', () => {
|
||||
assert.equal(STORAGE_KEY, 'godsEyeView.v8.panelPos.cctv-panel');
|
||||
assert.deepEqual(MIN_SIZE, { width: 300, height: 160 });
|
||||
});
|
||||
|
||||
test('pinned edges are exactly the sides a direction does not name', () => {
|
||||
assert.deepEqual(pinnedEdges('n'), ['bottom', 'right', 'left']);
|
||||
assert.deepEqual(pinnedEdges('se'), ['top', 'left']);
|
||||
assert.deepEqual(pinnedEdges('nw'), ['bottom', 'right']);
|
||||
for (const dir of RESIZE_DIRECTIONS)
|
||||
assert.equal(pinnedEdges(dir).length, 4 - dir.length);
|
||||
});
|
||||
|
||||
test('growth deltas push each named edge outward', () => {
|
||||
assert.deepEqual(growthDelta('n', 40), { dx: 0, dy: -40 });
|
||||
assert.deepEqual(growthDelta('e', 40), { dx: 40, dy: 0 });
|
||||
assert.deepEqual(growthDelta('sw', 40), { dx: -40, dy: 40 });
|
||||
assert.deepEqual(growthDelta('se', 40), { dx: 40, dy: 40 });
|
||||
});
|
||||
|
||||
test('handle points land inside the strip or corner for every direction', () => {
|
||||
const rect = { left: 100, top: 200, width: 400, height: 300 };
|
||||
rect.right = rect.left + rect.width;
|
||||
rect.bottom = rect.top + rect.height;
|
||||
assert.deepEqual(handlePoint(rect, 'n'), { x: 300, y: 201 });
|
||||
assert.deepEqual(handlePoint(rect, 'w'), { x: 101, y: 350 });
|
||||
assert.deepEqual(handlePoint(rect, 'se'), { x: 496, y: 496 });
|
||||
assert.deepEqual(handlePoint(rect, 'nw'), { x: 104, y: 204 });
|
||||
for (const dir of RESIZE_DIRECTIONS) {
|
||||
const { x, y } = handlePoint(rect, dir);
|
||||
assert.ok(x >= rect.left && x <= rect.right, `${dir} x inside`);
|
||||
assert.ok(y >= rect.top && y <= rect.bottom, `${dir} y inside`);
|
||||
}
|
||||
});
|
||||
|
||||
test('drift detection reports only pinned edges that moved', () => {
|
||||
const before = { left: 100, top: 100, width: 400, height: 300 };
|
||||
const grewEast = { left: 100, top: 100, width: 440, height: 300 };
|
||||
assert.deepEqual(driftedEdges(before, grewEast, 'e'), []);
|
||||
const slidWest = { left: 60, top: 100, width: 440, height: 300 };
|
||||
assert.deepEqual(driftedEdges(before, slidWest, 'e'), ['left']);
|
||||
assert.deepEqual(driftedEdges(before, slidWest, 'w'), []);
|
||||
const wobble = { left: 101, top: 100, width: 439, height: 300 };
|
||||
assert.deepEqual(driftedEdges(before, wobble, 'e'), []);
|
||||
});
|
||||
@@ -44,6 +44,7 @@ test('real dev and built-preview servers serve provider JSON and terminate unkno
|
||||
TOMTOM_API_KEY: '',
|
||||
GOOGLE_MAPS_API_KEY: '',
|
||||
GOOGLE_MAPS_SERVER_API_KEY: '',
|
||||
MAPILLARY_CLIENT_TOKEN: '',
|
||||
OPENAI_API_KEY: '',
|
||||
OPENSKY_AUTH_MODE: 'anon',
|
||||
OPENSKY_CLIENT_ID: '',
|
||||
@@ -108,6 +109,10 @@ test('real dev and built-preview servers serve provider JSON and terminate unkno
|
||||
['/api/cctv/sources', 200],
|
||||
['/api/gbfs/', 400],
|
||||
['/api/tomtom/status', 200],
|
||||
// The Street Level QA gate intercepts these in the browser; this is
|
||||
// the check that the real servers register them.
|
||||
['/api/mapillary/status', 200],
|
||||
['/api/mapillary/tiles/coverage/14/1/2', 503],
|
||||
['/api/radio/unknown', 404],
|
||||
['/api/setup/status', isPreview ? 404 : 200],
|
||||
['/api/setup/update', 404],
|
||||
@@ -128,6 +133,10 @@ test('real dev and built-preview servers serve provider JSON and terminate unkno
|
||||
const body = await response.json();
|
||||
if (route === '/api/cctv/sources')
|
||||
assert.equal(body.sources[0].id, 'fixture');
|
||||
if (route === '/api/mapillary/status')
|
||||
assert.deepEqual(body, { configured: false }, route);
|
||||
if (route.startsWith('/api/mapillary/tiles'))
|
||||
assert.deepEqual(body, { error: 'no_key', keyRequired: true }, route);
|
||||
if (
|
||||
route === '/api/does-not-exist' ||
|
||||
(isPreview && route.startsWith('/api/setup'))
|
||||
|
||||
@@ -0,0 +1,636 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import test from 'node:test';
|
||||
import { createApplicationStreetLevel } from '../app/layers/streetLevel.js';
|
||||
import {
|
||||
fixtureTile,
|
||||
EXPECTED_PROVIDERS,
|
||||
isCollapsed,
|
||||
PARK,
|
||||
STRICT_ALLOWED_SKIPS,
|
||||
strictViolations,
|
||||
} from '../../scripts/qa-street-level.mjs';
|
||||
import {
|
||||
answerMapillaryRequest,
|
||||
describeCall,
|
||||
metresApart,
|
||||
PHOTO_LINE,
|
||||
PHOTO_SEQUENCE_ID,
|
||||
photoImages,
|
||||
THUMB_HOST,
|
||||
} from '../../scripts/fixtures/street-level/mapillaryGraph.mjs';
|
||||
import { redact } from '../../scripts/qa-browserEvidence.mjs';
|
||||
|
||||
// The press helpers wait for paint; Node has no frames, so a tick stands in.
|
||||
globalThis.requestAnimationFrame ??= (callback) =>
|
||||
setTimeout(() => callback(Date.now()), 0);
|
||||
|
||||
test('the harness expects exactly the providers the app registers, in chip order', () => {
|
||||
// Any method a provider asks its source for is a no-op: only ids matter.
|
||||
const source = new Proxy({}, { get: () => () => {} });
|
||||
const layer = createApplicationStreetLevel({
|
||||
surface: null,
|
||||
sources: { mapillary: source },
|
||||
});
|
||||
assert.deepEqual([...EXPECTED_PROVIDERS], [...layer.providerIds]);
|
||||
});
|
||||
|
||||
test('a panel reads as collapsed only with the collapsed class', () => {
|
||||
assert.equal(isCollapsed(['panel-collapsible', 'collapsed']), true);
|
||||
assert.equal(isCollapsed(['panel-collapsible']), false);
|
||||
});
|
||||
|
||||
test('the harness only runs its browser flow when executed directly', () => {
|
||||
const source = fs.readFileSync(
|
||||
new URL('../../scripts/qa-street-level.mjs', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(
|
||||
source,
|
||||
/import\.meta\.url === pathToFileURL\(process\.argv\[1\]\)\.href/,
|
||||
);
|
||||
assert.match(source, /PUPPETEER_EXECUTABLE_PATH/);
|
||||
assert.match(source, /--url/);
|
||||
});
|
||||
|
||||
test('fixture tiles give the hermetic gate something real to filter', async () => {
|
||||
const { decodeCoverageTile } =
|
||||
await import('../layers/streetLevel/providers/mapillary/decode.js');
|
||||
const now = Date.UTC(2026, 9, 1);
|
||||
const street = decodeCoverageTile(fixtureTile(14, 2662, 6286, now), {
|
||||
x: 2662,
|
||||
y: 6286,
|
||||
z: 14,
|
||||
});
|
||||
const grid = street.sequences.filter((s) => s.id !== PHOTO_SEQUENCE_ID);
|
||||
assert.equal(grid.length, 8);
|
||||
const pano = grid.filter((s) => s.isPano).length;
|
||||
assert.ok(pano > 0 && pano < 8, '360° and flat both present');
|
||||
const year = 365 * 86_400_000;
|
||||
const old = grid.filter((s) => now - s.capturedAt > year).length;
|
||||
assert.ok(old > 0 && old < 8, 'recent and older both present');
|
||||
const orbit = decodeCoverageTile(fixtureTile(3, 1, 3, now), {
|
||||
x: 1,
|
||||
y: 3,
|
||||
z: 3,
|
||||
});
|
||||
assert.ok(orbit.overview.length > 0, 'overview points from orbit');
|
||||
assert.equal(fixtureTile(8, 1, 1, now).length, 0, 'nothing in between');
|
||||
});
|
||||
|
||||
/* ── The real status route probe ─────────────────────────────────────── */
|
||||
|
||||
test('the gate requires the server’s real Mapillary status route', async () => {
|
||||
const { assertRealStatusRoute } =
|
||||
await import('../../scripts/qa-street-level.mjs');
|
||||
const asked = [];
|
||||
const answer = (body, init) => async (url) => {
|
||||
asked.push(url);
|
||||
return typeof body === 'string'
|
||||
? new Response(body, init)
|
||||
: Response.json(body, init);
|
||||
};
|
||||
assert.deepEqual(
|
||||
await assertRealStatusRoute(
|
||||
'http://localhost:4173',
|
||||
answer({ configured: false }),
|
||||
),
|
||||
{ configured: false },
|
||||
);
|
||||
assert.deepEqual(asked, ['http://localhost:4173/api/mapillary/status']);
|
||||
for (const [why, fetchImpl] of [
|
||||
[
|
||||
'an unregistered route (the API 404)',
|
||||
answer({ error: 'Unknown API route' }, { status: 404 }),
|
||||
],
|
||||
[
|
||||
'the SPA fallback',
|
||||
answer('<!doctype html><title>GEV</title>', {
|
||||
headers: { 'content-type': 'text/html' },
|
||||
}),
|
||||
],
|
||||
['a malformed status', answer({ configured: 'yes' })],
|
||||
[
|
||||
'an unreachable server',
|
||||
async () => {
|
||||
throw new TypeError('fetch failed');
|
||||
},
|
||||
],
|
||||
])
|
||||
await assert.rejects(
|
||||
assertRealStatusRoute('http://localhost:4173', fetchImpl),
|
||||
/api\/mapillary\/status/,
|
||||
why,
|
||||
);
|
||||
});
|
||||
|
||||
/* ── Header press-and-verify shared with the panel gate ───────────────── */
|
||||
|
||||
/**
|
||||
* A page stand-in that runs page functions in Node against a fake document.
|
||||
* The hit check always sees the header; `presses` says where each pointerdown
|
||||
* lands ('header', 'other', 'moved' or 'none').
|
||||
*/
|
||||
function fakePanelPage({ presses = [], floating = false, stuck = false } = {}) {
|
||||
const state = { floating, downs: 0, last: null, dragging: false, shift: 0 };
|
||||
const listeners = [];
|
||||
const node = (tagName, id, classes, inHeader) => ({
|
||||
tagName,
|
||||
id,
|
||||
classList: classes,
|
||||
closest: (selector) =>
|
||||
inHeader && selector.includes('.panel-header') ? {} : null,
|
||||
});
|
||||
const header = node('DIV', '', ['panel-title'], true);
|
||||
const other = node('SECTION', 'weather-panel', ['panel'], false);
|
||||
const rect = (left, top, width, height) => () => ({
|
||||
left,
|
||||
top,
|
||||
width,
|
||||
height,
|
||||
});
|
||||
const document = {
|
||||
querySelector: () => ({ getBoundingClientRect: rect(120, 210, 100, 20) }),
|
||||
getElementById: () => ({
|
||||
getBoundingClientRect: () => rect(100, 200 + state.shift, 300, 400)(),
|
||||
classList: {
|
||||
contains: (name) => name === 'panel-floating' && state.floating,
|
||||
},
|
||||
}),
|
||||
elementFromPoint: () => header,
|
||||
};
|
||||
const window = {
|
||||
addEventListener: (type, listener) => listeners.push(listener),
|
||||
};
|
||||
const inPage = (fn, ...args) => {
|
||||
const saved = { document: globalThis.document, window: globalThis.window };
|
||||
Object.assign(globalThis, { document, window });
|
||||
try {
|
||||
return fn(...args);
|
||||
} finally {
|
||||
Object.assign(globalThis, saved);
|
||||
}
|
||||
};
|
||||
const evaluate = async (fn, ...args) => inPage(fn, ...args);
|
||||
const press = () => {
|
||||
state.last = presses[state.downs++] ?? 'header';
|
||||
// 'none': the pointerdown never reaches the page.
|
||||
if (state.last === 'none') return;
|
||||
const target = state.last === 'header' ? header : other;
|
||||
// 'moved': the panel's layout shifted just before the press landed.
|
||||
if (state.last === 'moved') state.shift = -116;
|
||||
for (const listener of listeners.splice(0)) inPage(listener, { target });
|
||||
state.shift = 0;
|
||||
};
|
||||
return {
|
||||
state,
|
||||
evaluate,
|
||||
async waitForFunction(fn, options, ...args) {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (await evaluate(fn, ...args)) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
}
|
||||
throw Object.assign(new Error('timed out'), { name: 'TimeoutError' });
|
||||
},
|
||||
mouse: {
|
||||
async move() {},
|
||||
async down() {
|
||||
press();
|
||||
state.dragging = true;
|
||||
},
|
||||
async up() {
|
||||
if (state.dragging && state.last === 'header' && !stuck)
|
||||
state.floating = true;
|
||||
state.dragging = false;
|
||||
},
|
||||
async click(x, y, { clickCount }) {
|
||||
if (clickCount === 1) press();
|
||||
else if (state.last === 'header' && !stuck) state.floating = false;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('a header press is retried only when it provably missed the header', async () => {
|
||||
const { pressMissed } = await import('../../scripts/qa-panelDrag.mjs');
|
||||
assert.equal(pressMissed({ done: true, pressed: null }), false);
|
||||
assert.equal(
|
||||
pressMissed({ done: false, pressed: { inHeader: true } }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
pressMissed({ done: false, pressed: { inHeader: false } }),
|
||||
true,
|
||||
);
|
||||
// No recorded press is a failure, not a miss to retry.
|
||||
assert.equal(pressMissed({ done: false, pressed: null }), false);
|
||||
});
|
||||
|
||||
test('a lift whose press the rail moved off the header is retried, then lifts', async () => {
|
||||
const { liftPanelByHeader, describePress } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
const page = fakePanelPage({ presses: ['other', 'header'] });
|
||||
const notes = [];
|
||||
const attempt = await liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: (line) => notes.push(line),
|
||||
pauseMs: 0,
|
||||
});
|
||||
assert.equal(attempt.done, true);
|
||||
assert.equal(page.state.downs, 2);
|
||||
assert.equal(notes.length, 2, 'a note and a GitHub warning');
|
||||
assert.match(notes[0], /missed the street-level-panel header/);
|
||||
assert.match(notes[0], /section#weather-panel\.panel/, 'names what it hit');
|
||||
assert.match(notes[0], /pre-press hit check passed/);
|
||||
assert.match(notes[0], /stillness (held|timed out)/);
|
||||
assert.match(
|
||||
notes[0],
|
||||
/panel box at hit check 100,200 300x400 -> at press 100,200 300x400/,
|
||||
);
|
||||
assert.match(notes[1], /^::warning title=Panel press retried::retry 1\/2/);
|
||||
assert.match(describePress(attempt), /in header: true, after 1 retry$/);
|
||||
});
|
||||
|
||||
test('a press in the header that does not lift fails at once, naming the target', async () => {
|
||||
const { liftPanelByHeader, describePress } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
const page = fakePanelPage({ stuck: true });
|
||||
const attempt = await liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: () => assert.fail('no retry'),
|
||||
pauseMs: 0,
|
||||
});
|
||||
assert.equal(attempt.done, false);
|
||||
assert.equal(page.state.downs, 1);
|
||||
assert.match(describePress(attempt), /on div\.panel-title, in header: true/);
|
||||
});
|
||||
|
||||
test('presses that keep missing stop after the retries and report the miss', async () => {
|
||||
const { liftPanelByHeader, PRESS_RETRIES } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
const page = fakePanelPage({ presses: ['other', 'other', 'other', 'other'] });
|
||||
const attempt = await liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: () => {},
|
||||
pauseMs: 0,
|
||||
});
|
||||
assert.equal(attempt.done, false);
|
||||
assert.equal(page.state.downs, PRESS_RETRIES + 1);
|
||||
assert.equal(attempt.pressed.inHeader, false);
|
||||
});
|
||||
|
||||
test('a header double-click docks, waiting on the docked state', async () => {
|
||||
const { dockPanelByDoubleClick } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
const page = fakePanelPage({ floating: true, presses: ['other', 'header'] });
|
||||
const attempt = await dockPanelByDoubleClick(page, 'street-level-panel', {
|
||||
log: () => {},
|
||||
pauseMs: 0,
|
||||
});
|
||||
assert.equal(attempt.done, true);
|
||||
assert.equal(page.state.floating, false);
|
||||
assert.equal(page.state.downs, 2, 'the missed first press was retried');
|
||||
});
|
||||
|
||||
test('both panel gates press the header through the shared helper', () => {
|
||||
for (const script of ['qa-street-level.mjs', 'qa-panel-resize.mjs']) {
|
||||
const source = fs.readFileSync(
|
||||
new URL(`../../scripts/${script}`, import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /from '\.\/qa-panelDrag\.mjs'/, script);
|
||||
assert.match(source, /liftPanelByHeader\(page, PANEL_ID/, script);
|
||||
assert.match(source, /dockPanelByDoubleClick\(page, PANEL_ID/, script);
|
||||
}
|
||||
});
|
||||
|
||||
/* ── Hermetic photo flow: the photo line and the Graph fixtures ────────── */
|
||||
|
||||
test('the photo sequence runs through the parked view, clipped to each tile it crosses', async () => {
|
||||
const { decodeCoverageTile } =
|
||||
await import('../layers/streetLevel/providers/mapillary/decode.js');
|
||||
const { lonToTileX, latToTileY } =
|
||||
await import('../layers/streetLevel/tileMath.js');
|
||||
const x = lonToTileX(PHOTO_LINE.lon, 14);
|
||||
const rows = new Set(
|
||||
[PHOTO_LINE.south, PHOTO_LINE.north].map((lat) => latToTileY(lat, 14)),
|
||||
);
|
||||
assert.ok(rows.size > 1, 'the line crosses a tile edge');
|
||||
let south = Infinity;
|
||||
let north = -Infinity;
|
||||
for (const y of rows) {
|
||||
const line = decodeCoverageTile(fixtureTile(14, x, y), {
|
||||
x,
|
||||
y,
|
||||
z: 14,
|
||||
}).sequences.find((s) => s.id === PHOTO_SEQUENCE_ID);
|
||||
assert.ok(line, `tile 14/${x}/${y} carries the photo line`);
|
||||
for (const [lon, lat] of line.parts[0]) {
|
||||
assert.ok(Math.abs(lon - PHOTO_LINE.lon) < 1e-4);
|
||||
south = Math.min(south, lat);
|
||||
north = Math.max(north, lat);
|
||||
}
|
||||
}
|
||||
assert.ok(Math.abs(south - PHOTO_LINE.south) < 1e-4);
|
||||
assert.ok(Math.abs(north - PHOTO_LINE.north) < 1e-4);
|
||||
// Far away there is no photo line, only the grid.
|
||||
const elsewhere = decodeCoverageTile(fixtureTile(14, 100, 100), {
|
||||
x: 100,
|
||||
y: 100,
|
||||
z: 14,
|
||||
});
|
||||
assert.equal(
|
||||
elsewhere.sequences.some((s) => s.id === PHOTO_SEQUENCE_ID),
|
||||
false,
|
||||
);
|
||||
// The parked camera looks down onto the line.
|
||||
assert.equal(PARK.lon, PHOTO_LINE.lon);
|
||||
assert.ok(PARK.lat > PHOTO_LINE.south && PARK.lat < PHOTO_LINE.north);
|
||||
});
|
||||
|
||||
test('the fixture photos: one sequence, 360° and flat, close enough for any nearest lookup on the line', () => {
|
||||
const images = photoImages(Date.UTC(2026, 9, 1));
|
||||
assert.ok(images.length >= 3);
|
||||
assert.ok(images.some((image) => image.isPano));
|
||||
assert.ok(images.some((image) => !image.isPano));
|
||||
for (let i = 1; i < images.length; i++) {
|
||||
const gap = metresApart(images[i - 1], images[i]);
|
||||
// Above the cones' 3 m thinning, and at most 50 m apart: any point on
|
||||
// the line has a photo within the Graph API's 50 m radius.
|
||||
assert.ok(gap > 3 && gap < 50, `gap ${gap}`);
|
||||
}
|
||||
assert.equal(new Set(images.map((image) => image.id)).size, images.length);
|
||||
});
|
||||
|
||||
const graph = (path, params) =>
|
||||
`https://graph.mapillary.com/${path}?${new URLSearchParams({ access_token: 'MLY|0|qa-fixture', ...params })}`;
|
||||
const answer = (url, method = 'GET') => {
|
||||
const out = answerMapillaryRequest({ method, url });
|
||||
return {
|
||||
...out,
|
||||
json: out.contentType === 'application/json' ? JSON.parse(out.body) : null,
|
||||
};
|
||||
};
|
||||
|
||||
test('the Graph fixtures answer the provider’s nearest and sequence lookups', () => {
|
||||
const images = photoImages();
|
||||
const middle = images[Math.floor(images.length / 2)];
|
||||
const near = answer(
|
||||
graph('images', {
|
||||
lat: String(middle.lat + 0.0001),
|
||||
lng: String(middle.lon),
|
||||
radius: '50',
|
||||
limit: '8',
|
||||
fields: 'id,geometry,is_pano,captured_at,sequence',
|
||||
}),
|
||||
);
|
||||
assert.equal(near.status, 200);
|
||||
assert.ok(near.json.data.length > 0);
|
||||
assert.ok(near.json.data.some((record) => record.id === middle.id));
|
||||
for (const record of near.json.data) {
|
||||
assert.deepEqual(Object.keys(record).sort(), [
|
||||
'captured_at',
|
||||
'geometry',
|
||||
'id',
|
||||
'is_pano',
|
||||
'sequence',
|
||||
]);
|
||||
assert.equal(record.sequence, PHOTO_SEQUENCE_ID);
|
||||
}
|
||||
const far = answer(
|
||||
graph('images', { lat: '0', lng: '0', radius: '50', fields: 'id' }),
|
||||
);
|
||||
assert.deepEqual(far.json.data, [], 'nothing outside the radius');
|
||||
const sequence = answer(
|
||||
graph('images', {
|
||||
sequence_ids: PHOTO_SEQUENCE_ID,
|
||||
fields: 'id,geometry,compass_angle,captured_at,is_pano',
|
||||
limit: '2000',
|
||||
}),
|
||||
);
|
||||
assert.equal(sequence.json.data.length, images.length);
|
||||
assert.deepEqual(
|
||||
answer(graph('images', { sequence_ids: 'other', fields: 'id' })).json.data,
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('the Graph fixtures answer every call MapillaryJS makes to open a photo', () => {
|
||||
const [first, second] = photoImages();
|
||||
const spatial = answer(
|
||||
graph('images', {
|
||||
image_ids: `${first.id},${second.id}`,
|
||||
fields:
|
||||
'id,computed_geometry,geometry,sequence,camera_type,computed_rotation,thumb_1024_url,thumb_2048_url,width,height,merge_cc,sfm_cluster,mesh,creator,captured_at',
|
||||
}),
|
||||
);
|
||||
assert.equal(spatial.status, 200);
|
||||
assert.equal(spatial.headers['Access-Control-Allow-Origin'], '*');
|
||||
const [pano, flat] = spatial.json.data;
|
||||
assert.equal(pano.camera_type, 'spherical');
|
||||
assert.equal(flat.camera_type, 'perspective');
|
||||
assert.equal(pano.merge_cc, null, 'unmerged: no mesh request');
|
||||
assert.equal(pano.sfm_cluster, null, 'no cluster request');
|
||||
assert.equal(pano.computed_rotation.length, 3);
|
||||
assert.equal(new URL(pano.thumb_2048_url).hostname, THUMB_HOST);
|
||||
assert.equal(
|
||||
answer(graph('images', { s2: '9749618446378729472', fields: 'id' })).json
|
||||
.data.length,
|
||||
0,
|
||||
);
|
||||
assert.deepEqual(
|
||||
answer(graph('image_ids', { sequence_id: PHOTO_SEQUENCE_ID })).json.data[0],
|
||||
{ id: first.id },
|
||||
);
|
||||
assert.deepEqual(
|
||||
answer(graph(`${first.id}/tiles`, { z: '11', fields: 'url,z,x,y' })).json,
|
||||
{ data: [] },
|
||||
);
|
||||
const photo = answer(pano.thumb_2048_url);
|
||||
assert.equal(photo.status, 200);
|
||||
assert.equal(photo.contentType, 'image/jpeg');
|
||||
assert.deepEqual([...photo.body.subarray(0, 3)], [0xff, 0xd8, 0xff]);
|
||||
const preflight = answer(graph('images', { image_ids: first.id }), 'OPTIONS');
|
||||
assert.equal(preflight.status, 204);
|
||||
assert.match(
|
||||
preflight.headers['Access-Control-Allow-Headers'],
|
||||
/Authorization/,
|
||||
);
|
||||
});
|
||||
|
||||
test('a Mapillary call no fixture covers is answered but flagged unknown', () => {
|
||||
for (const url of [
|
||||
graph('images', { bbox: '0,0,1,1' }),
|
||||
graph('map_features', { fields: 'id' }),
|
||||
'https://tiles.mapillary.com/maps/vtp/mly1_public/2/14/1/1',
|
||||
]) {
|
||||
const out = answer(url);
|
||||
assert.equal(out.known, false, url);
|
||||
assert.ok(out.status >= 400);
|
||||
}
|
||||
});
|
||||
|
||||
test('answered calls are named without the token', () => {
|
||||
const call = describeCall(
|
||||
'GET',
|
||||
new URL(graph('9100000000000/tiles', { z: '11', fields: 'url' })),
|
||||
);
|
||||
assert.equal(call, 'GET graph.mapillary.com/{imageId}/tiles?fields&z');
|
||||
assert.doesNotMatch(call, /MLY|access_token/);
|
||||
assert.equal(
|
||||
describeCall(
|
||||
'GET',
|
||||
new URL('https://qa-fixture.mapillary.com/thumb/9100000000016.jpg'),
|
||||
),
|
||||
'GET qa-fixture.mapillary.com/thumb/{imageId}.jpg',
|
||||
);
|
||||
});
|
||||
|
||||
test('evidence never carries a Mapillary token', () => {
|
||||
assert.equal(
|
||||
redact(
|
||||
'GET https://graph.mapillary.com/images?access_token=MLY|123|abc&fields=id',
|
||||
),
|
||||
'GET https://graph.mapillary.com/images?access_token=REDACTED&fields=id',
|
||||
);
|
||||
assert.equal(
|
||||
redact('token MLY|123|abc in a log'),
|
||||
'token MLY|REDACTED in a log',
|
||||
);
|
||||
assert.equal(
|
||||
redact(
|
||||
'https://tile.googleapis.com/v1/3dtiles/x.glb?session=s1&key=AIzaSyBexampleexampleexampleexample00',
|
||||
),
|
||||
'https://tile.googleapis.com/v1/3dtiles/x.glb?session=s1&key=REDACTED',
|
||||
);
|
||||
assert.equal(
|
||||
redact('loaded with AIzaSyBexampleexampleexampleexample00 inline'),
|
||||
'loaded with AIza…REDACTED inline',
|
||||
);
|
||||
});
|
||||
|
||||
test('--strict accepts only the documented skips', () => {
|
||||
assert.deepEqual(
|
||||
[...STRICT_ALLOWED_SKIPS],
|
||||
['no Google 3D', 'fixtures only'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
strictViolations([
|
||||
{ reason: 'no Google 3D', label: 'terrain' },
|
||||
{ reason: 'fixtures only', label: 'keyless page' },
|
||||
{ reason: 'no Mapillary key', label: 'the keyed steps' },
|
||||
]),
|
||||
[{ reason: 'no Mapillary key', label: 'the keyed steps' }],
|
||||
);
|
||||
});
|
||||
|
||||
test('a press that records no pointerdown fails at once, without a retry', async () => {
|
||||
const { liftPanelByHeader, describePress } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
const page = fakePanelPage({ presses: ['none', 'header'] });
|
||||
const attempt = await liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: () => assert.fail('no retry'),
|
||||
pauseMs: 0,
|
||||
});
|
||||
assert.equal(attempt.done, false);
|
||||
assert.equal(attempt.pressed, null);
|
||||
assert.equal(page.state.downs, 1);
|
||||
assert.match(describePress(attempt), /no pointerdown reached the page/);
|
||||
});
|
||||
|
||||
test('QA_FAIL_ON_RETRY turns a retry into a failure that says why', async () => {
|
||||
const { liftPanelByHeader, failOnRetryDefault } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
assert.equal(failOnRetryDefault({ QA_FAIL_ON_RETRY: '1' }, []), true);
|
||||
assert.equal(failOnRetryDefault({}, ['--fail-on-retry']), true);
|
||||
assert.equal(failOnRetryDefault({}, []), false);
|
||||
const page = fakePanelPage({ presses: ['other', 'header'] });
|
||||
await assert.rejects(
|
||||
liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: () => {},
|
||||
pauseMs: 0,
|
||||
failOnRetry: true,
|
||||
}),
|
||||
/QA_FAIL_ON_RETRY=1 .*missed the street-level-panel header.*pre-press hit check passed/,
|
||||
);
|
||||
assert.equal(page.state.downs, 1);
|
||||
});
|
||||
|
||||
test('QA_FAIL_ON_RETRY retries, with a warning, a miss the panel moving explains', async () => {
|
||||
const { liftPanelByHeader, panelMoved } =
|
||||
await import('../../scripts/qa-panelDrag.mjs');
|
||||
assert.equal(panelMoved({ left: 0, top: 382 }, { left: 0, top: 266 }), true);
|
||||
assert.equal(panelMoved({ left: 0, top: 382 }, { left: 1, top: 383 }), false);
|
||||
const page = fakePanelPage({ presses: ['moved', 'header'] });
|
||||
const lines = [];
|
||||
const attempt = await liftPanelByHeader(page, 'street-level-panel', {
|
||||
dx: -400,
|
||||
dy: 80,
|
||||
log: (line) => lines.push(line),
|
||||
pauseMs: 0,
|
||||
failOnRetry: true,
|
||||
});
|
||||
assert.equal(attempt.done, true, 'the second press lifted it');
|
||||
assert.equal(page.state.downs, 2);
|
||||
assert.ok(
|
||||
lines.some((line) =>
|
||||
/::warning title=Panel press retried::.*-> at press 100,84/.test(line),
|
||||
),
|
||||
lines.join('\n'),
|
||||
);
|
||||
});
|
||||
|
||||
test('both gates save failure evidence and fail on render-loop errors', () => {
|
||||
for (const script of ['qa-street-level.mjs', 'qa-panel-resize.mjs']) {
|
||||
const source = fs.readFileSync(
|
||||
new URL(`../../scripts/${script}`, import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(source, /saveFailureArtifacts\(/, script);
|
||||
assert.match(source, /hookRenderErrors\(/, script);
|
||||
assert.match(source, /readRenderErrors\(/, script);
|
||||
assert.match(source, /--fail-on-retry/, script);
|
||||
}
|
||||
});
|
||||
|
||||
test('CI runs the gates hermetically against a production build, strict, with evidence', () => {
|
||||
const ci = fs.readFileSync(
|
||||
new URL('../../.github/workflows/ci.yml', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const job = ci.slice(ci.indexOf('street-level-browser:'));
|
||||
const gate = job.slice(0, job.indexOf('\n windows-onboarding:'));
|
||||
// The dummy token is baked into the bundle at build time and the preview
|
||||
// server's status route reads it at run time: both steps carry it.
|
||||
assert.match(
|
||||
gate,
|
||||
/MAPILLARY_CLIENT_TOKEN: 'MLY\|0\|qa-fixture'\s+run: npm run build/,
|
||||
);
|
||||
assert.match(
|
||||
gate,
|
||||
/MAPILLARY_CLIENT_TOKEN: 'MLY\|0\|qa-fixture'\s+run: \|\s+npx vite preview --port 4173 --strictPort/,
|
||||
);
|
||||
assert.doesNotMatch(gate, /npx vite --port/);
|
||||
assert.equal(
|
||||
(gate.match(/MAPILLARY_CLIENT_TOKEN: 'MLY\|0\|qa-fixture'/g) || []).length,
|
||||
2,
|
||||
);
|
||||
assert.match(gate, /qa:street-level:fixtures -- --strict/);
|
||||
assert.match(
|
||||
gate,
|
||||
/if: failure\(\)[\s\S]*actions\/upload-artifact@[0-9a-f]{40}/,
|
||||
);
|
||||
assert.match(gate, /path: qa-artifacts\//);
|
||||
assert.match(
|
||||
gate,
|
||||
/QA_FAIL_ON_RETRY: '1'/,
|
||||
'a missed header press fails CI now that the rail settles',
|
||||
);
|
||||
});
|
||||
@@ -675,7 +675,7 @@ test('public transit documentation uses courtesy attribution and the approved gr
|
||||
);
|
||||
assert.ok(
|
||||
readme.includes(
|
||||
'Nineteen layers and map sources. **Seventeen have a keyless path.**',
|
||||
'Twenty layers and map sources. **Seventeen have a keyless path.**',
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
|
||||
@@ -38,7 +38,14 @@ test('explicit build inputs preserve browser-only defines, plugin order and loop
|
||||
assert.deepEqual(config.define, {
|
||||
'import.meta.env.GOOGLE_MAPS_API_KEY': '"browser-fixture"',
|
||||
'import.meta.env.CESIUM_ION_TOKEN': '"ion-fixture"',
|
||||
'import.meta.env.MAPILLARY_CLIENT_TOKEN': '""',
|
||||
});
|
||||
assert.equal(
|
||||
createBrowserViteConfig({ mapillaryToken: 'MLY|1|abc' }).define[
|
||||
'import.meta.env.MAPILLARY_CLIENT_TOKEN'
|
||||
],
|
||||
'"MLY|1|abc"',
|
||||
);
|
||||
// A wildcard bind keeps the Host-header check; LAN names are explicit.
|
||||
for (const host of ['0.0.0.0', '::'])
|
||||
assert.deepEqual(
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
import { GEV_ACTION_SCHEMAS } from '../voice/actionSchemas.js';
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
normalizeSchemaForChat,
|
||||
toChatTools,
|
||||
toGeminiDeclarations,
|
||||
toGeminiSchema,
|
||||
chatNormalizationDiff,
|
||||
} from '../../scripts/voice-bench/toolFormats.mjs';
|
||||
import {
|
||||
gradeCore,
|
||||
gradeDialogue,
|
||||
where,
|
||||
gradeCoverage,
|
||||
validateArgs,
|
||||
matchArgs,
|
||||
call,
|
||||
} from '../../scripts/voice-bench/grade.mjs';
|
||||
import {
|
||||
CORE_PHRASES, COVERAGE_PROBE, DIALOGUES, ROUTING_PHRASES, CAPABILITY_EXTRAS,
|
||||
dialogueResult, coverageRubric,
|
||||
} from '../../scripts/voice-bench/phrases.mjs';
|
||||
import { stubResult } from '../../scripts/voice-bench/stub.mjs';
|
||||
import { summarize, percentile } from '../../scripts/voice-bench/report.mjs';
|
||||
import { geminiUsageCostUsd } from '../../scripts/voice-bench/pricing.mjs';
|
||||
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
// The 4b56d0e9 tool schema: the coverage v1 rubric grades against it.
|
||||
const BASELINE_TOOLS = JSON.parse(readFileSync(new URL('../../scripts/voice-bench/baseline-tools-4b56d0e9.json', import.meta.url), 'utf8'));
|
||||
|
||||
const byPhrase = (text) => CORE_PHRASES.find((p) => p.phrase === text);
|
||||
const coverage = (id) => COVERAGE_PROBE.find((c) => c.id === id);
|
||||
|
||||
test('toChatTools wraps every Realtime tool without changing names or descriptions', () => {
|
||||
const chat = toChatTools(GEV_REALTIME_TOOLS);
|
||||
assert.equal(chat.length, GEV_REALTIME_TOOLS.length);
|
||||
chat.forEach((t, i) => {
|
||||
assert.equal(t.type, 'function');
|
||||
assert.equal(t.function.name, GEV_REALTIME_TOOLS[i].name);
|
||||
assert.equal(t.function.description, GEV_REALTIME_TOOLS[i].description);
|
||||
assert.equal(t.function.parameters.type, 'object');
|
||||
});
|
||||
});
|
||||
|
||||
test('normalizeSchemaForChat types the untyped analyst_query filter value and nothing else', () => {
|
||||
const analyst = GEV_REALTIME_TOOLS.find((t) => t.name === 'analyst_query');
|
||||
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
|
||||
const normalized = normalizeSchemaForChat(analyst.parameters);
|
||||
assert.deepEqual(normalized.properties.filters.items.properties.value.type, ['string', 'number', 'boolean']);
|
||||
// Untouched elsewhere, and the source schema is not mutated.
|
||||
assert.deepEqual(normalized.properties.layers, analyst.parameters.properties.layers);
|
||||
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
|
||||
// Every property in every normalized tool now declares a type or an enum/anyOf.
|
||||
const untyped = [];
|
||||
const visit = (node, where) => {
|
||||
if (!node || typeof node !== 'object') return;
|
||||
for (const [k, v] of Object.entries(node.properties || {})) {
|
||||
if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v)) untyped.push(`${where}.${k}`);
|
||||
visit(v, `${where}.${k}`);
|
||||
}
|
||||
if (node.items) visit(node.items, `${where}[]`);
|
||||
};
|
||||
for (const t of toChatTools(GEV_REALTIME_TOOLS)) visit(t.function.parameters, t.function.name);
|
||||
assert.deepEqual(untyped, []);
|
||||
});
|
||||
|
||||
test('toGeminiSchema drops unsupported keywords and expresses unions as anyOf', () => {
|
||||
const out = toGeminiSchema({
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: { a: { type: ['string', 'number'] }, b: {}, c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 } },
|
||||
required: ['a'],
|
||||
});
|
||||
assert.equal('additionalProperties' in out, false);
|
||||
assert.deepEqual(out.properties.a, { anyOf: [{ type: 'string' }, { type: 'number' }] });
|
||||
assert.deepEqual(out.properties.b.anyOf.map((s) => s.type), ['string', 'number', 'boolean']);
|
||||
assert.deepEqual(out.properties.c, { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 });
|
||||
assert.deepEqual(out.required, ['a']);
|
||||
});
|
||||
|
||||
test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode', () => {
|
||||
// The full action schemas carry the query-or-referent rule; the OpenAI
|
||||
// Realtime tools omit it because that API rejects top-level anyOf.
|
||||
const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({ type: 'function', ...schema }));
|
||||
const track = toGeminiDeclarations(fullTools, {
|
||||
mode: 'openapi',
|
||||
}).find((tool) => tool.name === 'track_entity');
|
||||
assert.deepEqual(
|
||||
track.parameters.anyOf.map((branch) => ({
|
||||
type: branch.type,
|
||||
required: branch.required,
|
||||
})),
|
||||
[
|
||||
{ type: 'object', required: ['query'] },
|
||||
{ type: 'object', required: ['referent'] },
|
||||
],
|
||||
);
|
||||
assert.equal(track.parameters.properties.query.pattern, '\\S');
|
||||
assert.deepEqual(
|
||||
track.parameters.properties.referent.enum,
|
||||
[-1, 1, 2, 3, 4, 5],
|
||||
);
|
||||
});
|
||||
|
||||
test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empty parameter objects', () => {
|
||||
const json = toGeminiDeclarations(GEV_REALTIME_TOOLS);
|
||||
assert.deepEqual(json[0].parametersJsonSchema, GEV_REALTIME_TOOLS[0].parameters);
|
||||
const openapi = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: 'openapi' });
|
||||
const globe = openapi.find((d) => d.name === 'zoom_to_globe');
|
||||
assert.equal('parameters' in globe, false);
|
||||
const serialized = JSON.stringify(openapi);
|
||||
assert.equal(serialized.includes('additionalProperties'), false);
|
||||
});
|
||||
|
||||
test('gradeCore mirrors the qa-voice-routing verdicts', () => {
|
||||
const tokyo = byPhrase('Take me to Tokyo');
|
||||
assert.equal(gradeCore(tokyo, [{ name: 'fly_to_location', args: { locationId: 'tokyo' } }]).ok, true);
|
||||
assert.equal(gradeCore(tokyo, []).ok, false);
|
||||
|
||||
const chat = byPhrase('How is your evening going?');
|
||||
const fp = gradeCore(chat, [{ name: 'get_entity_context', args: {} }]);
|
||||
assert.equal(fp.ok, false);
|
||||
assert.equal(fp.falsePositive, true);
|
||||
assert.equal(gradeCore(chat, []).ok, true);
|
||||
|
||||
const multi = byPhrase('Switch to night vision and turn on the flights layer');
|
||||
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok, false);
|
||||
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }, { name: 'set_layer_visibility', args: {} }]).ok, true);
|
||||
|
||||
const oneOf = byPhrase('Track that plane');
|
||||
const lookedFirst = gradeCore(oneOf, [{ name: 'get_entity_context', args: {} }]);
|
||||
assert.equal(lookedFirst.ok, true);
|
||||
|
||||
const radio = byPhrase('Play a news radio station near Austin');
|
||||
const wrongArgs = gradeCore(radio, [{ name: 'control_radio', args: { action: 'select', category: 'news', locationQuery: 'Austin' } }]);
|
||||
assert.equal(wrongArgs.toolOk, true);
|
||||
assert.equal(wrongArgs.argsOk, false);
|
||||
assert.equal(wrongArgs.ok, false);
|
||||
|
||||
const globeRadio = byPhrase('Go to full planet view and then turn on the radio');
|
||||
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'enable' } }]).ok, false);
|
||||
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'play' } }]).ok, true);
|
||||
|
||||
const volume = byPhrase('Set the radio volume to thirty percent');
|
||||
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } }]).ok, true);
|
||||
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } }]).ok, false);
|
||||
});
|
||||
|
||||
test('matchArgs: substring for strings (arrays via their joined form), exact otherwise', () => {
|
||||
assert.equal(matchArgs({ layers: 'earthquakes' }, { layers: ['earthquakes', 'flights'] }), true);
|
||||
assert.equal(matchArgs({ enabled: true }, { enabled: 'true' }), false);
|
||||
assert.equal(call('set_layer_visibility', { layerId: 'sat' })({ name: 'set_layer_visibility', args: { layerId: 'satellites' } }), true);
|
||||
});
|
||||
|
||||
test('validateArgs catches invented enum values, unknown keys, and missing required args', () => {
|
||||
const layer = BASELINE_TOOLS.find((t) => t.name === 'set_layer_visibility').parameters;
|
||||
assert.deepEqual(validateArgs(layer, { layerId: 'flights', enabled: true }), []);
|
||||
assert.match(validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0], /not one of/);
|
||||
assert.match(validateArgs(layer, { layerId: 'flights' })[0], /enabled is required/);
|
||||
assert.match(validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0], /not a known argument/);
|
||||
assert.equal(stubResult('coverage', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, false);
|
||||
assert.equal(stubResult('core', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, true);
|
||||
});
|
||||
|
||||
test('gradeCoverage sorts turns into the four verdicts', () => {
|
||||
const tools = BASELINE_TOOLS;
|
||||
const radar = coverage('weather-radar-on');
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }], 'Weather radar on.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [], "I can't turn on weather radar — there's no radar layer I can control.", tools).verdict, 'honest-refusal');
|
||||
assert.equal(gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'set_visual_style', args: { style: 'thermal' } }], 'Done.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'zoom_to_globe', args: {} }], 'Here is the globe.', tools).verdict, 'wrong-tool');
|
||||
|
||||
const imagery = coverage('recent-imagery');
|
||||
assert.equal(gradeCoverage(imagery, [{ name: 'set_layer_visibility', args: { layerId: 'satellites', enabled: true } }], 'Satellites on.', tools).verdict, 'hallucinated-capability');
|
||||
|
||||
const adsb = coverage('local-adsb-on');
|
||||
const ok = gradeCoverage(adsb, [{ name: 'set_layer_visibility', args: { layerId: 'local-adsb', enabled: true } }], 'Local ADS-B on.', tools);
|
||||
assert.equal(ok.verdict, 'correct');
|
||||
assert.equal(ok.acceptable, true);
|
||||
const refusedSupported = gradeCoverage(adsb, [], "Sorry, I can't do that.", tools);
|
||||
assert.equal(refusedSupported.verdict, 'honest-refusal');
|
||||
assert.equal(refusedSupported.acceptable, false);
|
||||
|
||||
const bikes = coverage('bikes-near-me');
|
||||
assert.equal(gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(bikes, [{ name: 'set_layer_visibility', args: { layerId: 'bikeshare', enabled: true } }], "I've turned on bike share, but I can't count available bikes.", tools).verdict, 'honest-refusal');
|
||||
|
||||
const directions = coverage('driving-directions');
|
||||
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'route', mode: 'driving' }] } }], 'Route drawn.', tools).verdict, 'correct');
|
||||
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }], 'Arrow.', tools).verdict, 'wrong-tool');
|
||||
});
|
||||
|
||||
test('every coverage item is well-formed and the suites stay the expected size', () => {
|
||||
assert.equal(COVERAGE_PROBE.length, 24);
|
||||
assert.ok(CORE_PHRASES.length >= 60);
|
||||
for (const item of COVERAGE_PROBE) {
|
||||
assert.equal(typeof item.utterance, 'string');
|
||||
assert.equal(typeof item.supported, 'boolean');
|
||||
if (item.supported) assert.ok(item.correct?.length, `${item.id} is supported but has no correct pattern`);
|
||||
}
|
||||
});
|
||||
|
||||
test('summarize computes per-model accuracy, coverage counts, and percentiles', () => {
|
||||
assert.equal(percentile([5, 1, 3, 2, 4], 50), 3);
|
||||
assert.equal(percentile([5, 1, 3, 2, 4], 90), 5);
|
||||
const rows = [
|
||||
{ type: 'run', provider: 'p', model: 'm', settings: { t: 1 } },
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 1, phrase: 'a', grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true }, tFirstCallMs: 100, tDoneMs: 200, costUsd: 0.5, calls: [] },
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 2, phrase: 'a', grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false }, tFirstCallMs: null, tDoneMs: 300, costUsd: 0.25, calls: [] },
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'coverage', rep: 1, phraseId: 'x', phrase: 'x', supported: false, grade: { verdict: 'honest-refusal', acceptable: true }, tFirstCallMs: null, tDoneMs: 50, costUsd: 0.25, calls: [] },
|
||||
];
|
||||
const s = summarize(rows);
|
||||
const m = s.models[0];
|
||||
assert.equal(m.core.accuracyPct, 50);
|
||||
assert.deepEqual(m.core.perRepAccuracyPct, [100, 0]);
|
||||
assert.equal(m.coverage.counts['honest-refusal'], 1);
|
||||
assert.equal(m.coverage.acceptablePct, 100);
|
||||
assert.equal(m.latencyMs.toolCallP50, 100);
|
||||
assert.equal(s.cloudSpendUsd, 1);
|
||||
});
|
||||
|
||||
test('typed argument checks feed full-call accuracy without changing the legacy verdict', () => {
|
||||
const tokyo = byPhrase('Zoom in a bit');
|
||||
const wrongAmount = gradeCore(tokyo, [{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } }]);
|
||||
assert.equal(wrongAmount.ok, true);
|
||||
assert.equal(wrongAmount.typedArgsOk, false);
|
||||
assert.equal(wrongAmount.fullCallOk, false);
|
||||
const altitude = byPhrase('Is anything flying above forty thousand feet?');
|
||||
const inFeet = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }] } }]);
|
||||
assert.equal(inFeet.fullCallOk, false);
|
||||
const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]);
|
||||
assert.equal(inMeters.fullCallOk, true);
|
||||
assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true);
|
||||
});
|
||||
|
||||
test('gradeDialogue scores turn 2 in the context of turn 1', () => {
|
||||
const d = DIALOGUES.find((x) => x.id === 'track-then-stop');
|
||||
const good = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }, { name: 'stop_tracking', args: {} }]]);
|
||||
assert.equal(good.ok, true);
|
||||
assert.equal(good.turn1Ok, true);
|
||||
const bad = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }]]);
|
||||
assert.equal(bad.ok, false);
|
||||
const thanks = DIALOGUES.find((x) => x.id === 'flights-then-thanks');
|
||||
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []]).ok, true);
|
||||
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], [{ name: 'get_entity_context', args: {} }]]).ok, false);
|
||||
assert.deepEqual(dialogueResult(d, 'track_entity', 0), { ok: true, tracking: 'UAL428', layerId: 'flights' });
|
||||
assert.equal(dialogueResult(d, 'fly_to_location', 1), null);
|
||||
assert.equal(DIALOGUES.every((x) => x.turns.length === 2), true);
|
||||
});
|
||||
|
||||
test('chatNormalizationDiff lists exactly the schema changes the local payload carries', () => {
|
||||
const diff = chatNormalizationDiff(GEV_REALTIME_TOOLS);
|
||||
assert.deepEqual(diff, [
|
||||
{ tool: 'analyst_query', path: 'analyst_query.filters[].value', from: '(untyped)', to: ['string', 'number', 'boolean'] },
|
||||
]);
|
||||
});
|
||||
|
||||
test('geminiUsageCostUsd bills text/audio/thought tokens at their own rates', () => {
|
||||
const usd = geminiUsageCostUsd('gemini-3.8-live', {
|
||||
promptTokenCount: 1_000_000,
|
||||
promptTokensDetails: [{ modality: 'TEXT', tokenCount: 1_000_000 }],
|
||||
responseTokenCount: 1_000_000,
|
||||
responseTokensDetails: [{ modality: 'AUDIO', tokenCount: 1_000_000 }],
|
||||
thoughtsTokenCount: 1_000_000,
|
||||
});
|
||||
assert.equal(usd, 0.75 + 12 + 4.5);
|
||||
});
|
||||
|
||||
test('coverage v2 grades the same turn against the current tool surface', () => {
|
||||
const radar = coverage('weather-radar-on');
|
||||
const turn = [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }];
|
||||
assert.equal(gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(coverageRubric(radar, 'v2'), turn, 'Radar on.', GEV_REALTIME_TOOLS).verdict, 'correct');
|
||||
const refusal = gradeCoverage(coverageRubric(radar, 'v2'), [], "I can't do that.", GEV_REALTIME_TOOLS);
|
||||
assert.equal(refusal.acceptable, false);
|
||||
assert.equal(coverageRubric(radar, 'v1'), radar);
|
||||
assert.equal(COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'), true);
|
||||
});
|
||||
|
||||
test('ROUTING_PHRASES is the core table; capability items are well-formed', () => {
|
||||
assert.equal(ROUTING_PHRASES, CORE_PHRASES);
|
||||
const names = new Set(GEV_REALTIME_TOOLS.map((t) => t.name));
|
||||
for (const p of CAPABILITY_EXTRAS.filter((c) => !c.turns)) {
|
||||
const expected = typeof p.expect === 'string' ? [p.expect] : p.expect.oneOf;
|
||||
for (const n of expected) assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
|
||||
}
|
||||
const tellMe = CAPABILITY_EXTRAS.find((c) => c.phrase === 'Tell me about this');
|
||||
assert.equal(tellMe.context.type, 'pointer_context');
|
||||
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'pointer' } }]).fullCallOk, true);
|
||||
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }]).fullCallOk, false);
|
||||
const second = CAPABILITY_EXTRAS.find((c) => c.id === 'flights-then-second');
|
||||
const count = { name: 'analyst_query', args: { layers: ['flights'], scope: { kind: 'view' } } };
|
||||
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { referent: 2 } }]]).ok, true);
|
||||
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { query: 'nearest' } }]]).ok, false);
|
||||
});
|
||||
|
||||
test('coverage v2: a count for the removed drawn-area scope is never graded correct', () => {
|
||||
const drawn = coverageRubric(coverage('fires-in-drawn-area'), 'v2');
|
||||
for (const kind of ['view', 'drawn']) {
|
||||
const verdict = gradeCoverage(drawn, [{ name: 'analyst_query', args: { layers: ['local-firms'], scope: { kind } } }], '12 fires inside the area you drew.');
|
||||
assert.notEqual(verdict.verdict, 'correct');
|
||||
assert.equal(verdict.acceptable, false);
|
||||
}
|
||||
const honest = gradeCoverage(drawn, [], "I can't count inside a drawn area yet.");
|
||||
assert.equal(honest.verdict, 'honest-refusal');
|
||||
assert.equal(honest.acceptable, true);
|
||||
});
|
||||
|
||||
|
||||
test('coverage validates all calls before accepting a matching capability', () => {
|
||||
const item = coverageRubric(coverage('transit-on'), 'v2');
|
||||
const valid = { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } };
|
||||
const invalid = { name: 'set_layer_visibility', args: { layerId: 'transit-invented', enabled: true } };
|
||||
for (const calls of [[invalid], [valid, invalid]]) {
|
||||
const result = gradeCoverage(item, calls, 'Transit on.', GEV_REALTIME_TOOLS);
|
||||
assert.equal(result.verdict, 'hallucinated-capability');
|
||||
assert.equal(result.acceptable, false);
|
||||
}
|
||||
assert.equal(gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict, 'correct');
|
||||
});
|
||||
Reference in new issue
Block a user