mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
refactor(voice): isolate gemini live provider core
This commit is contained in:
commit
10916227cb
655 files changed
+45946
-26483
No files matched your search
@@ -0,0 +1,319 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import {
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
writeFile,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
} from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
FRAGMENT_DIR,
|
||||
STAGING_DIR,
|
||||
assembleChangelog,
|
||||
assembleFragments,
|
||||
orderFragmentNames,
|
||||
recoverInterruptedBatch,
|
||||
} from '../../scripts/assemble-changelog.mjs';
|
||||
|
||||
const EXISTING =
|
||||
'# Changelog\n\n- An entry that was already released (someone, #1).\n';
|
||||
|
||||
const repositoryRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
|
||||
async function fixture(t, fragments = {}) {
|
||||
const root = await mkdtemp(path.join(tmpdir(), 'gev-changelog-'));
|
||||
t.after(() => rm(root, { recursive: true, force: true }));
|
||||
await mkdir(path.join(root, FRAGMENT_DIR));
|
||||
await writeFile(
|
||||
path.join(root, FRAGMENT_DIR, 'README.md'),
|
||||
'# How to add one\n',
|
||||
);
|
||||
await writeFile(path.join(root, 'CHANGELOG.md'), EXISTING);
|
||||
for (const [name, body] of Object.entries(fragments)) {
|
||||
await writeFile(path.join(root, FRAGMENT_DIR, name), body);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
test('the batch reads newest pull request first', () => {
|
||||
assert.deepEqual(orderFragmentNames(['7-b.md', '112-c.md', '19-a.md']), [
|
||||
'112-c.md',
|
||||
'19-a.md',
|
||||
'7-b.md',
|
||||
]);
|
||||
});
|
||||
|
||||
test('the directory README is documentation, never an entry', () => {
|
||||
assert.deepEqual(orderFragmentNames(['README.md', '5-a.md']), ['5-a.md']);
|
||||
});
|
||||
|
||||
test('two fragments claiming one pull request fail loudly, not silently', () => {
|
||||
assert.throws(
|
||||
() => orderFragmentNames(['5-first.md', '5-second.md']),
|
||||
/Two fragments claim pull request 5/,
|
||||
);
|
||||
});
|
||||
|
||||
test('a name that is not <pull request>-<slug>.md is refused', () => {
|
||||
for (const name of ['notes.md', '5_underscore.md', '5-Caps.md', '5-a.txt']) {
|
||||
assert.throws(() => orderFragmentNames([name]), /Unexpected file/, name);
|
||||
}
|
||||
});
|
||||
|
||||
test('fragment text is spliced in unchanged, above the existing entries', () => {
|
||||
// Wrapped, indented and punctuated the way a real entry is: the continuation
|
||||
// indent and the blank line inside the entry are the parts a reflow would eat.
|
||||
const entry = [
|
||||
'- A new thing, with `code` and a — dash. The second line is indented the',
|
||||
' way the surrounding entries wrap, and it keeps its two spaces.',
|
||||
'',
|
||||
' A second paragraph, because some entries have one (someone, #9).',
|
||||
].join('\n');
|
||||
const result = assembleChangelog(EXISTING, [`${entry}\n\n\n`]);
|
||||
assert.equal(
|
||||
result,
|
||||
`# Changelog\n\n${entry}\n\n- An entry that was already released (someone, #1).\n`,
|
||||
);
|
||||
// The entry survives byte for byte: drawTool.test.mjs reads this file as
|
||||
// prose and slices a fixed window, so reflowing would move text across it.
|
||||
assert.ok(result.includes(entry));
|
||||
});
|
||||
|
||||
test('a CHANGELOG without the heading is refused rather than prepended to', () => {
|
||||
assert.throws(
|
||||
() => assembleChangelog('- orphan entry\n', ['- new\n']),
|
||||
/must start with "# Changelog"/,
|
||||
);
|
||||
});
|
||||
|
||||
test('assembling consumes the fragments and leaves the README', async (t) => {
|
||||
const root = await fixture(t, {
|
||||
'20-second.md': '- Second (someone, #20).',
|
||||
'100-first.md': '- First (someone, #100).',
|
||||
});
|
||||
const { consumed } = await assembleFragments(root, '--write');
|
||||
assert.deepEqual(consumed, ['100-first.md', '20-second.md']);
|
||||
const changelog = await readFile(path.join(root, 'CHANGELOG.md'), 'utf8');
|
||||
assert.match(
|
||||
changelog,
|
||||
/# Changelog\n\n- First \(someone, #100\)\.\n\n- Second \(someone, #20\)\.\n\n- An entry that was already released/,
|
||||
);
|
||||
assert.deepEqual(await readdir(path.join(root, FRAGMENT_DIR)), ['README.md']);
|
||||
});
|
||||
|
||||
test('a rejected batch leaves the directory and the file untouched', async (t) => {
|
||||
const root = await fixture(t, {
|
||||
'8-first.md': '- First (someone, #8).',
|
||||
'8-again.md': '- Again (someone, #8).',
|
||||
});
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write'),
|
||||
/Two fragments claim pull request 8/,
|
||||
);
|
||||
assert.equal(
|
||||
await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
EXISTING,
|
||||
);
|
||||
assert.deepEqual((await readdir(path.join(root, FRAGMENT_DIR))).sort(), [
|
||||
'8-again.md',
|
||||
'8-first.md',
|
||||
'README.md',
|
||||
]);
|
||||
});
|
||||
|
||||
test('check mode changes nothing on disk', async (t) => {
|
||||
const root = await fixture(t, { '3-a.md': '- Something (someone, #3).' });
|
||||
const { consumed } = await assembleFragments(root, '--check');
|
||||
assert.deepEqual(consumed, ['3-a.md']);
|
||||
assert.equal(
|
||||
await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
EXISTING,
|
||||
);
|
||||
assert.deepEqual((await readdir(path.join(root, FRAGMENT_DIR))).sort(), [
|
||||
'3-a.md',
|
||||
'README.md',
|
||||
]);
|
||||
});
|
||||
|
||||
test('an empty directory is a no-op, so a release can always run it', async (t) => {
|
||||
const root = await fixture(t);
|
||||
const { consumed } = await assembleFragments(root, '--write');
|
||||
assert.deepEqual(consumed, []);
|
||||
assert.equal(
|
||||
await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
EXISTING,
|
||||
);
|
||||
});
|
||||
|
||||
test('a mode that is neither check nor write is refused', async (t) => {
|
||||
const root = await fixture(t);
|
||||
await assert.rejects(assembleFragments(root, '--force'), /--check\|--write/);
|
||||
});
|
||||
|
||||
// Failure injection. Every filesystem call the write path makes is a point
|
||||
// where a release can die: a full disk, a permissions error, a killed process.
|
||||
|
||||
const IO_CALLS = ['readdir', 'readFile', 'mkdir', 'writeFile', 'rename', 'rm'];
|
||||
const BATCH = {
|
||||
'20-second.md': '- Second (someone, #20).',
|
||||
'100-first.md': '- First (someone, #100).',
|
||||
'7-third.md': '- Third (someone, #7).',
|
||||
};
|
||||
|
||||
/**
|
||||
* The real filesystem, except that `call` throws the `nth` time it runs. With
|
||||
* `crash`, every later call throws as well, the way a killed process makes no
|
||||
* further calls: the in-process rollback cannot run either, so the next
|
||||
* invocation is the only thing that can recover.
|
||||
*/
|
||||
function failing(call, nth, { crash = false } = {}) {
|
||||
let seen = 0;
|
||||
let dead = false;
|
||||
const io = { ...fs };
|
||||
for (const name of IO_CALLS) {
|
||||
io[name] = async (...args) => {
|
||||
if (dead) throw new Error('process already gone');
|
||||
if (name === call && ++seen === nth) {
|
||||
dead = crash;
|
||||
throw new Error(`injected ${call} failure #${nth}`);
|
||||
}
|
||||
return fs[name](...args);
|
||||
};
|
||||
}
|
||||
return io;
|
||||
}
|
||||
|
||||
/** How many times one uninterrupted `--write` makes each filesystem call. */
|
||||
async function countCalls(t) {
|
||||
const root = await fixture(t, BATCH);
|
||||
const counts = Object.fromEntries(IO_CALLS.map((name) => [name, 0]));
|
||||
const io = { ...fs };
|
||||
for (const name of IO_CALLS) {
|
||||
io[name] = (...args) => {
|
||||
counts[name] += 1;
|
||||
return fs[name](...args);
|
||||
};
|
||||
}
|
||||
await assembleFragments(root, '--write', io);
|
||||
return {
|
||||
counts,
|
||||
expected: await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
};
|
||||
}
|
||||
|
||||
test('a write that fails at any call never lets a retry duplicate or lose an entry', async (t) => {
|
||||
const { counts, expected } = await countCalls(t);
|
||||
// The write path itself: one mkdir, one writeFile, a rename per fragment plus
|
||||
// the commit rename, and the cleanup rm. Fewer would mean a step went missing.
|
||||
assert.ok(counts.rename >= Object.keys(BATCH).length + 1);
|
||||
for (const crash of [false, true]) {
|
||||
for (const call of IO_CALLS) {
|
||||
for (let nth = 1; nth <= counts[call]; nth += 1) {
|
||||
const label = `${crash ? 'crash' : 'error'} at ${call} #${nth}`;
|
||||
const root = await fixture(t, BATCH);
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write', failing(call, nth, { crash })),
|
||||
/injected|already gone/,
|
||||
label,
|
||||
);
|
||||
await assembleFragments(root, '--write');
|
||||
const changelog = await readFile(
|
||||
path.join(root, 'CHANGELOG.md'),
|
||||
'utf8',
|
||||
);
|
||||
assert.equal(changelog, expected, label);
|
||||
for (const pr of [100, 20, 7, 1]) {
|
||||
assert.equal(
|
||||
changelog.split(`(someone, #${pr}).`).length - 1,
|
||||
1,
|
||||
`${label}: #${pr} appears exactly once`,
|
||||
);
|
||||
}
|
||||
assert.deepEqual(
|
||||
await readdir(path.join(root, FRAGMENT_DIR)),
|
||||
['README.md'],
|
||||
label,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('an error before the commit rolls the batch back in the same run', async (t) => {
|
||||
const root = await fixture(t, BATCH);
|
||||
// The second rename has already moved one fragment into the staging directory.
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write', failing('rename', 2)),
|
||||
/injected rename failure #2/,
|
||||
);
|
||||
assert.equal(
|
||||
await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
EXISTING,
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await readdir(path.join(root, FRAGMENT_DIR))).sort(),
|
||||
[...Object.keys(BATCH), 'README.md'].sort(),
|
||||
);
|
||||
});
|
||||
|
||||
test('check mode reports a batch an interrupted release left staged', async (t) => {
|
||||
const root = await fixture(t, BATCH);
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write', failing('rename', 2, { crash: true })),
|
||||
);
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--check'),
|
||||
new RegExp(`${STAGING_DIR}/ holds a batch an interrupted release`),
|
||||
);
|
||||
assert.equal(await recoverInterruptedBatch(root), 'rolled-back');
|
||||
assert.equal(await recoverInterruptedBatch(root), 'none');
|
||||
});
|
||||
|
||||
test('recovery refuses to roll a fragment back over a file of the same name', async (t) => {
|
||||
const root = await fixture(t, BATCH);
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write', failing('rename', 2, { crash: true })),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(root, FRAGMENT_DIR, '100-first.md'),
|
||||
'- Rewritten.',
|
||||
);
|
||||
await assert.rejects(
|
||||
assembleFragments(root, '--write'),
|
||||
/Cannot roll back .*100-first\.md/,
|
||||
);
|
||||
assert.equal(
|
||||
await readFile(path.join(root, 'CHANGELOG.md'), 'utf8'),
|
||||
EXISTING,
|
||||
);
|
||||
});
|
||||
|
||||
// The two below run against the repository itself, so a fragment that the
|
||||
// release could not assemble fails `npm test` in the pull request that adds it
|
||||
// rather than at release time, when its author has moved on.
|
||||
|
||||
test('every pending fragment in the repository can be assembled', async () => {
|
||||
const names = await readdir(path.join(repositoryRoot, FRAGMENT_DIR));
|
||||
assert.ok(
|
||||
names.includes('README.md'),
|
||||
`${FRAGMENT_DIR}/README.md documents the convention and keeps the directory tracked`,
|
||||
);
|
||||
// Throws on a name that is not <pull request>-<slug>.md, and on two fragments
|
||||
// claiming one pull request number. Every other file is an entry.
|
||||
assert.equal(orderFragmentNames(names).length, names.length - 1);
|
||||
});
|
||||
|
||||
test('CHANGELOG.md still carries the heading a release splices under', async () => {
|
||||
const changelog = await readFile(
|
||||
path.join(repositoryRoot, 'CHANGELOG.md'),
|
||||
'utf8',
|
||||
);
|
||||
assert.ok(changelog.startsWith('# Changelog\n'));
|
||||
});
|
||||
@@ -7,9 +7,10 @@ import { firmsProxy } from 'gods-eye-view/server/providers/firms';
|
||||
import { gbfsProxy } from 'gods-eye-view/server/providers/gbfs';
|
||||
import { localProviderPlugins } from '../../server/providers/local.js';
|
||||
|
||||
function install(plugin) {
|
||||
function install(plugin, hook = 'configureServer') {
|
||||
const routes = new Map();
|
||||
plugin.configureServer({
|
||||
assert.equal(typeof plugin[hook], 'function', `${hook} is available`);
|
||||
plugin[hook]({
|
||||
middlewares: {
|
||||
use(route, handler) {
|
||||
routes.set(route, handler);
|
||||
@@ -17,9 +18,16 @@ function install(plugin) {
|
||||
},
|
||||
});
|
||||
assert.equal(routes.size, 1);
|
||||
return async (url = '/', method = 'GET') => {
|
||||
return async (url = '/', method = 'GET', headers = {}) => {
|
||||
const res = {
|
||||
headersSent: false,
|
||||
headers: {},
|
||||
set statusCode(status) {
|
||||
this.status = status;
|
||||
},
|
||||
setHeader(name, value) {
|
||||
this.headers[name.toLowerCase()] = value;
|
||||
},
|
||||
writeHead(status, headers) {
|
||||
Object.assign(this, { status, headers, headersSent: true });
|
||||
},
|
||||
@@ -27,7 +35,7 @@ function install(plugin) {
|
||||
this.body = body;
|
||||
},
|
||||
};
|
||||
await [...routes.values()][0]({ url, method }, res);
|
||||
await [...routes.values()][0]({ url, method, headers, socket: {} }, res);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
@@ -191,6 +199,64 @@ test('traffic middleware preserves keyless mode, caching, stale budget fallback
|
||||
assert.equal(calls, 2);
|
||||
});
|
||||
|
||||
test('traffic refuses cross-site requests in dev and preview before spending the TomTom budget', async (t) => {
|
||||
isolate(t, { TOMTOM_API_KEY: 'fixture-key', TOMTOM_DAILY_TILE_BUDGET: '3' });
|
||||
let calls = 0;
|
||||
t.mock.method(globalThis, 'fetch', async () => {
|
||||
calls++;
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
});
|
||||
const host = 'localhost:5173';
|
||||
const refused = [
|
||||
{ host, 'sec-fetch-site': 'cross-site' }, // <img src> from any web page
|
||||
{ host, 'sec-fetch-site': 'same-site' },
|
||||
{ host, origin: 'https://evil.example' }, // fetch from another page
|
||||
{ host, origin: 'null' }, // sandboxed frame
|
||||
{ host, 'x-forwarded-for': '203.0.113.9' }, // through a proxy
|
||||
];
|
||||
for (const [mode, hook] of [
|
||||
['dev', 'configureServer'],
|
||||
['preview', 'configurePreviewServer'],
|
||||
]) {
|
||||
const request = install(tomtomProxy(), hook);
|
||||
// Each refused shape asks for distinct uncached tiles: without the gate
|
||||
// every one would be a billed attempt counted against the daily budget.
|
||||
let x = 0;
|
||||
for (const headers of refused)
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const res = await request(`/flow/16/${x++}/0.pbf`, 'GET', headers);
|
||||
assert.equal(res.status, 403, `${mode}: ${JSON.stringify(headers)}`);
|
||||
}
|
||||
assert.equal(
|
||||
(await request('/status', 'GET', refused[0])).status,
|
||||
403,
|
||||
`${mode}: status is gated too`,
|
||||
);
|
||||
assert.equal(
|
||||
calls,
|
||||
mode === 'dev' ? 0 : 2,
|
||||
`${mode}: no refused upstream work`,
|
||||
);
|
||||
const app = { host, 'sec-fetch-site': 'same-origin' };
|
||||
assert.equal(json(await request('/status', 'GET', app)).dailyCount, 0);
|
||||
// The app itself and loopback tools (no browser headers) still get tiles.
|
||||
const tile = await request('/flow/8/1/1.pbf', 'GET', app);
|
||||
assert.equal(tile.status, 200);
|
||||
assert.equal(tile.headers['x-tomtom-cache'], 'MISS');
|
||||
assert.equal((await request('/flow/8/2/1.pbf')).status, 200);
|
||||
assert.equal(
|
||||
(
|
||||
await request('/flow/8/1/1.pbf', 'GET', {
|
||||
host,
|
||||
origin: `http://${host}`,
|
||||
})
|
||||
).headers['x-tomtom-cache'],
|
||||
'HIT',
|
||||
);
|
||||
assert.equal(calls, mode === 'dev' ? 2 : 4);
|
||||
}
|
||||
});
|
||||
|
||||
test('traffic middleware answers 502 to an oversized tile without caching it', async (t) => {
|
||||
isolate(t, { TOMTOM_API_KEY: 'fixture-key', TOMTOM_DAILY_TILE_BUDGET: '' });
|
||||
const bodies = [];
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import { launchLibraryRecentUrl } from '../data/spaceProviderRequests.js';
|
||||
import {
|
||||
launchLibraryRequestHeaders,
|
||||
rocketLaunchesProxy,
|
||||
} from '../../server/providers/space/launch-library.js';
|
||||
import { keySetupStatus } from '../keySetupCore.mjs';
|
||||
import { initKeySetup } from '../keySetup.js';
|
||||
|
||||
test('LL2 requests from the same UTC minute share a cache key without mutating callers', () => {
|
||||
const first = new Date('2026-10-17T16:20:00.001Z');
|
||||
const last = new Date('2026-10-17T16:20:59.999Z');
|
||||
assert.equal(
|
||||
launchLibraryRecentUrl(first).href,
|
||||
launchLibraryRecentUrl(last).href,
|
||||
);
|
||||
assert.equal(first.toISOString(), '2026-10-17T16:20:00.001Z');
|
||||
assert.equal(last.toISOString(), '2026-10-17T16:20:59.999Z');
|
||||
});
|
||||
|
||||
for (const [date, end] of [
|
||||
['2026-10-17T16:20:00.001Z', '2026-10-17T16:20:00.000Z'],
|
||||
['2026-10-17T16:20:59.999Z', '2026-10-17T16:20:00.000Z'],
|
||||
['2026-10-17T16:21:00.000Z', '2026-10-17T16:21:00.000Z'],
|
||||
['2026-01-01T00:00:00.001Z', '2026-01-01T00:00:00.000Z'],
|
||||
]) {
|
||||
test(`LL2 bounds retain the 30-day UTC window at ${date}`, () => {
|
||||
const url = launchLibraryRecentUrl(new Date(date));
|
||||
assert.equal(url.origin, 'https://ll.thespacedevs.com');
|
||||
assert.equal(url.pathname, '/2.3.0/launches/');
|
||||
assert.equal(url.searchParams.get('net__lte'), end);
|
||||
assert.equal(
|
||||
Date.parse(end) - Date.parse(url.searchParams.get('net__gte')),
|
||||
30 * 86400000,
|
||||
);
|
||||
assert.equal(url.searchParams.get('limit'), '100');
|
||||
assert.equal(url.searchParams.get('mode'), 'detailed');
|
||||
});
|
||||
}
|
||||
|
||||
test('LL2 headers identify the app and preserve optional server-only authorization', () => {
|
||||
const publicHeaders = launchLibraryRequestHeaders('');
|
||||
assert.match(publicHeaders['User-Agent'], /^gods-eye-view.*\/1\.0 /);
|
||||
assert.ok(
|
||||
publicHeaders['User-Agent'].includes(
|
||||
'https://github.com/bilawalsidhu/gods-eye-view',
|
||||
),
|
||||
);
|
||||
assert.equal(publicHeaders.Accept, 'application/json');
|
||||
assert.equal(publicHeaders.Authorization, undefined);
|
||||
const paidHeaders = launchLibraryRequestHeaders(' owned-fixture-token ');
|
||||
assert.equal(paidHeaders.Authorization, 'Token owned-fixture-token');
|
||||
assert.equal(paidHeaders['User-Agent'], publicHeaders['User-Agent']);
|
||||
});
|
||||
|
||||
test('actual setup rendering distinguishes optional paid LL2 from unrelated free keys', async () => {
|
||||
const created = [];
|
||||
const node = () => {
|
||||
const value = {
|
||||
dataset: {},
|
||||
children: [],
|
||||
append(...children) {
|
||||
this.children.push(...children);
|
||||
},
|
||||
querySelector() {
|
||||
return null;
|
||||
},
|
||||
querySelectorAll() {
|
||||
return [];
|
||||
},
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
setAttribute() {},
|
||||
remove() {},
|
||||
};
|
||||
created.push(value);
|
||||
return value;
|
||||
};
|
||||
const rows = node();
|
||||
const root = node();
|
||||
root.querySelector = (selector) =>
|
||||
selector === '[data-key-setup-rows]' ? rows : null;
|
||||
const chip = node();
|
||||
const status = keySetupStatus({});
|
||||
status.keys = status.keys.filter((key) =>
|
||||
['launch-library', 'mapillary'].includes(key.id),
|
||||
);
|
||||
const setup = await initKeySetup({
|
||||
documentRef: {
|
||||
getElementById: (id) => (id === 'key-setup-chip' ? chip : root),
|
||||
createElement: node,
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
},
|
||||
fetchImpl: async () => ({ ok: true, json: async () => status }),
|
||||
});
|
||||
assert.ok(setup);
|
||||
try {
|
||||
const ll2 = rows.children.find(
|
||||
(row) => row.dataset.keyId === 'launch-library',
|
||||
);
|
||||
const free = rows.children.find((row) => row.dataset.keyId === 'mapillary');
|
||||
const tier = (row) =>
|
||||
row.children[0].children.find(
|
||||
(child) => child.className === 'key-setup-tier',
|
||||
);
|
||||
assert.match(tier(ll2).title, /optional paid token/i);
|
||||
assert.match(ll2.children[1].textContent, /leave blank/i);
|
||||
assert.match(ll2.children[1].textContent, /no key/i);
|
||||
assert.equal(tier(free).title, 'Free key — register, paste, done');
|
||||
const serialized = JSON.stringify(status);
|
||||
assert.ok(!serialized.includes('owned-fixture-token'));
|
||||
} finally {
|
||||
setup.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('real LL2 proxy sends cacheable headers and bounds, retaining its memory cache', async (t) => {
|
||||
for (const method of ['readFile', 'stat'])
|
||||
t.mock.method(fsp, method, async () => {
|
||||
throw new Error('owned fixture has no disk cache');
|
||||
});
|
||||
t.mock.method(fsp, 'mkdir', async () => {});
|
||||
t.mock.method(fsp, 'writeFile', async () => {});
|
||||
const calls = [];
|
||||
t.mock.method(globalThis, 'fetch', async (url, options) => {
|
||||
calls.push({ url: new URL(url), options });
|
||||
return new Response(
|
||||
JSON.stringify({ results: [{ id: 'owned-launch-fixture' }] }),
|
||||
);
|
||||
});
|
||||
let handler;
|
||||
rocketLaunchesProxy().configureServer({
|
||||
middlewares: {
|
||||
use(route, value) {
|
||||
assert.equal(route, '/api/launches');
|
||||
handler = value;
|
||||
},
|
||||
},
|
||||
});
|
||||
async function request() {
|
||||
const res = {
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end(body) {
|
||||
this.body = body;
|
||||
},
|
||||
};
|
||||
await handler({ method: 'GET' }, res);
|
||||
return res;
|
||||
}
|
||||
const first = await request();
|
||||
const second = await request();
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(second.body, first.body);
|
||||
assert.equal(second.headers['X-GEV-Cache'], 'HIT');
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0].options.headers['User-Agent'], /gods-eye-view/);
|
||||
assert.match(calls[0].url.searchParams.get('net__lte'), /:00\.000Z$/);
|
||||
assert.match(calls[0].url.searchParams.get('net__gte'), /:00\.000Z$/);
|
||||
});
|
||||
|
||||
for (const preview of [false, true]) {
|
||||
test(`LL2 proxy sends its identifying headers over real owned HTTP in ${preview ? 'preview' : 'development'}`, async (t) => {
|
||||
for (const method of ['readFile', 'stat'])
|
||||
t.mock.method(fsp, method, async () => {
|
||||
throw new Error('no owned cache');
|
||||
});
|
||||
t.mock.method(fsp, 'mkdir', async () => {});
|
||||
t.mock.method(fsp, 'writeFile', async () => {});
|
||||
const received = [];
|
||||
const upstream = http.createServer((req, res) => {
|
||||
received.push({
|
||||
url: new URL(req.url, 'http://fixture.invalid'),
|
||||
headers: req.headers,
|
||||
});
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ results: [{ id: 'owned-wire-fixture' }] }));
|
||||
});
|
||||
await new Promise((resolve) => upstream.listen(0, '127.0.0.1', resolve));
|
||||
const originalFetch = globalThis.fetch;
|
||||
t.mock.method(globalThis, 'fetch', (url, options) => {
|
||||
const parsed = new URL(url);
|
||||
assert.equal(parsed.origin, 'https://ll.thespacedevs.com');
|
||||
return originalFetch(
|
||||
`http://127.0.0.1:${upstream.address().port}${parsed.pathname}${parsed.search}`,
|
||||
options,
|
||||
);
|
||||
});
|
||||
let handler;
|
||||
const plugin = rocketLaunchesProxy();
|
||||
plugin[preview ? 'configurePreviewServer' : 'configureServer']({
|
||||
middlewares: {
|
||||
use(_route, value) {
|
||||
handler = value;
|
||||
},
|
||||
},
|
||||
});
|
||||
const proxy = http.createServer((req, res) => {
|
||||
void handler(req, res);
|
||||
});
|
||||
await new Promise((resolve) => proxy.listen(0, '127.0.0.1', resolve));
|
||||
try {
|
||||
const result = await originalFetch(
|
||||
`http://127.0.0.1:${proxy.address().port}/api/launches`,
|
||||
);
|
||||
assert.equal(result.status, 200);
|
||||
assert.deepEqual(await result.json(), {
|
||||
results: [{ id: 'owned-wire-fixture' }],
|
||||
});
|
||||
assert.equal(received.length, 1);
|
||||
assert.match(
|
||||
received[0].headers['user-agent'],
|
||||
/^gods-eye-view-launch-library-proxy\/1\.0 /,
|
||||
);
|
||||
assert.equal(received[0].headers.authorization, undefined);
|
||||
assert.match(received[0].url.searchParams.get('net__lte'), /:00\.000Z$/);
|
||||
} finally {
|
||||
proxy.closeAllConnections();
|
||||
upstream.closeAllConnections();
|
||||
await Promise.all([
|
||||
new Promise((resolve) => proxy.close(resolve)),
|
||||
new Promise((resolve) => upstream.close(resolve)),
|
||||
]);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1180,33 +1180,3 @@ test('the voice instructions send vessels, however heard, to the ships layer', (
|
||||
assert.match(text, /"vessels" is easily heard as "visuals"/);
|
||||
assert.match(text, /set_layer_visibility\{layerId:"ais-live-vessels"\}/);
|
||||
});
|
||||
|
||||
test('built-in token endpoint only advertises a provider dropdown when its host opts in', async (t) => {
|
||||
env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', '0');
|
||||
env(t, 'GEV_OPENAI_AUTH', undefined);
|
||||
const sent = [];
|
||||
for (const providerDropdown of [false, true]) {
|
||||
const realtime = {
|
||||
providerDropdown,
|
||||
resolveApiKey: () => 'fixture-key',
|
||||
fetchImpl: async (_url, options) => {
|
||||
sent.push(JSON.parse(options.body));
|
||||
return Response.json({ value: 'fixture-secret' });
|
||||
},
|
||||
};
|
||||
const response = await request(
|
||||
install(openAiRealtimeProxy({ realtime })).get('/api/realtime/token'),
|
||||
);
|
||||
assert.equal(response.status, 200);
|
||||
if (providerDropdown)
|
||||
assert.match(
|
||||
sent.at(-1).session.instructions,
|
||||
/Hover over the Voice panel and use the provider dropdown/,
|
||||
);
|
||||
else
|
||||
assert.doesNotMatch(
|
||||
sent.at(-1).session.instructions,
|
||||
/Hover over the Voice panel|provider dropdown/,
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -81,8 +81,8 @@ test('portable requests keep fixed origins, encode group data and preserve the 3
|
||||
assert.equal(url.origin, 'https://ll.thespacedevs.com');
|
||||
assert.equal(url.pathname, '/2.3.0/launches/');
|
||||
assert.deepEqual(Object.fromEntries(url.searchParams), {
|
||||
net__gte: '2026-01-30T12:34:56.000Z',
|
||||
net__lte: '2026-03-01T12:34:56.000Z',
|
||||
net__gte: '2026-01-30T12:34:00.000Z',
|
||||
net__lte: '2026-03-01T12:34:00.000Z',
|
||||
limit: '100',
|
||||
mode: 'detailed',
|
||||
});
|
||||
|
||||
@@ -88,18 +88,46 @@ test('root config retains existing named exports and standalone provider order',
|
||||
for (const [name, value] of Object.entries(providers))
|
||||
assert.equal(compatibility[name], value, name);
|
||||
const config = standaloneConfig({ mode: 'test' });
|
||||
const standaloneProviders = config.plugins
|
||||
.slice(3, -5)
|
||||
.map((plugin) => plugin.name);
|
||||
assert.equal(
|
||||
standaloneProviders.filter((name) => name === 'gemini-live-proxy').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
standaloneProviders.indexOf('gemini-live-proxy'),
|
||||
standaloneProviders.indexOf('openai-realtime-proxy') + 1,
|
||||
);
|
||||
assert.deepEqual(
|
||||
config.plugins.slice(3, -4).map((plugin) => plugin.name),
|
||||
standaloneProviders.filter((name) => name !== 'gemini-live-proxy'),
|
||||
providers.localProviderPlugins().map((plugin) => plugin.name),
|
||||
);
|
||||
assert.equal(config.plugins.at(-5).name, 'gev-key-setup');
|
||||
assert.equal(config.plugins.at(-6).name, 'gev-key-setup');
|
||||
// The local MCP route follows every provider and precedes the API fallback.
|
||||
assert.equal(config.plugins.at(-4).name, 'local-mcp');
|
||||
assert.equal(config.plugins.at(-3).name, 'api-not-found');
|
||||
assert.equal(config.plugins.at(-5).name, 'local-mcp');
|
||||
assert.equal(config.plugins.at(-4).name, 'api-not-found');
|
||||
// Config-only, so it sits after the provider slice rather than in it.
|
||||
assert.equal(config.plugins.at(-3).name, 'gev-unwatched-runtime-dirs');
|
||||
assert.equal(config.plugins.at(-2).name, 'embed-framing');
|
||||
assert.equal(config.plugins.at(-1).name, 'panel-build');
|
||||
});
|
||||
|
||||
test('directories the server writes while it runs are kept out of the dev watcher', () => {
|
||||
// Providers write cache files and logs here while the app runs; none of it is
|
||||
// source, so the dev watcher has no reason to stat it.
|
||||
const config = standaloneConfig({ mode: 'test' });
|
||||
const plugin = config.plugins.find(
|
||||
({ name }) => name === 'gev-unwatched-runtime-dirs',
|
||||
);
|
||||
// Dev only: preview and build have no watcher.
|
||||
assert.equal(plugin.apply, 'serve');
|
||||
assert.deepEqual(plugin.config().server.watch.ignored, [
|
||||
'**/.gev-cache/**',
|
||||
'**/.gev-logs/**',
|
||||
]);
|
||||
});
|
||||
|
||||
test('build export resolves in Node and has no browser fallback', async () => {
|
||||
const exported = await import('gods-eye-view/build/vite');
|
||||
assert.equal(exported.createBrowserViteConfig, createBrowserViteConfig);
|
||||
|
||||
+81
-505
@@ -18,13 +18,8 @@ import {
|
||||
call,
|
||||
} from '../../scripts/voice-bench/grade.mjs';
|
||||
import {
|
||||
CORE_PHRASES,
|
||||
COVERAGE_PROBE,
|
||||
DIALOGUES,
|
||||
ROUTING_PHRASES,
|
||||
CAPABILITY_EXTRAS,
|
||||
dialogueResult,
|
||||
coverageRubric,
|
||||
CORE_PHRASES, COVERAGE_PROBE, DIALOGUES, ROUTING_PHRASES, CAPABILITY_EXTRAS,
|
||||
dialogueResult, coverageRubric,
|
||||
} from '../../scripts/voice-bench/phrases.mjs';
|
||||
import { stubResult } from '../../scripts/voice-bench/stub.mjs';
|
||||
import { summarize, percentile } from '../../scripts/voice-bench/report.mjs';
|
||||
@@ -33,15 +28,7 @@ import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
// The 4b56d0e9 tool schema: the coverage v1 rubric grades against it.
|
||||
const BASELINE_TOOLS = JSON.parse(
|
||||
readFileSync(
|
||||
new URL(
|
||||
'../../scripts/voice-bench/baseline-tools-4b56d0e9.json',
|
||||
import.meta.url,
|
||||
),
|
||||
'utf8',
|
||||
),
|
||||
);
|
||||
const BASELINE_TOOLS = JSON.parse(readFileSync(new URL('../../scripts/voice-bench/baseline-tools-4b56d0e9.json', import.meta.url), 'utf8'));
|
||||
|
||||
const byPhrase = (text) => CORE_PHRASES.find((p) => p.phrase === text);
|
||||
const coverage = (id) => COVERAGE_PROBE.find((c) => c.id === id);
|
||||
@@ -59,38 +46,23 @@ test('toChatTools wraps every Realtime tool without changing names or descriptio
|
||||
|
||||
test('normalizeSchemaForChat types the untyped analyst_query filter value and nothing else', () => {
|
||||
const analyst = GEV_REALTIME_TOOLS.find((t) => t.name === 'analyst_query');
|
||||
assert.deepEqual(
|
||||
analyst.parameters.properties.filters.items.properties.value,
|
||||
{},
|
||||
);
|
||||
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
|
||||
const normalized = normalizeSchemaForChat(analyst.parameters);
|
||||
assert.deepEqual(normalized.properties.filters.items.properties.value.type, [
|
||||
'string',
|
||||
'number',
|
||||
'boolean',
|
||||
]);
|
||||
assert.deepEqual(normalized.properties.filters.items.properties.value.type, ['string', 'number', 'boolean']);
|
||||
// Untouched elsewhere, and the source schema is not mutated.
|
||||
assert.deepEqual(
|
||||
normalized.properties.layers,
|
||||
analyst.parameters.properties.layers,
|
||||
);
|
||||
assert.deepEqual(
|
||||
analyst.parameters.properties.filters.items.properties.value,
|
||||
{},
|
||||
);
|
||||
assert.deepEqual(normalized.properties.layers, analyst.parameters.properties.layers);
|
||||
assert.deepEqual(analyst.parameters.properties.filters.items.properties.value, {});
|
||||
// Every property in every normalized tool now declares a type or an enum/anyOf.
|
||||
const untyped = [];
|
||||
const visit = (node, where) => {
|
||||
if (!node || typeof node !== 'object') return;
|
||||
for (const [k, v] of Object.entries(node.properties || {})) {
|
||||
if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v))
|
||||
untyped.push(`${where}.${k}`);
|
||||
if (!['type', 'enum', 'anyOf', 'oneOf'].some((key) => key in v)) untyped.push(`${where}.${k}`);
|
||||
visit(v, `${where}.${k}`);
|
||||
}
|
||||
if (node.items) visit(node.items, `${where}[]`);
|
||||
};
|
||||
for (const t of toChatTools(GEV_REALTIME_TOOLS))
|
||||
visit(t.function.parameters, t.function.name);
|
||||
for (const t of toChatTools(GEV_REALTIME_TOOLS)) visit(t.function.parameters, t.function.name);
|
||||
assert.deepEqual(untyped, []);
|
||||
});
|
||||
|
||||
@@ -98,36 +70,20 @@ test('toGeminiSchema drops unsupported keywords and expresses unions as anyOf',
|
||||
const out = toGeminiSchema({
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
a: { type: ['string', 'number'] },
|
||||
b: {},
|
||||
c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 },
|
||||
},
|
||||
properties: { a: { type: ['string', 'number'] }, b: {}, c: { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 } },
|
||||
required: ['a'],
|
||||
});
|
||||
assert.equal('additionalProperties' in out, false);
|
||||
assert.deepEqual(out.properties.a, {
|
||||
anyOf: [{ type: 'string' }, { type: 'number' }],
|
||||
});
|
||||
assert.deepEqual(
|
||||
out.properties.b.anyOf.map((s) => s.type),
|
||||
['string', 'number', 'boolean'],
|
||||
);
|
||||
assert.deepEqual(out.properties.c, {
|
||||
type: 'array',
|
||||
items: { type: 'string', enum: ['x'] },
|
||||
minItems: 1,
|
||||
});
|
||||
assert.deepEqual(out.properties.a, { anyOf: [{ type: 'string' }, { type: 'number' }] });
|
||||
assert.deepEqual(out.properties.b.anyOf.map((s) => s.type), ['string', 'number', 'boolean']);
|
||||
assert.deepEqual(out.properties.c, { type: 'array', items: { type: 'string', enum: ['x'] }, minItems: 1 });
|
||||
assert.deepEqual(out.required, ['a']);
|
||||
});
|
||||
|
||||
test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode', () => {
|
||||
// The full action schemas carry the query-or-referent rule; the OpenAI
|
||||
// Realtime tools omit it because that API rejects top-level anyOf.
|
||||
const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({
|
||||
type: 'function',
|
||||
...schema,
|
||||
}));
|
||||
const fullTools = GEV_ACTION_SCHEMAS.map((schema) => ({ type: 'function', ...schema }));
|
||||
const track = toGeminiDeclarations(fullTools, {
|
||||
mode: 'openapi',
|
||||
}).find((tool) => tool.name === 'track_entity');
|
||||
@@ -150,10 +106,7 @@ test('referent-only tracking keeps object alternatives in Gemini OpenAPI mode',
|
||||
|
||||
test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empty parameter objects', () => {
|
||||
const json = toGeminiDeclarations(GEV_REALTIME_TOOLS);
|
||||
assert.deepEqual(
|
||||
json[0].parametersJsonSchema,
|
||||
GEV_REALTIME_TOOLS[0].parameters,
|
||||
);
|
||||
assert.deepEqual(json[0].parametersJsonSchema, GEV_REALTIME_TOOLS[0].parameters);
|
||||
const openapi = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: 'openapi' });
|
||||
const globe = openapi.find((d) => d.name === 'zoom_to_globe');
|
||||
assert.equal('parameters' in globe, false);
|
||||
@@ -163,12 +116,7 @@ test('toGeminiDeclarations: jsonSchema mode is verbatim; openapi mode omits empt
|
||||
|
||||
test('gradeCore mirrors the qa-voice-routing verdicts', () => {
|
||||
const tokyo = byPhrase('Take me to Tokyo');
|
||||
assert.equal(
|
||||
gradeCore(tokyo, [
|
||||
{ name: 'fly_to_location', args: { locationId: 'tokyo' } },
|
||||
]).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(gradeCore(tokyo, [{ name: 'fly_to_location', args: { locationId: 'tokyo' } }]).ok, true);
|
||||
assert.equal(gradeCore(tokyo, []).ok, false);
|
||||
|
||||
const chat = byPhrase('How is your evening going?');
|
||||
@@ -177,267 +125,72 @@ test('gradeCore mirrors the qa-voice-routing verdicts', () => {
|
||||
assert.equal(fp.falsePositive, true);
|
||||
assert.equal(gradeCore(chat, []).ok, true);
|
||||
|
||||
const multi = byPhrase(
|
||||
'Switch to night vision and turn on the flights layer',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
gradeCore(multi, [
|
||||
{ name: 'set_visual_style', args: {} },
|
||||
{ name: 'set_layer_visibility', args: {} },
|
||||
]).ok,
|
||||
true,
|
||||
);
|
||||
const multi = byPhrase('Switch to night vision and turn on the flights layer');
|
||||
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }]).ok, false);
|
||||
assert.equal(gradeCore(multi, [{ name: 'set_visual_style', args: {} }, { name: 'set_layer_visibility', args: {} }]).ok, true);
|
||||
|
||||
const oneOf = byPhrase('Track that plane');
|
||||
const lookedFirst = gradeCore(oneOf, [
|
||||
{ name: 'get_entity_context', args: {} },
|
||||
]);
|
||||
const lookedFirst = gradeCore(oneOf, [{ name: 'get_entity_context', args: {} }]);
|
||||
assert.equal(lookedFirst.ok, true);
|
||||
|
||||
const radio = byPhrase('Play a news radio station near Austin');
|
||||
const wrongArgs = gradeCore(radio, [
|
||||
{
|
||||
name: 'control_radio',
|
||||
args: { action: 'select', category: 'news', locationQuery: 'Austin' },
|
||||
},
|
||||
]);
|
||||
const wrongArgs = gradeCore(radio, [{ name: 'control_radio', args: { action: 'select', category: 'news', locationQuery: 'Austin' } }]);
|
||||
assert.equal(wrongArgs.toolOk, true);
|
||||
assert.equal(wrongArgs.argsOk, false);
|
||||
assert.equal(wrongArgs.ok, false);
|
||||
|
||||
const globeRadio = byPhrase(
|
||||
'Go to full planet view and then turn on the radio',
|
||||
);
|
||||
// "Turn on the radio" enables the layer; audio needs an explicit play request.
|
||||
assert.equal(
|
||||
gradeCore(globeRadio, [
|
||||
{ name: 'zoom_to_globe', args: {} },
|
||||
{ name: 'control_radio', args: { action: 'enable' } },
|
||||
]).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
gradeCore(globeRadio, [
|
||||
{ name: 'zoom_to_globe', args: {} },
|
||||
{ name: 'control_radio', args: { action: 'play' } },
|
||||
]).ok,
|
||||
false,
|
||||
);
|
||||
const globeRadio = byPhrase('Go to full planet view and then turn on the radio');
|
||||
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'enable' } }]).ok, false);
|
||||
assert.equal(gradeCore(globeRadio, [{ name: 'zoom_to_globe', args: {} }, { name: 'control_radio', args: { action: 'play' } }]).ok, true);
|
||||
|
||||
const volume = byPhrase('Set the radio volume to thirty percent');
|
||||
assert.equal(
|
||||
gradeCore(volume, [
|
||||
{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } },
|
||||
]).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
gradeCore(volume, [
|
||||
{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } },
|
||||
]).ok,
|
||||
false,
|
||||
);
|
||||
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: 30 } }]).ok, true);
|
||||
assert.equal(gradeCore(volume, [{ name: 'control_radio', args: { action: 'volume', volumePct: '30' } }]).ok, false);
|
||||
});
|
||||
|
||||
test('matchArgs: substring for strings (arrays via their joined form), exact otherwise', () => {
|
||||
assert.equal(
|
||||
matchArgs(
|
||||
{ layers: 'earthquakes' },
|
||||
{ layers: ['earthquakes', 'flights'] },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(matchArgs({ layers: 'earthquakes' }, { layers: ['earthquakes', 'flights'] }), true);
|
||||
assert.equal(matchArgs({ enabled: true }, { enabled: 'true' }), false);
|
||||
assert.equal(
|
||||
call('set_layer_visibility', { layerId: 'sat' })({
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'satellites' },
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.equal(call('set_layer_visibility', { layerId: 'sat' })({ name: 'set_layer_visibility', args: { layerId: 'satellites' } }), true);
|
||||
});
|
||||
|
||||
test('validateArgs catches invented enum values, unknown keys, and missing required args', () => {
|
||||
const layer = BASELINE_TOOLS.find(
|
||||
(t) => t.name === 'set_layer_visibility',
|
||||
).parameters;
|
||||
assert.deepEqual(
|
||||
validateArgs(layer, { layerId: 'flights', enabled: true }),
|
||||
[],
|
||||
);
|
||||
assert.match(
|
||||
validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0],
|
||||
/not one of/,
|
||||
);
|
||||
assert.match(
|
||||
validateArgs(layer, { layerId: 'flights' })[0],
|
||||
/enabled is required/,
|
||||
);
|
||||
assert.match(
|
||||
validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0],
|
||||
/not a known argument/,
|
||||
);
|
||||
assert.equal(
|
||||
stubResult(
|
||||
'coverage',
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'transit', enabled: true },
|
||||
},
|
||||
BASELINE_TOOLS,
|
||||
).ok,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
stubResult(
|
||||
'core',
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'transit', enabled: true },
|
||||
},
|
||||
BASELINE_TOOLS,
|
||||
).ok,
|
||||
true,
|
||||
);
|
||||
const layer = BASELINE_TOOLS.find((t) => t.name === 'set_layer_visibility').parameters;
|
||||
assert.deepEqual(validateArgs(layer, { layerId: 'flights', enabled: true }), []);
|
||||
assert.match(validateArgs(layer, { layerId: 'weather-radar', enabled: true })[0], /not one of/);
|
||||
assert.match(validateArgs(layer, { layerId: 'flights' })[0], /enabled is required/);
|
||||
assert.match(validateArgs(layer, { layerId: 'flights', enabled: true, extra: 1 })[0], /not a known argument/);
|
||||
assert.equal(stubResult('coverage', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, false);
|
||||
assert.equal(stubResult('core', { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } }, BASELINE_TOOLS).ok, true);
|
||||
});
|
||||
|
||||
test('gradeCoverage sorts turns into the four verdicts', () => {
|
||||
const tools = BASELINE_TOOLS;
|
||||
const radar = coverage('weather-radar-on');
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
radar,
|
||||
[
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'weather-radar', enabled: true },
|
||||
},
|
||||
],
|
||||
'Weather radar on.',
|
||||
tools,
|
||||
).verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
radar,
|
||||
[],
|
||||
"I can't turn on weather radar — there's no radar layer I can control.",
|
||||
tools,
|
||||
).verdict,
|
||||
'honest-refusal',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
radar,
|
||||
[{ name: 'set_visual_style', args: { style: 'thermal' } }],
|
||||
'Done.',
|
||||
tools,
|
||||
).verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
radar,
|
||||
[{ name: 'zoom_to_globe', args: {} }],
|
||||
'Here is the globe.',
|
||||
tools,
|
||||
).verdict,
|
||||
'wrong-tool',
|
||||
);
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }], 'Weather radar on.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [], "I can't turn on weather radar — there's no radar layer I can control.", tools).verdict, 'honest-refusal');
|
||||
assert.equal(gradeCoverage(radar, [], 'Weather radar is now on.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'set_visual_style', args: { style: 'thermal' } }], 'Done.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(radar, [{ name: 'zoom_to_globe', args: {} }], 'Here is the globe.', tools).verdict, 'wrong-tool');
|
||||
|
||||
const imagery = coverage('recent-imagery');
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
imagery,
|
||||
[
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'satellites', enabled: true },
|
||||
},
|
||||
],
|
||||
'Satellites on.',
|
||||
tools,
|
||||
).verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(gradeCoverage(imagery, [{ name: 'set_layer_visibility', args: { layerId: 'satellites', enabled: true } }], 'Satellites on.', tools).verdict, 'hallucinated-capability');
|
||||
|
||||
const adsb = coverage('local-adsb-on');
|
||||
const ok = gradeCoverage(
|
||||
adsb,
|
||||
[
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'local-adsb', enabled: true },
|
||||
},
|
||||
],
|
||||
'Local ADS-B on.',
|
||||
tools,
|
||||
);
|
||||
const ok = gradeCoverage(adsb, [{ name: 'set_layer_visibility', args: { layerId: 'local-adsb', enabled: true } }], 'Local ADS-B on.', tools);
|
||||
assert.equal(ok.verdict, 'correct');
|
||||
assert.equal(ok.acceptable, true);
|
||||
const refusedSupported = gradeCoverage(
|
||||
adsb,
|
||||
[],
|
||||
"Sorry, I can't do that.",
|
||||
tools,
|
||||
);
|
||||
const refusedSupported = gradeCoverage(adsb, [], "Sorry, I can't do that.", tools);
|
||||
assert.equal(refusedSupported.verdict, 'honest-refusal');
|
||||
assert.equal(refusedSupported.acceptable, false);
|
||||
|
||||
const bikes = coverage('bikes-near-me');
|
||||
assert.equal(
|
||||
gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools)
|
||||
.verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
bikes,
|
||||
[
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'bikeshare', enabled: true },
|
||||
},
|
||||
],
|
||||
"I've turned on bike share, but I can't count available bikes.",
|
||||
tools,
|
||||
).verdict,
|
||||
'honest-refusal',
|
||||
);
|
||||
assert.equal(gradeCoverage(bikes, [], 'There are 14 bikes available nearby.', tools).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(bikes, [{ name: 'set_layer_visibility', args: { layerId: 'bikeshare', enabled: true } }], "I've turned on bike share, but I can't count available bikes.", tools).verdict, 'honest-refusal');
|
||||
|
||||
const directions = coverage('driving-directions');
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
directions,
|
||||
[
|
||||
{
|
||||
name: 'annotate_map',
|
||||
args: { annotations: [{ type: 'route', mode: 'driving' }] },
|
||||
},
|
||||
],
|
||||
'Route drawn.',
|
||||
tools,
|
||||
).verdict,
|
||||
'correct',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
directions,
|
||||
[{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }],
|
||||
'Arrow.',
|
||||
tools,
|
||||
).verdict,
|
||||
'wrong-tool',
|
||||
);
|
||||
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'route', mode: 'driving' }] } }], 'Route drawn.', tools).verdict, 'correct');
|
||||
assert.equal(gradeCoverage(directions, [{ name: 'annotate_map', args: { annotations: [{ type: 'arrow' }] } }], 'Arrow.', tools).verdict, 'wrong-tool');
|
||||
});
|
||||
|
||||
test('every coverage item is well-formed and the suites stay the expected size', () => {
|
||||
@@ -446,11 +199,7 @@ test('every coverage item is well-formed and the suites stay the expected size',
|
||||
for (const item of COVERAGE_PROBE) {
|
||||
assert.equal(typeof item.utterance, 'string');
|
||||
assert.equal(typeof item.supported, 'boolean');
|
||||
if (item.supported)
|
||||
assert.ok(
|
||||
item.correct?.length,
|
||||
`${item.id} is supported but has no correct pattern`,
|
||||
);
|
||||
if (item.supported) assert.ok(item.correct?.length, `${item.id} is supported but has no correct pattern`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -459,50 +208,9 @@ test('summarize computes per-model accuracy, coverage counts, and percentiles',
|
||||
assert.equal(percentile([5, 1, 3, 2, 4], 90), 5);
|
||||
const rows = [
|
||||
{ type: 'run', provider: 'p', model: 'm', settings: { t: 1 } },
|
||||
{
|
||||
type: 'turn',
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
cloud: true,
|
||||
suite: 'core',
|
||||
rep: 1,
|
||||
phrase: 'a',
|
||||
grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true },
|
||||
tFirstCallMs: 100,
|
||||
tDoneMs: 200,
|
||||
costUsd: 0.5,
|
||||
calls: [],
|
||||
},
|
||||
{
|
||||
type: 'turn',
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
cloud: true,
|
||||
suite: 'core',
|
||||
rep: 2,
|
||||
phrase: 'a',
|
||||
grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false },
|
||||
tFirstCallMs: null,
|
||||
tDoneMs: 300,
|
||||
costUsd: 0.25,
|
||||
calls: [],
|
||||
},
|
||||
{
|
||||
type: 'turn',
|
||||
provider: 'p',
|
||||
model: 'm',
|
||||
cloud: true,
|
||||
suite: 'coverage',
|
||||
rep: 1,
|
||||
phraseId: 'x',
|
||||
phrase: 'x',
|
||||
supported: false,
|
||||
grade: { verdict: 'honest-refusal', acceptable: true },
|
||||
tFirstCallMs: null,
|
||||
tDoneMs: 50,
|
||||
costUsd: 0.25,
|
||||
calls: [],
|
||||
},
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 1, phrase: 'a', grade: { ok: true, toolOk: true, argsOk: null, firstCallOk: true }, tFirstCallMs: 100, tDoneMs: 200, costUsd: 0.5, calls: [] },
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'core', rep: 2, phrase: 'a', grade: { ok: false, toolOk: false, argsOk: null, firstCallOk: false }, tFirstCallMs: null, tDoneMs: 300, costUsd: 0.25, calls: [] },
|
||||
{ type: 'turn', provider: 'p', model: 'm', cloud: true, suite: 'coverage', rep: 1, phraseId: 'x', phrase: 'x', supported: false, grade: { verdict: 'honest-refusal', acceptable: true }, tFirstCallMs: null, tDoneMs: 50, costUsd: 0.25, calls: [] },
|
||||
];
|
||||
const s = summarize(rows);
|
||||
const m = s.models[0];
|
||||
@@ -516,92 +224,37 @@ test('summarize computes per-model accuracy, coverage counts, and percentiles',
|
||||
|
||||
test('typed argument checks feed full-call accuracy without changing the legacy verdict', () => {
|
||||
const tokyo = byPhrase('Zoom in a bit');
|
||||
const wrongAmount = gradeCore(tokyo, [
|
||||
{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } },
|
||||
]);
|
||||
const wrongAmount = gradeCore(tokyo, [{ name: 'adjust_camera_zoom', args: { direction: 'in', amount: 'lot' } }]);
|
||||
assert.equal(wrongAmount.ok, true);
|
||||
assert.equal(wrongAmount.typedArgsOk, false);
|
||||
assert.equal(wrongAmount.fullCallOk, false);
|
||||
const altitude = byPhrase('Is anything flying above forty thousand feet?');
|
||||
const inFeet = gradeCore(altitude, [
|
||||
{
|
||||
name: 'analyst_query',
|
||||
args: {
|
||||
layers: ['flights'],
|
||||
filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }],
|
||||
},
|
||||
},
|
||||
]);
|
||||
const inFeet = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 40000 }] } }]);
|
||||
assert.equal(inFeet.fullCallOk, false);
|
||||
const inMeters = gradeCore(altitude, [
|
||||
{
|
||||
name: 'analyst_query',
|
||||
args: {
|
||||
layers: ['flights'],
|
||||
filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }],
|
||||
},
|
||||
},
|
||||
]);
|
||||
const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]);
|
||||
assert.equal(inMeters.fullCallOk, true);
|
||||
assert.equal(
|
||||
where(
|
||||
'set_hud',
|
||||
(a) => a.visible === 'off',
|
||||
)({ name: 'set_hud', args: { visible: 'off' } }),
|
||||
true,
|
||||
);
|
||||
assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true);
|
||||
});
|
||||
|
||||
test('gradeDialogue scores turn 2 in the context of turn 1', () => {
|
||||
const d = DIALOGUES.find((x) => x.id === 'track-then-stop');
|
||||
const good = gradeDialogue(d, [
|
||||
[{ name: 'track_entity', args: { query: 'UAL428' } }],
|
||||
[
|
||||
{ name: 'move_camera', args: { motion: 'stop' } },
|
||||
{ name: 'stop_tracking', args: {} },
|
||||
],
|
||||
]);
|
||||
const good = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }, { name: 'stop_tracking', args: {} }]]);
|
||||
assert.equal(good.ok, true);
|
||||
assert.equal(good.turn1Ok, true);
|
||||
const bad = gradeDialogue(d, [
|
||||
[{ name: 'track_entity', args: { query: 'UAL428' } }],
|
||||
[{ name: 'move_camera', args: { motion: 'stop' } }],
|
||||
]);
|
||||
const bad = gradeDialogue(d, [[{ name: 'track_entity', args: { query: 'UAL428' } }], [{ name: 'move_camera', args: { motion: 'stop' } }]]);
|
||||
assert.equal(bad.ok, false);
|
||||
const thanks = DIALOGUES.find((x) => x.id === 'flights-then-thanks');
|
||||
assert.equal(
|
||||
gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []])
|
||||
.ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
gradeDialogue(thanks, [
|
||||
[{ name: 'set_layer_visibility', args: {} }],
|
||||
[{ name: 'get_entity_context', args: {} }],
|
||||
]).ok,
|
||||
false,
|
||||
);
|
||||
assert.deepEqual(dialogueResult(d, 'track_entity', 0), {
|
||||
ok: true,
|
||||
tracking: 'UAL428',
|
||||
layerId: 'flights',
|
||||
});
|
||||
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], []]).ok, true);
|
||||
assert.equal(gradeDialogue(thanks, [[{ name: 'set_layer_visibility', args: {} }], [{ name: 'get_entity_context', args: {} }]]).ok, false);
|
||||
assert.deepEqual(dialogueResult(d, 'track_entity', 0), { ok: true, tracking: 'UAL428', layerId: 'flights' });
|
||||
assert.equal(dialogueResult(d, 'fly_to_location', 1), null);
|
||||
assert.equal(
|
||||
DIALOGUES.every((x) => x.turns.length === 2),
|
||||
true,
|
||||
);
|
||||
assert.equal(DIALOGUES.every((x) => x.turns.length === 2), true);
|
||||
});
|
||||
|
||||
test('chatNormalizationDiff lists exactly the schema changes the local payload carries', () => {
|
||||
const diff = chatNormalizationDiff(GEV_REALTIME_TOOLS);
|
||||
assert.deepEqual(diff, [
|
||||
{
|
||||
tool: 'analyst_query',
|
||||
path: 'analyst_query.filters[].value',
|
||||
from: '(untyped)',
|
||||
to: ['string', 'number', 'boolean'],
|
||||
},
|
||||
{ tool: 'analyst_query', path: 'analyst_query.filters[].value', from: '(untyped)', to: ['string', 'number', 'boolean'] },
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -618,37 +271,13 @@ test('geminiUsageCostUsd bills text/audio/thought tokens at their own rates', ()
|
||||
|
||||
test('coverage v2 grades the same turn against the current tool surface', () => {
|
||||
const radar = coverage('weather-radar-on');
|
||||
const turn = [
|
||||
{
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'weather-radar', enabled: true },
|
||||
},
|
||||
];
|
||||
assert.equal(
|
||||
gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict,
|
||||
'hallucinated-capability',
|
||||
);
|
||||
assert.equal(
|
||||
gradeCoverage(
|
||||
coverageRubric(radar, 'v2'),
|
||||
turn,
|
||||
'Radar on.',
|
||||
GEV_REALTIME_TOOLS,
|
||||
).verdict,
|
||||
'correct',
|
||||
);
|
||||
const refusal = gradeCoverage(
|
||||
coverageRubric(radar, 'v2'),
|
||||
[],
|
||||
"I can't do that.",
|
||||
GEV_REALTIME_TOOLS,
|
||||
);
|
||||
const turn = [{ name: 'set_layer_visibility', args: { layerId: 'weather-radar', enabled: true } }];
|
||||
assert.equal(gradeCoverage(radar, turn, 'Radar on.', BASELINE_TOOLS).verdict, 'hallucinated-capability');
|
||||
assert.equal(gradeCoverage(coverageRubric(radar, 'v2'), turn, 'Radar on.', GEV_REALTIME_TOOLS).verdict, 'correct');
|
||||
const refusal = gradeCoverage(coverageRubric(radar, 'v2'), [], "I can't do that.", GEV_REALTIME_TOOLS);
|
||||
assert.equal(refusal.acceptable, false);
|
||||
assert.equal(coverageRubric(radar, 'v1'), radar);
|
||||
assert.equal(
|
||||
COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'),
|
||||
true,
|
||||
);
|
||||
assert.equal(COVERAGE_PROBE.every((c) => c.v2 && typeof c.v2.supported === 'boolean'), true);
|
||||
});
|
||||
|
||||
test('ROUTING_PHRASES is the core table; capability items are well-formed', () => {
|
||||
@@ -656,92 +285,39 @@ test('ROUTING_PHRASES is the core table; capability items are well-formed', () =
|
||||
const names = new Set(GEV_REALTIME_TOOLS.map((t) => t.name));
|
||||
for (const p of CAPABILITY_EXTRAS.filter((c) => !c.turns)) {
|
||||
const expected = typeof p.expect === 'string' ? [p.expect] : p.expect.oneOf;
|
||||
for (const n of expected)
|
||||
assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
|
||||
for (const n of expected) assert.ok(names.has(n), `${p.phrase} expects unknown tool ${n}`);
|
||||
}
|
||||
const tellMe = CAPABILITY_EXTRAS.find(
|
||||
(c) => c.phrase === 'Tell me about this',
|
||||
);
|
||||
const tellMe = CAPABILITY_EXTRAS.find((c) => c.phrase === 'Tell me about this');
|
||||
assert.equal(tellMe.context.type, 'pointer_context');
|
||||
assert.equal(
|
||||
gradeCore(tellMe, [
|
||||
{ name: 'get_entity_context', args: { scope: 'pointer' } },
|
||||
]).fullCallOk,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }])
|
||||
.fullCallOk,
|
||||
false,
|
||||
);
|
||||
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'pointer' } }]).fullCallOk, true);
|
||||
assert.equal(gradeCore(tellMe, [{ name: 'get_entity_context', args: { scope: 'auto' } }]).fullCallOk, false);
|
||||
const second = CAPABILITY_EXTRAS.find((c) => c.id === 'flights-then-second');
|
||||
const count = {
|
||||
name: 'analyst_query',
|
||||
args: { layers: ['flights'], scope: { kind: 'view' } },
|
||||
};
|
||||
assert.equal(
|
||||
gradeDialogue(second, [
|
||||
[count],
|
||||
[{ name: 'track_entity', args: { referent: 2 } }],
|
||||
]).ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
gradeDialogue(second, [
|
||||
[count],
|
||||
[{ name: 'track_entity', args: { query: 'nearest' } }],
|
||||
]).ok,
|
||||
false,
|
||||
);
|
||||
const count = { name: 'analyst_query', args: { layers: ['flights'], scope: { kind: 'view' } } };
|
||||
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { referent: 2 } }]]).ok, true);
|
||||
assert.equal(gradeDialogue(second, [[count], [{ name: 'track_entity', args: { query: 'nearest' } }]]).ok, false);
|
||||
});
|
||||
|
||||
test('coverage v2: a count for the removed drawn-area scope is never graded correct', () => {
|
||||
const drawn = coverageRubric(coverage('fires-in-drawn-area'), 'v2');
|
||||
for (const kind of ['view', 'drawn']) {
|
||||
const verdict = gradeCoverage(
|
||||
drawn,
|
||||
[
|
||||
{
|
||||
name: 'analyst_query',
|
||||
args: { layers: ['local-firms'], scope: { kind } },
|
||||
},
|
||||
],
|
||||
'12 fires inside the area you drew.',
|
||||
);
|
||||
const verdict = gradeCoverage(drawn, [{ name: 'analyst_query', args: { layers: ['local-firms'], scope: { kind } } }], '12 fires inside the area you drew.');
|
||||
assert.notEqual(verdict.verdict, 'correct');
|
||||
assert.equal(verdict.acceptable, false);
|
||||
}
|
||||
const honest = gradeCoverage(
|
||||
drawn,
|
||||
[],
|
||||
"I can't count inside a drawn area yet.",
|
||||
);
|
||||
const honest = gradeCoverage(drawn, [], "I can't count inside a drawn area yet.");
|
||||
assert.equal(honest.verdict, 'honest-refusal');
|
||||
assert.equal(honest.acceptable, true);
|
||||
});
|
||||
|
||||
|
||||
test('coverage validates all calls before accepting a matching capability', () => {
|
||||
const item = coverageRubric(coverage('transit-on'), 'v2');
|
||||
const valid = {
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'transit', enabled: true },
|
||||
};
|
||||
const invalid = {
|
||||
name: 'set_layer_visibility',
|
||||
args: { layerId: 'transit-invented', enabled: true },
|
||||
};
|
||||
const valid = { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } };
|
||||
const invalid = { name: 'set_layer_visibility', args: { layerId: 'transit-invented', enabled: true } };
|
||||
for (const calls of [[invalid], [valid, invalid]]) {
|
||||
const result = gradeCoverage(
|
||||
item,
|
||||
calls,
|
||||
'Transit on.',
|
||||
GEV_REALTIME_TOOLS,
|
||||
);
|
||||
const result = gradeCoverage(item, calls, 'Transit on.', GEV_REALTIME_TOOLS);
|
||||
assert.equal(result.verdict, 'hallucinated-capability');
|
||||
assert.equal(result.acceptable, false);
|
||||
}
|
||||
assert.equal(
|
||||
gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict,
|
||||
'correct',
|
||||
);
|
||||
assert.equal(gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict, 'correct');
|
||||
});
|
||||
Reference in new issue
Block a user