mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 02:44:02 +00:00
Merge branch 'main' into fix/unit-runner-discovers-server-tests
This commit is contained in:
Sameh Khamis committed
commit
3bc41cc12a
671 files changed
+107238
-5592
No files matched your search
@@ -0,0 +1,759 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the bundled administrative-boundary packs read by
|
||||
* `src/data/adminBoundaries.js`:
|
||||
*
|
||||
* src/data/local_data/natural_earth/states_provinces.json
|
||||
* Natural Earth 10m admin-1 states and provinces (public domain).
|
||||
* src/data/local_data/us_census_counties/counties.json
|
||||
* US Census Bureau cartographic boundary counties (public domain).
|
||||
*
|
||||
* Both sources are pinned by URL and SHA-256, so a rerun reproduces the packs
|
||||
* byte for byte. Downloads are cached outside the repository (default: the
|
||||
* system temp directory) and are never committed.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/build-admin-packs.mjs [--cache <dir>] [--only countries|admin1|counties]
|
||||
*
|
||||
* The provenance READMEs beside each pack record these parameters.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { inflateRawSync } from 'node:zlib';
|
||||
import { normalizeAdminName } from '../src/data/adminBoundaries.js';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const NE_COMMIT = 'ca96624a56bd078437bca8184e78163e5039ad19';
|
||||
const NE_BASE = `https://raw.githubusercontent.com/nvkelso/natural-earth-vector/${NE_COMMIT}/geojson`;
|
||||
|
||||
export const SOURCES = Object.freeze({
|
||||
countries: {
|
||||
url: `${NE_BASE}/ne_10m_admin_0_countries.geojson`,
|
||||
sha256: '239eec57ac17f100a11e2536cffc56752c318b50ae765b0918ff7aab4ce8f255',
|
||||
},
|
||||
mapUnits: {
|
||||
url: `${NE_BASE}/ne_10m_admin_0_map_units.geojson`,
|
||||
sha256: '57da82be755f4afccd8f3b14251bb2752f5df1395f47d2d86f817470c4a48862',
|
||||
},
|
||||
admin1: {
|
||||
url: `${NE_BASE}/ne_10m_admin_1_states_provinces.geojson`,
|
||||
sha256: '22d0e3ad85eb3e27f17cabf8ba2d50e554fbc27a87796ff891d958185da62fb5',
|
||||
},
|
||||
places: {
|
||||
url: `${NE_BASE}/ne_10m_populated_places_simple.geojson`,
|
||||
sha256: 'fd3fa867a320cbd5c5b6bb5bc550afeec2939fb2cef688e508007282a55ac42f',
|
||||
},
|
||||
counties: {
|
||||
url: 'https://www2.census.gov/geo/tiger/GENZ2025/shp/cb_2025_us_county_5m.zip',
|
||||
sha256: 'faec522080681e79be5be435c981009a77891206ff8a7f1d142f3bf5da9ebd74',
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Pack parameters. The Douglas-Peucker tolerance scales with the unit's size
|
||||
* (`toleranceFactor` × √area, clamped to `[minToleranceDeg, maxToleranceDeg]`)
|
||||
* because a unit is viewed at a zoom that fits it: Texas keeps ~2 km detail,
|
||||
* Bavaria ~1 km, Travis County ~150 m. Parts smaller than `minPartKm2` or than
|
||||
* `minPartShare` of the largest part are dropped (the largest part always
|
||||
* stays); holes under `minHoleKm2` are dropped.
|
||||
*/
|
||||
export const PARAMS = Object.freeze({
|
||||
countries: {
|
||||
toleranceFactor: 0.004,
|
||||
minToleranceDeg: 0.005,
|
||||
maxToleranceDeg: 0.02,
|
||||
decimals: 3,
|
||||
minPartKm2: 20,
|
||||
minPartShare: 0.00003,
|
||||
minHoleKm2: 20,
|
||||
},
|
||||
admin1: {
|
||||
toleranceFactor: 0.004,
|
||||
minToleranceDeg: 0.005,
|
||||
maxToleranceDeg: 0.02,
|
||||
decimals: 3,
|
||||
minPartKm2: 20,
|
||||
minPartShare: 0.0003,
|
||||
minHoleKm2: 20,
|
||||
},
|
||||
counties: {
|
||||
toleranceFactor: 0.003,
|
||||
minToleranceDeg: 0.0005,
|
||||
maxToleranceDeg: 0.01,
|
||||
decimals: 4,
|
||||
minPartKm2: 1,
|
||||
minPartShare: 0.001,
|
||||
minHoleKm2: 1,
|
||||
},
|
||||
// A bare admin-1 name is marked ambiguous — the resolver then lets a
|
||||
// geocoder decide — when it is also a country, the name of a city of at
|
||||
// least `insideMinPopulation` inside the unit's box ("New York", "Madrid"),
|
||||
// or the name of a city of at least `elsewhereMinPopulation` outside it
|
||||
// ("Santa Fe", "Victoria"). A unit no larger than `cityStateMaxKm2` whose
|
||||
// own city lies inside is the city itself (Berlin, Vienna, Paris): it is
|
||||
// marked `cityState` and may be named alone however small.
|
||||
ambiguity: {
|
||||
insideMinPopulation: 200_000,
|
||||
elsewhereMinPopulation: 50_000,
|
||||
cityStateMaxKm2: 1_000,
|
||||
},
|
||||
});
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const cacheDir = path.resolve(
|
||||
args.includes('--cache')
|
||||
? args[args.indexOf('--cache') + 1]
|
||||
: path.join(os.tmpdir(), 'gev-admin-packs'),
|
||||
);
|
||||
const only = args.includes('--only') ? args[args.indexOf('--only') + 1] : null;
|
||||
|
||||
async function fetchPinned(name) {
|
||||
const { url, sha256 } = SOURCES[name];
|
||||
const file = path.join(cacheDir, path.basename(new URL(url).pathname));
|
||||
let bytes = await readFile(file).catch(() => null);
|
||||
if (!bytes) {
|
||||
console.log(`Downloading ${url}`);
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error(`HTTP ${response.status} for ${url}`);
|
||||
bytes = Buffer.from(await response.arrayBuffer());
|
||||
await mkdir(cacheDir, { recursive: true });
|
||||
await writeFile(file, bytes);
|
||||
}
|
||||
const digest = createHash('sha256').update(bytes).digest('hex');
|
||||
if (digest !== sha256)
|
||||
throw new Error(
|
||||
`${name}: SHA-256 ${digest} does not match the pinned ${sha256}`,
|
||||
);
|
||||
return { bytes, digest };
|
||||
}
|
||||
|
||||
// ── geometry ────────────────────────────────────────────────────────────
|
||||
|
||||
const EARTH_RADIUS_KM = 6371;
|
||||
const toRad = (d) => (d * Math.PI) / 180;
|
||||
|
||||
/** Spherical ring area (km²); matches src/data/naturalEarthRegions.js. */
|
||||
function ringAreaKm2(ring) {
|
||||
const n = ring.length;
|
||||
if (n < 3) return 0;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const [lon1, lat1] = ring[i];
|
||||
const [lon2, lat2] = ring[(i + 1) % n];
|
||||
sum +=
|
||||
toRad(lon2 - lon1) * (2 + Math.sin(toRad(lat1)) + Math.sin(toRad(lat2)));
|
||||
}
|
||||
return Math.abs((sum * EARTH_RADIUS_KM * EARTH_RADIUS_KM) / 2);
|
||||
}
|
||||
|
||||
/** Signed planar (shoelace) area: positive counter-clockwise in lon/lat. */
|
||||
function signedArea(ring) {
|
||||
let sum = 0;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++)
|
||||
sum += ring[j][0] * ring[i][1] - ring[i][0] * ring[j][1];
|
||||
return sum / 2;
|
||||
}
|
||||
|
||||
function segDist(p, a, b) {
|
||||
let [x, y] = p;
|
||||
const [x1, y1] = a;
|
||||
const [x2, y2] = b;
|
||||
const dx = x2 - x1;
|
||||
const dy = y2 - y1;
|
||||
if (dx !== 0 || dy !== 0) {
|
||||
const t = ((x - x1) * dx + (y - y1) * dy) / (dx * dx + dy * dy);
|
||||
if (t > 1) {
|
||||
x -= x2;
|
||||
y -= y2;
|
||||
return Math.hypot(x, y);
|
||||
}
|
||||
if (t > 0) {
|
||||
x -= x1 + dx * t;
|
||||
y -= y1 + dy * t;
|
||||
return Math.hypot(x, y);
|
||||
}
|
||||
}
|
||||
return Math.hypot(x - x1, y - y1);
|
||||
}
|
||||
|
||||
/** Iterative Douglas-Peucker over an open point list. */
|
||||
function douglasPeucker(points, tolerance) {
|
||||
if (points.length <= 2) return points.slice();
|
||||
const keep = new Uint8Array(points.length);
|
||||
keep[0] = keep[points.length - 1] = 1;
|
||||
const stack = [[0, points.length - 1]];
|
||||
while (stack.length) {
|
||||
const [first, last] = stack.pop();
|
||||
let maxDist = 0;
|
||||
let index = -1;
|
||||
for (let i = first + 1; i < last; i++) {
|
||||
const d = segDist(points[i], points[first], points[last]);
|
||||
if (d > maxDist) {
|
||||
maxDist = d;
|
||||
index = i;
|
||||
}
|
||||
}
|
||||
if (maxDist > tolerance && index !== -1) {
|
||||
keep[index] = 1;
|
||||
stack.push([first, index], [index, last]);
|
||||
}
|
||||
}
|
||||
return points.filter((_, i) => keep[i]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify a closed ring to an OPEN rounded ring. The ring is split at its
|
||||
* farthest vertex from the start so Douglas-Peucker has two anchored halves
|
||||
* (a closed ring has no chord to measure against). Returns null when fewer
|
||||
* than four distinct vertices survive.
|
||||
*/
|
||||
function simplifyRing(ring, tolerance, decimals) {
|
||||
let open = ring.slice();
|
||||
const first = open[0];
|
||||
const last = open[open.length - 1];
|
||||
if (open.length > 1 && first[0] === last[0] && first[1] === last[1])
|
||||
open = open.slice(0, -1);
|
||||
if (open.length < 4) return null;
|
||||
let far = 1;
|
||||
let farDist = -1;
|
||||
for (let i = 1; i < open.length; i++) {
|
||||
const d = Math.hypot(open[i][0] - first[0], open[i][1] - first[1]);
|
||||
if (d > farDist) {
|
||||
farDist = d;
|
||||
far = i;
|
||||
}
|
||||
}
|
||||
const a = douglasPeucker(open.slice(0, far + 1), tolerance);
|
||||
const b = douglasPeucker([...open.slice(far), first], tolerance);
|
||||
const simplified = [...a, ...b.slice(1, -1)];
|
||||
const factor = 10 ** decimals;
|
||||
const out = [];
|
||||
for (const [lon, lat] of simplified) {
|
||||
const p = [
|
||||
Math.round(lon * factor) / factor,
|
||||
Math.round(lat * factor) / factor,
|
||||
];
|
||||
const prev = out[out.length - 1];
|
||||
if (prev && prev[0] === p[0] && prev[1] === p[1]) continue;
|
||||
out.push(p);
|
||||
}
|
||||
while (
|
||||
out.length > 1 &&
|
||||
out[0][0] === out[out.length - 1][0] &&
|
||||
out[0][1] === out[out.length - 1][1]
|
||||
)
|
||||
out.pop();
|
||||
return out.length >= 4 ? out : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify polygons ([outer, ...holes][]): drop parts and holes under the
|
||||
* area floors (the largest part always stays), orient outers
|
||||
* counter-clockwise and holes clockwise, sort parts by area descending.
|
||||
*/
|
||||
function simplifyPolygons(polygons, params, tolerance, decimals) {
|
||||
const parts = [];
|
||||
for (const poly of polygons) {
|
||||
const outer = simplifyRing(poly[0], tolerance, decimals);
|
||||
if (!outer) continue;
|
||||
const area = ringAreaKm2(outer);
|
||||
if (signedArea(outer) < 0) outer.reverse();
|
||||
const holes = [];
|
||||
for (const hole of poly.slice(1)) {
|
||||
const ring = simplifyRing(hole, tolerance, decimals);
|
||||
if (!ring || ringAreaKm2(ring) < params.minHoleKm2) continue;
|
||||
if (signedArea(ring) > 0) ring.reverse();
|
||||
holes.push(ring);
|
||||
}
|
||||
parts.push({ rings: [outer, ...holes], area });
|
||||
}
|
||||
parts.sort((x, y) => y.area - x.area);
|
||||
const floor = Math.max(
|
||||
params.minPartKm2,
|
||||
(parts[0]?.area || 0) * params.minPartShare,
|
||||
);
|
||||
return parts
|
||||
.filter((part, index) => index === 0 || part.area >= floor)
|
||||
.map((part) => part.rings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify one unit at its size-scaled tolerance. A unit too small to
|
||||
* survive the pack precision (a Maldives atoll) is kept at one more decimal
|
||||
* and a fifth of the minimum tolerance, or failing that unsimplified at two
|
||||
* more decimals; the caller records `d`.
|
||||
*/
|
||||
function simplifyUnit(polygons, params) {
|
||||
const sourceArea = polygons.reduce(
|
||||
(sum, poly) => sum + ringAreaKm2(poly[0]),
|
||||
0,
|
||||
);
|
||||
const tolerance = Math.min(
|
||||
params.maxToleranceDeg,
|
||||
Math.max(
|
||||
params.minToleranceDeg,
|
||||
(params.toleranceFactor * Math.sqrt(sourceArea)) / 111.32,
|
||||
),
|
||||
);
|
||||
const coarse = simplifyPolygons(polygons, params, tolerance, params.decimals);
|
||||
if (coarse.length) return { polygons: coarse, decimals: params.decimals };
|
||||
const fine = simplifyPolygons(
|
||||
polygons,
|
||||
params,
|
||||
params.minToleranceDeg / 5,
|
||||
params.decimals + 1,
|
||||
);
|
||||
if (fine.length) return { polygons: fine, decimals: params.decimals + 1 };
|
||||
// Smaller still (Vatican City): the source ring as drawn.
|
||||
const exact = simplifyPolygons(polygons, params, 0, params.decimals + 2);
|
||||
return { polygons: exact, decimals: params.decimals + 2 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a rounded open ring as integers in units of 10^-decimals degrees:
|
||||
* the first vertex absolute, every later vertex as a delta from the previous
|
||||
* one — [lon0, lat0, dLon1, dLat1, …]. Decoded by `decodeRing` in
|
||||
* src/data/adminBoundaries.js.
|
||||
*/
|
||||
function encodeRing(ring, decimals) {
|
||||
const factor = 10 ** decimals;
|
||||
const out = [];
|
||||
let px = 0;
|
||||
let py = 0;
|
||||
for (const [lon, lat] of ring) {
|
||||
const x = Math.round(lon * factor);
|
||||
const y = Math.round(lat * factor);
|
||||
out.push(x - px, y - py);
|
||||
px = x;
|
||||
py = y;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function encodeUnit({ polygons, decimals }) {
|
||||
return polygons.map((poly) => poly.map((ring) => encodeRing(ring, decimals)));
|
||||
}
|
||||
|
||||
function geojsonPolygons(geometry) {
|
||||
if (geometry?.type === 'Polygon') return [geometry.coordinates];
|
||||
if (geometry?.type === 'MultiPolygon') return geometry.coordinates;
|
||||
return [];
|
||||
}
|
||||
|
||||
function bboxOf(polygons) {
|
||||
let w = Infinity;
|
||||
let s = Infinity;
|
||||
let e = -Infinity;
|
||||
let n = -Infinity;
|
||||
for (const poly of polygons)
|
||||
for (const [lon, lat] of poly[0]) {
|
||||
if (lon < w) w = lon;
|
||||
if (lon > e) e = lon;
|
||||
if (lat < s) s = lat;
|
||||
if (lat > n) n = lat;
|
||||
}
|
||||
return [w, s, e, n];
|
||||
}
|
||||
|
||||
// ── zip / shapefile / dbf ───────────────────────────────────────────────
|
||||
|
||||
/** Read named entries from a zip archive (stored or deflated). */
|
||||
function unzip(buffer) {
|
||||
let eocd = buffer.length - 22;
|
||||
while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--;
|
||||
if (eocd < 0) throw new Error('zip: end of central directory not found');
|
||||
const count = buffer.readUInt16LE(eocd + 10);
|
||||
let offset = buffer.readUInt32LE(eocd + 16);
|
||||
const files = new Map();
|
||||
for (let i = 0; i < count; i++) {
|
||||
if (buffer.readUInt32LE(offset) !== 0x02014b50)
|
||||
throw new Error('zip: bad central directory entry');
|
||||
const method = buffer.readUInt16LE(offset + 10);
|
||||
const compressed = buffer.readUInt32LE(offset + 20);
|
||||
const nameLength = buffer.readUInt16LE(offset + 28);
|
||||
const extraLength = buffer.readUInt16LE(offset + 30);
|
||||
const commentLength = buffer.readUInt16LE(offset + 32);
|
||||
const local = buffer.readUInt32LE(offset + 42);
|
||||
const name = buffer.toString('utf8', offset + 46, offset + 46 + nameLength);
|
||||
const dataStart =
|
||||
local +
|
||||
30 +
|
||||
buffer.readUInt16LE(local + 26) +
|
||||
buffer.readUInt16LE(local + 28);
|
||||
const data = buffer.subarray(dataStart, dataStart + compressed);
|
||||
if (method === 0) files.set(name, data);
|
||||
else if (method === 8) files.set(name, inflateRawSync(data));
|
||||
else throw new Error(`zip: unsupported method ${method} for ${name}`);
|
||||
offset += 46 + nameLength + extraLength + commentLength;
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** Polygon records (shape type 5) as arrays of closed [lon, lat] rings. */
|
||||
function readShp(buffer) {
|
||||
const shapes = [];
|
||||
let offset = 100;
|
||||
while (offset < buffer.length) {
|
||||
const length = buffer.readInt32BE(offset + 4) * 2;
|
||||
const content = offset + 8;
|
||||
const type = buffer.readInt32LE(content);
|
||||
const rings = [];
|
||||
if (type === 5) {
|
||||
const numParts = buffer.readInt32LE(content + 36);
|
||||
const numPoints = buffer.readInt32LE(content + 40);
|
||||
const partsAt = content + 44;
|
||||
const pointsAt = partsAt + numParts * 4;
|
||||
for (let p = 0; p < numParts; p++) {
|
||||
const start = buffer.readInt32LE(partsAt + p * 4);
|
||||
const end =
|
||||
p + 1 < numParts
|
||||
? buffer.readInt32LE(partsAt + (p + 1) * 4)
|
||||
: numPoints;
|
||||
const ring = [];
|
||||
for (let i = start; i < end; i++)
|
||||
ring.push([
|
||||
buffer.readDoubleLE(pointsAt + i * 16),
|
||||
buffer.readDoubleLE(pointsAt + i * 16 + 8),
|
||||
]);
|
||||
rings.push(ring);
|
||||
}
|
||||
} else if (type !== 0) {
|
||||
throw new Error(`shp: unsupported shape type ${type}`);
|
||||
}
|
||||
shapes.push(rings);
|
||||
offset = content + length;
|
||||
}
|
||||
return shapes;
|
||||
}
|
||||
|
||||
/** dBASE records as plain objects (character and numeric fields). */
|
||||
function readDbf(buffer) {
|
||||
const count = buffer.readUInt32LE(4);
|
||||
const headerLength = buffer.readUInt16LE(8);
|
||||
const recordLength = buffer.readUInt16LE(10);
|
||||
const fields = [];
|
||||
for (let at = 32; buffer[at] !== 0x0d; at += 32) {
|
||||
fields.push({
|
||||
name: buffer.toString('latin1', at, at + 11).replace(/\0.*$/, ''),
|
||||
length: buffer[at + 16],
|
||||
});
|
||||
}
|
||||
const records = [];
|
||||
for (let r = 0; r < count; r++) {
|
||||
let at = headerLength + r * recordLength + 1;
|
||||
const record = {};
|
||||
for (const field of fields) {
|
||||
record[field.name] = buffer
|
||||
.toString('utf8', at, at + field.length)
|
||||
.trim();
|
||||
at += field.length;
|
||||
}
|
||||
records.push(record);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
/** Group shapefile rings into polygons: clockwise outers, holes assigned by containment. */
|
||||
function shpPolygons(rings) {
|
||||
const outers = [];
|
||||
const holes = [];
|
||||
for (const ring of rings) {
|
||||
if (ring.length < 4) continue;
|
||||
(signedArea(ring) < 0 ? outers : holes).push(ring);
|
||||
}
|
||||
const polygons = outers.map((outer) => [outer]);
|
||||
for (const hole of holes) {
|
||||
const [x, y] = hole[0];
|
||||
const owner = polygons.find(([outer]) => pointInRing(outer, x, y));
|
||||
if (owner) owner.push(hole);
|
||||
else polygons.push([hole.slice().reverse()]);
|
||||
}
|
||||
return polygons;
|
||||
}
|
||||
|
||||
function pointInRing(ring, x, y) {
|
||||
let inside = false;
|
||||
for (let i = 0, j = ring.length - 1; i < ring.length; j = i++) {
|
||||
const [xi, yi] = ring[i];
|
||||
const [xj, yj] = ring[j];
|
||||
if (yi > y !== yj > y && x < ((xj - xi) * (y - yi)) / (yj - yi) + xi)
|
||||
inside = !inside;
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
// ── packs ────────────────────────────────────────────────────────────────
|
||||
|
||||
const LATIN = /^[\p{Script=Latin}\p{N}\s.,'’()\-/&]+$/u;
|
||||
|
||||
function uniqueNames(values, exclude) {
|
||||
const seen = new Set(exclude.map(normalizeAdminName));
|
||||
const out = [];
|
||||
for (const value of values) {
|
||||
const text = String(value || '').trim();
|
||||
const key = normalizeAdminName(text);
|
||||
if (!text || !key || seen.has(key) || !LATIN.test(text)) continue;
|
||||
seen.add(key);
|
||||
out.push(text);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function buildCountries() {
|
||||
const source = await fetchPinned('countries');
|
||||
const mapUnits = await fetchPinned('mapUnits');
|
||||
const params = PARAMS.countries;
|
||||
const features = [];
|
||||
for (const { properties: p, geometry } of JSON.parse(source.bytes).features) {
|
||||
const unit = simplifyUnit(geojsonPolygons(geometry), params);
|
||||
if (!unit.polygons.length) continue;
|
||||
const name = p.ADMIN || p.NAME_EN || p.NAME;
|
||||
features.push({
|
||||
name,
|
||||
alt: uniqueNames(
|
||||
[p.NAME, p.NAME_LONG, p.NAME_EN, p.FORMAL_EN, p.BRK_NAME].filter(
|
||||
Boolean,
|
||||
),
|
||||
[name],
|
||||
),
|
||||
country: name,
|
||||
iso: p.ADM0_A3,
|
||||
iso2: p.ISO_A2_EH || p.ISO_A2,
|
||||
type: 'Country',
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
}
|
||||
for (const { properties: p, geometry } of JSON.parse(mapUnits.bytes)
|
||||
.features) {
|
||||
if (p.ADM0_A3 !== 'GBR') continue;
|
||||
const unit = simplifyUnit(geojsonPolygons(geometry), params);
|
||||
features.push({
|
||||
name: p.GEOUNIT,
|
||||
country: p.ADMIN,
|
||||
iso: p.GU_A3,
|
||||
iso2: 'GB',
|
||||
type: 'Constituent country',
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
}
|
||||
features.sort((a, b) => a.name.localeCompare(b.name));
|
||||
return {
|
||||
meta: {
|
||||
title: 'Natural Earth admin-0 countries',
|
||||
mapUnits: SOURCES.mapUnits,
|
||||
source: 'ne_10m_admin_0_countries',
|
||||
url: SOURCES.countries.url,
|
||||
sha256: source.digest,
|
||||
commit: NE_COMMIT,
|
||||
license:
|
||||
'Public domain (https://www.naturalearthdata.com/about/terms-of-use/)',
|
||||
decimals: params.decimals,
|
||||
curation: params,
|
||||
script: 'scripts/build-admin-packs.mjs',
|
||||
},
|
||||
features,
|
||||
};
|
||||
}
|
||||
|
||||
async function buildAdmin1() {
|
||||
const params = PARAMS.admin1;
|
||||
const source = await fetchPinned('admin1');
|
||||
const places = await fetchPinned('places');
|
||||
const collection = JSON.parse(source.bytes.toString('utf8'));
|
||||
const { ambiguity } = PARAMS;
|
||||
const cities = JSON.parse(places.bytes.toString('utf8'))
|
||||
.features.map((f) => f.properties)
|
||||
.filter((p) => p.pop_max >= ambiguity.elsewhereMinPopulation)
|
||||
.map((p) => ({
|
||||
keys: new Set([p.name, p.nameascii].map(normalizeAdminName)),
|
||||
lon: p.longitude,
|
||||
lat: p.latitude,
|
||||
population: p.pop_max,
|
||||
country: p.adm0name,
|
||||
}));
|
||||
const countryNames = new Set();
|
||||
for (const { properties: p } of collection.features)
|
||||
for (const name of [p.admin, p.geonunit])
|
||||
if (name) countryNames.add(normalizeAdminName(name));
|
||||
|
||||
const features = [];
|
||||
for (const { properties: p, geometry } of collection.features) {
|
||||
const name = String(p.name || '').trim();
|
||||
if (!name) continue;
|
||||
const unit = simplifyUnit(geojsonPolygons(geometry), params);
|
||||
if (!unit.polygons.length) continue;
|
||||
const nameEn =
|
||||
p.name_en && normalizeAdminName(p.name_en) !== normalizeAdminName(name)
|
||||
? String(p.name_en).trim()
|
||||
: null;
|
||||
const alt = uniqueNames(
|
||||
String(p.name_alt || '').split('|'),
|
||||
[name, nameEn].filter(Boolean),
|
||||
);
|
||||
const bbox = bboxOf(unit.polygons);
|
||||
const areaKm2 = unit.polygons.reduce(
|
||||
(sum, poly) => sum + ringAreaKm2(poly[0]),
|
||||
0,
|
||||
);
|
||||
const keys = new Set(
|
||||
[name, nameEn, ...alt].filter(Boolean).map(normalizeAdminName),
|
||||
);
|
||||
const amb = [];
|
||||
const ambAbroad = [];
|
||||
let cityState = false;
|
||||
const inBox = (c) =>
|
||||
c.lon >= bbox[0] &&
|
||||
c.lon <= bbox[2] &&
|
||||
c.lat >= bbox[1] &&
|
||||
c.lat <= bbox[3];
|
||||
for (const key of keys) {
|
||||
const namesakes = cities.filter((c) => c.keys.has(key));
|
||||
const ownCity = namesakes.some(
|
||||
(c) => inBox(c) && c.population >= ambiguity.insideMinPopulation,
|
||||
);
|
||||
const cityInside = ownCity && areaKm2 > ambiguity.cityStateMaxKm2;
|
||||
if (ownCity && !cityInside) cityState = true;
|
||||
const elsewhere = namesakes.filter((c) => !inBox(c));
|
||||
if (
|
||||
countryNames.has(key) ||
|
||||
cityInside ||
|
||||
elsewhere.some((c) => c.country === p.admin)
|
||||
)
|
||||
amb.push(key);
|
||||
// Only abroad: "Victoria, Australia" settles it; "Victoria" alone not.
|
||||
else if (elsewhere.length) ambAbroad.push(key);
|
||||
}
|
||||
const postal = String(p.postal || '').trim();
|
||||
features.push({
|
||||
name,
|
||||
...(nameEn ? { nameEn } : {}),
|
||||
...(alt.length ? { alt } : {}),
|
||||
...(/^[A-Z]{2}$/.test(postal) ? { postal } : {}),
|
||||
type: p.type_en || p.type || null,
|
||||
...(Number.isFinite(p.min_label) ? { rank: p.min_label } : {}),
|
||||
country: p.admin,
|
||||
iso2: p.iso_a2 && p.iso_a2 !== '-99' ? p.iso_a2 : null,
|
||||
iso: p.iso_3166_2 && !p.iso_3166_2.includes('~') ? p.iso_3166_2 : null,
|
||||
...(amb.length ? { amb: amb.sort() } : {}),
|
||||
...(ambAbroad.length ? { ambAbroad: ambAbroad.sort() } : {}),
|
||||
...(cityState ? { cityState } : {}),
|
||||
...(Number.isFinite(p.longitude) && Number.isFinite(p.latitude)
|
||||
? {
|
||||
label: [
|
||||
Math.round(p.longitude * 1000) / 1000,
|
||||
Math.round(p.latitude * 1000) / 1000,
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
}
|
||||
features.sort(
|
||||
(a, b) =>
|
||||
a.country.localeCompare(b.country) ||
|
||||
a.name.localeCompare(b.name) ||
|
||||
(a.iso || '').localeCompare(b.iso || ''),
|
||||
);
|
||||
return {
|
||||
meta: {
|
||||
title: 'Natural Earth admin-1 states and provinces',
|
||||
source: 'ne_10m_admin_1_states_provinces',
|
||||
url: SOURCES.admin1.url,
|
||||
sha256: source.digest,
|
||||
ambiguitySource: SOURCES.places.url,
|
||||
ambiguitySha256: places.digest,
|
||||
commit: NE_COMMIT,
|
||||
license:
|
||||
'Public domain (https://www.naturalearthdata.com/about/terms-of-use/)',
|
||||
schema:
|
||||
'features[]: name, nameEn?, alt?[], postal?, type, rank? (min_label: the zoom at which Natural Earth labels the unit; lower is more prominent), country, iso2, iso (ISO 3166-2), amb?[] (normalized names that are also a country or a city — alone or with the country named), ambAbroad?[] (names shared only with a city in another country — ambiguous alone, settled by naming the country), cityState? (the unit is its own city), label? [lon, lat] label point, d? (decimals when not meta.decimals), polygons[][ring] (first ring outer, rest holes; each ring open and encoded as integers in 10^-d degrees, first vertex absolute then [dLon, dLat] deltas)',
|
||||
decimals: params.decimals,
|
||||
curation: { ...params, ambiguity },
|
||||
script: 'scripts/build-admin-packs.mjs',
|
||||
},
|
||||
features,
|
||||
};
|
||||
}
|
||||
|
||||
async function buildCounties() {
|
||||
const params = PARAMS.counties;
|
||||
const source = await fetchPinned('counties');
|
||||
const files = unzip(source.bytes);
|
||||
const base = path.basename(new URL(SOURCES.counties.url).pathname, '.zip');
|
||||
const shapes = readShp(files.get(`${base}.shp`));
|
||||
const records = readDbf(files.get(`${base}.dbf`));
|
||||
if (shapes.length !== records.length)
|
||||
throw new Error('counties: shp/dbf record counts differ');
|
||||
const features = [];
|
||||
records.forEach((record, index) => {
|
||||
const unit = simplifyUnit(shpPolygons(shapes[index]), params);
|
||||
if (!unit.polygons.length) return;
|
||||
const lsad = record.NAMELSAD.startsWith(record.NAME)
|
||||
? record.NAMELSAD.slice(record.NAME.length).trim()
|
||||
: '';
|
||||
features.push({
|
||||
name: record.NAME,
|
||||
full: record.NAMELSAD,
|
||||
...(lsad ? { lsad } : {}),
|
||||
state: record.STATE_NAME,
|
||||
st: record.STUSPS,
|
||||
geoid: record.GEOID,
|
||||
...(unit.decimals !== params.decimals ? { d: unit.decimals } : {}),
|
||||
polygons: encodeUnit(unit),
|
||||
});
|
||||
});
|
||||
features.sort((a, b) => a.geoid.localeCompare(b.geoid));
|
||||
return {
|
||||
meta: {
|
||||
title: 'US Census Bureau cartographic boundary counties',
|
||||
source: base,
|
||||
url: SOURCES.counties.url,
|
||||
sha256: source.digest,
|
||||
license: 'Public domain (U.S. Government work, 17 U.S.C. § 105)',
|
||||
schema:
|
||||
'features[]: name, full (NAMELSAD), lsad? (County, Parish, Borough, …), state, st (USPS), geoid, d? (decimals when not meta.decimals), polygons[][ring] (first ring outer, rest holes; each ring open and encoded as integers in 10^-d degrees, first vertex absolute then [dLon, dLat] deltas)',
|
||||
decimals: params.decimals,
|
||||
curation: params,
|
||||
script: 'scripts/build-admin-packs.mjs',
|
||||
},
|
||||
features,
|
||||
};
|
||||
}
|
||||
|
||||
async function writePack(relative, pack) {
|
||||
const file = path.join(ROOT, relative);
|
||||
await mkdir(path.dirname(file), { recursive: true });
|
||||
const text = JSON.stringify(pack) + '\n';
|
||||
await writeFile(file, text);
|
||||
const parts = pack.features.reduce((n, f) => n + f.polygons.length, 0);
|
||||
const vertices = pack.features.reduce(
|
||||
(n, f) => n + f.polygons.flat().reduce((m, ring) => m + ring.length / 2, 0),
|
||||
0,
|
||||
);
|
||||
console.log(
|
||||
`${relative}: ${pack.features.length} features, ${parts} parts, ${vertices} vertices, ${Buffer.byteLength(text)} bytes`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!only || only === 'countries')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/countries.json',
|
||||
await buildCountries(),
|
||||
);
|
||||
if (!only || only === 'admin1')
|
||||
await writePack(
|
||||
'src/data/local_data/natural_earth/states_provinces.json',
|
||||
await buildAdmin1(),
|
||||
);
|
||||
if (!only || only === 'counties')
|
||||
await writePack(
|
||||
'src/data/local_data/us_census_counties/counties.json',
|
||||
await buildCounties(),
|
||||
);
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Maintainer-only Overture military-name pack build (DuckDB 1.5.5)."""
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RELEASE = '2026-09-23.1'
|
||||
CLASSES = ['military_land', 'airfield', 'naval_base', 'range', 'barracks', 'base', 'training_area']
|
||||
PACK = ROOT / 'src/data/local_data/osm_military_names'
|
||||
|
||||
|
||||
def assemble(rows):
|
||||
records = []
|
||||
snapshots = set()
|
||||
for row in rows:
|
||||
source = next(s for s in row['sources'] if s['dataset'] == 'OpenStreetMap')
|
||||
match = re.fullmatch(r'([nwr])(\d+)@\d+', source['record_id'])
|
||||
if not match:
|
||||
raise ValueError('Invalid typed OSM identity')
|
||||
ident = match[1] + match[2]
|
||||
bbox = [row['bbox'][k] for k in ['xmin', 'ymin', 'xmax', 'ymax']]
|
||||
lon, lat, area = row['lon'], row['lat'], row['area']
|
||||
if not all(math.isfinite(v) for v in [lon, lat, area, *bbox]):
|
||||
raise ValueError('Invalid geometry measurement')
|
||||
if not bbox[0] - 1e-7 <= lon <= bbox[2] + 1e-7 or not bbox[1] - 1e-7 <= lat <= bbox[3] + 1e-7:
|
||||
raise ValueError('Label outside source bounds')
|
||||
kind = row['military'] if row['military'] in CLASSES else 'military_land'
|
||||
records.append([ident, row['name'], round(lon, 5), round(lat, 5),
|
||||
*[round(v, 4) for v in bbox], CLASSES.index(kind), max(1, round(abs(area)))])
|
||||
snapshots.add(source['version'][:10])
|
||||
records.sort(key=lambda r: (r[0][0], int(r[0][1:])))
|
||||
if len(set(r[0] for r in records)) != len(records):
|
||||
raise ValueError('Duplicate OSM identity')
|
||||
return {'release': RELEASE, 'snapshots': sorted(snapshots), 'classes': CLASSES, 'records': records}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--cache', type=Path, required=True)
|
||||
parser.add_argument('--extensions', type=Path)
|
||||
parser.add_argument('--verify', action='store_true')
|
||||
args = parser.parse_args()
|
||||
args.cache.mkdir(parents=True, exist_ok=True)
|
||||
cached = args.cache / ('military-names-' + RELEASE + '.json')
|
||||
if not cached.exists():
|
||||
import duckdb
|
||||
if duckdb.__version__ != '1.5.5':
|
||||
raise RuntimeError('Use duckdb==1.5.5')
|
||||
config = {'threads': '8', 'memory_limit': '3GB', 'temp_directory': str(args.cache / 'temp')}
|
||||
if args.extensions:
|
||||
config['extension_directory'] = str(args.extensions)
|
||||
connection = duckdb.connect(config=config)
|
||||
if not args.extensions:
|
||||
connection.execute('INSTALL httpfs; INSTALL spatial;')
|
||||
connection.execute('LOAD httpfs; LOAD spatial; SET geometry_always_xy=true; SET enable_progress_bar=false;')
|
||||
files = json.loads((ROOT / 'scripts/military-names-files.json').read_text())
|
||||
if not all('/' + RELEASE + '/' in url for url in files):
|
||||
raise ValueError('Unexpected source release')
|
||||
cursor = connection.execute((ROOT / 'scripts/military-names.sql').read_text(), [files])
|
||||
fields = [d[0] for d in cursor.description]
|
||||
rows = [dict(zip(fields, row)) for row in cursor.fetchall()]
|
||||
cached.write_text(json.dumps(rows, ensure_ascii=False, separators=(',', ':')))
|
||||
pack = assemble(json.loads(cached.read_text()))
|
||||
raw = json.dumps(pack, ensure_ascii=False, separators=(',', ':')).encode()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
if args.verify:
|
||||
expected = (PACK / 'names.sha256').read_text().split()[0]
|
||||
if digest != expected:
|
||||
raise ValueError(f'Output SHA-256 mismatch: {digest}')
|
||||
else:
|
||||
PACK.mkdir(parents=True, exist_ok=True)
|
||||
(PACK / 'names.json').write_bytes(raw)
|
||||
(PACK / 'names.sha256').write_text(digest + ' names.json\n')
|
||||
print(json.dumps({'records': len(pack['records']), 'rawBytes': len(raw),
|
||||
'gzipBytes': len(gzip.compress(raw, compresslevel=9, mtime=0)), 'sha256': digest}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the MCP Apps panel: the app under dist/panel, served at /panel/.
|
||||
* See build/panel.js.
|
||||
*/
|
||||
|
||||
import { readFile, rename, rm, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { build } from 'vite';
|
||||
import standaloneConfig from '../server/standalone/vite.config.js';
|
||||
import {
|
||||
PANEL_BASE,
|
||||
PANEL_OUT_DIR,
|
||||
PANEL_WORKER_FILES,
|
||||
PANEL_WORKER_PRELUDE_PATH,
|
||||
panelBuildConfig,
|
||||
workerFilesPrelude,
|
||||
} from '../build/panel.js';
|
||||
|
||||
const root = fileURLToPath(new URL('../', import.meta.url));
|
||||
const outDir = join(root, PANEL_OUT_DIR);
|
||||
|
||||
const config = standaloneConfig({ command: 'build', mode: 'production' });
|
||||
await build({
|
||||
configFile: false,
|
||||
root,
|
||||
...panelBuildConfig(config),
|
||||
});
|
||||
|
||||
// Cesium's plugin copies its files under the base path inside outDir.
|
||||
const nested = join(outDir, PANEL_BASE, 'cesium');
|
||||
await rm(join(outDir, 'cesium'), { recursive: true, force: true });
|
||||
await rename(nested, join(outDir, 'cesium'));
|
||||
await rm(join(outDir, PANEL_BASE.split('/')[1]), { recursive: true });
|
||||
|
||||
// Cesium's workers load these files themselves; embed them in a prelude the
|
||||
// panel runs ahead of Cesium's workers script.
|
||||
const files = {};
|
||||
for (const name of PANEL_WORKER_FILES)
|
||||
files[name] = await readFile(join(outDir, 'cesium', name), 'utf8');
|
||||
await writeFile(
|
||||
join(outDir, PANEL_WORKER_PRELUDE_PATH),
|
||||
workerFilesPrelude(files),
|
||||
);
|
||||
console.log(`Panel build written to ${PANEL_OUT_DIR}`);
|
||||
@@ -37,6 +37,10 @@ const portableExport = (key) =>
|
||||
) ||
|
||||
[
|
||||
'./director',
|
||||
'./tools',
|
||||
'./tools/mcp',
|
||||
'./tools/services',
|
||||
'./view',
|
||||
'./voice/action-schemas',
|
||||
'./voice/session',
|
||||
'./data/lifecycle',
|
||||
|
||||
@@ -347,7 +347,7 @@ case "${OPENSKY_AUTH_MODE}" in
|
||||
echo "OpenSky auth: disabled (anonymous mode)"
|
||||
;;
|
||||
esac
|
||||
[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set — GEV MIC disabled"
|
||||
[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set — GEV MIC off unless a Codex ChatGPT sign-in is selected in Provider Settings (experimental)"
|
||||
[[ -n "${AISSTREAM_API_KEY}" ]] && echo "AISStream key (live vessels): configured" || echo "AISStream key (live vessels): not set — ships layer empty"
|
||||
if [[ -n "${GOOGLE_MAPS_API_KEY}" ]]; then
|
||||
echo "Startup map: Google Photorealistic 3D Tiles (direct)"
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
import { deflateSync } from 'node:zlib';
|
||||
import { tileBounds } from '../../../src/layers/streetLevel/tileMath.js';
|
||||
import { metresApart, photoJpeg } from './mapillaryGraph.mjs';
|
||||
|
||||
/**
|
||||
* KartaView's API, image proxy and photo storage answered from fixtures, so
|
||||
* the gate's KartaView steps need no network. One drive runs north along
|
||||
* KV_LINE; every tenth photo is a 360° one. Photos live on storage2 while
|
||||
* their `fileurl*` name storage1, as KartaView's API does, so the gate also
|
||||
* proves the provider builds addresses from `storage`. Every odd-indexed
|
||||
* photo is archived (storage answers 409, the image proxy still serves it),
|
||||
* so the nearest photo opened is even and › must skip the one after it. The
|
||||
* first nearest-photo lookup is rate-limited (429, Retry-After: 0), as
|
||||
* KartaView does under load.
|
||||
*
|
||||
* Beside the drive runs KV_OLD_STREET, photographed flat in 2019 only: the
|
||||
* imagery filter drops it. KartaView filters coverage only in its PNG
|
||||
* tiles, so filtered PNG tiles are drawn here for the streets that pass.
|
||||
*/
|
||||
|
||||
export const KV_SEQUENCE_ID = '900001';
|
||||
export const KV_LINE = Object.freeze({
|
||||
lon: -121.4931,
|
||||
south: 38.5795,
|
||||
north: 38.5875,
|
||||
});
|
||||
/** A street ~130 m east of the drive with only flat 2019 photos. */
|
||||
export const KV_OLD_STREET = Object.freeze({
|
||||
lon: -121.4916,
|
||||
south: 38.58,
|
||||
north: 38.587,
|
||||
});
|
||||
const OLD_STREET_UPLOADED = '2019-06-01';
|
||||
const KV_SPACING_M = 25;
|
||||
const METRES_PER_DEG_LAT = 110_540;
|
||||
/** KartaView's time format for `ms`. */
|
||||
const kartaViewTime = (ms) =>
|
||||
new Date(ms).toISOString().replace('T', ' ').slice(0, 23);
|
||||
/** The drive is a month old, whenever the gate runs: LAST YEAR keeps it. */
|
||||
const SHOT = kartaViewTime(Date.now() - 30 * 86_400_000);
|
||||
|
||||
const CORS = Object.freeze({ 'Access-Control-Allow-Origin': '*' });
|
||||
|
||||
/** Lookups answered so far: the first one is rate-limited. */
|
||||
let nearestLookups = 0;
|
||||
|
||||
/** Forget the lookups answered, so the next one is rate-limited again. */
|
||||
export function resetKartaViewFixture() {
|
||||
nearestLookups = 0;
|
||||
}
|
||||
|
||||
/** Nearest lookups answered since the last reset (the first got a 429). */
|
||||
export function kartaViewLookups() {
|
||||
return nearestLookups;
|
||||
}
|
||||
|
||||
/** The drive's photos, south to north; every tenth is 360°, every odd one archived. */
|
||||
export function kartaViewPhotos() {
|
||||
const step = KV_SPACING_M / METRES_PER_DEG_LAT;
|
||||
const count = Math.floor((KV_LINE.north - KV_LINE.south) / step) + 1;
|
||||
return Array.from({ length: count }, (_, i) => ({
|
||||
id: String(77_000_000 + i),
|
||||
index: i,
|
||||
lon: KV_LINE.lon,
|
||||
lat: Number((KV_LINE.south + i * step).toFixed(7)),
|
||||
isPano: i % 10 === 5,
|
||||
archived: i % 2 === 1,
|
||||
}));
|
||||
}
|
||||
|
||||
function record(photo) {
|
||||
const name = `${KV_SEQUENCE_ID}_${photo.id}.jpg`;
|
||||
const folder = photo.isPano ? 'wrapped_proc' : 'proc';
|
||||
const path = (prefix) => `2024/5/2/${prefix}/${name}`;
|
||||
const storage1 = (prefix) =>
|
||||
`https://storage1.openstreetcam.org/files/photo/${path(prefix)}`;
|
||||
return {
|
||||
id: photo.id,
|
||||
sequenceId: KV_SEQUENCE_ID,
|
||||
sequenceIndex: String(photo.index),
|
||||
lat: String(photo.lat),
|
||||
lng: String(photo.lon),
|
||||
matchLat: String(photo.lat),
|
||||
matchLng: String(photo.lon),
|
||||
// A dashcam phone's compass, disturbed by the car: the drive runs north.
|
||||
heading: '200',
|
||||
fieldOfView: photo.isPano ? '360' : '0',
|
||||
projection: photo.isPano ? 'SPHERE' : 'PLANE',
|
||||
shotDate: SHOT,
|
||||
dateAdded: SHOT,
|
||||
storage: 'storage2',
|
||||
filepath: path('{{sizeprefix}}'),
|
||||
filepathLTh: path('lth'),
|
||||
filepathProc: path(folder),
|
||||
fileurl: storage1('{{sizeprefix}}'),
|
||||
fileurlLTh: storage1('lth'),
|
||||
fileurlProc: storage1(folder),
|
||||
};
|
||||
}
|
||||
|
||||
const ok = (result) => ({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
headers: CORS,
|
||||
body: JSON.stringify({ status: { apiCode: 600, httpCode: 200 }, result }),
|
||||
});
|
||||
const raw = (status, body) => ({
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
headers: CORS,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
/** The streets with photos: the drive, and the old street beside it. */
|
||||
function streets() {
|
||||
return [
|
||||
{ line: KV_LINE, uploaded: SHOT.slice(0, 10), views: ['360', 'plane'] },
|
||||
{ line: KV_OLD_STREET, uploaded: OLD_STREET_UPLOADED, views: ['plane'] },
|
||||
];
|
||||
}
|
||||
|
||||
/** A street's stretch inside tile `z/x/y`, as [from, to] latitudes, or null. */
|
||||
function stretch(line, z, x, y) {
|
||||
const { west, east, south, north } = tileBounds(x, y, z);
|
||||
if (line.lon < west || line.lon > east) return null;
|
||||
const from = Math.max(line.south, south);
|
||||
const to = Math.min(line.north, north);
|
||||
return from < to ? [from, to] : null;
|
||||
}
|
||||
|
||||
/** The streets' tracks inside tile `z/x/y`, as KartaView's JSON tile. */
|
||||
function tileLines(z, x, y) {
|
||||
const lines = [];
|
||||
for (const { line } of streets()) {
|
||||
const inside = stretch(line, z, x, y);
|
||||
if (!inside) continue;
|
||||
const [from, to] = inside;
|
||||
// Two segments meeting end to start, as KartaView splits ways.
|
||||
const mid = (from + to) / 2;
|
||||
const at = (lat) => ({ lat: String(lat), lng: String(line.lon) });
|
||||
lines.push(
|
||||
{ track: [at(from), at(mid)], matched: '1' },
|
||||
{ track: [at(mid), at(to)], matched: '1' },
|
||||
);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Whether a street has photos passing a filtered tile's query. */
|
||||
function passesTileQuery(street, query) {
|
||||
const view = query.get('fieldOfView');
|
||||
if (view && !street.views.includes(view)) return false;
|
||||
const since = query.get('startDate');
|
||||
return !since || street.uploaded >= since;
|
||||
}
|
||||
|
||||
const CRC_TABLE = Array.from({ length: 256 }, (_, n) => {
|
||||
let c = n;
|
||||
for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
||||
return c >>> 0;
|
||||
});
|
||||
function crc32(bytes) {
|
||||
let c = 0xffffffff;
|
||||
for (const byte of bytes) c = CRC_TABLE[(c ^ byte) & 0xff] ^ (c >>> 8);
|
||||
return (c ^ 0xffffffff) >>> 0;
|
||||
}
|
||||
function pngChunk(type, data) {
|
||||
const head = Buffer.alloc(8);
|
||||
head.writeUInt32BE(data.length, 0);
|
||||
head.write(type, 4, 'ascii');
|
||||
const crc = Buffer.alloc(4);
|
||||
crc.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), data])), 0);
|
||||
return Buffer.concat([head, data, crc]);
|
||||
}
|
||||
|
||||
/**
|
||||
* A filtered coverage tile as KartaView draws it: 512 px, transparent, the
|
||||
* passing streets three pixels wide.
|
||||
*/
|
||||
export function maskTilePng(z, x, y, query) {
|
||||
const size = 512;
|
||||
const rgba = Buffer.alloc(size * size * 4);
|
||||
const { west, east } = tileBounds(x, y, z);
|
||||
const n = 2 ** z;
|
||||
const row = (lat) => {
|
||||
const sin = Math.sin((lat * Math.PI) / 180);
|
||||
const my = 0.5 - Math.log((1 + sin) / (1 - sin)) / (4 * Math.PI);
|
||||
return Math.round((my * n - y) * size);
|
||||
};
|
||||
for (const street of streets()) {
|
||||
if (!passesTileQuery(street, query)) continue;
|
||||
const inside = stretch(street.line, z, x, y);
|
||||
if (!inside) continue;
|
||||
const column = Math.round(
|
||||
((street.line.lon - west) / (east - west)) * size,
|
||||
);
|
||||
const top = Math.max(0, row(inside[1]));
|
||||
const bottom = Math.min(size - 1, row(inside[0]));
|
||||
for (let py = top; py <= bottom; py++)
|
||||
for (let px = column - 1; px <= column + 1; px++) {
|
||||
if (px < 0 || px >= size) continue;
|
||||
rgba.set([99, 91, 255, 255], (py * size + px) * 4);
|
||||
}
|
||||
}
|
||||
const rows = Buffer.alloc(size * (size * 4 + 1));
|
||||
for (let py = 0; py < size; py++)
|
||||
rgba.copy(
|
||||
rows,
|
||||
py * (size * 4 + 1) + 1,
|
||||
py * size * 4,
|
||||
(py + 1) * size * 4,
|
||||
);
|
||||
const header = Buffer.alloc(13);
|
||||
header.writeUInt32BE(size, 0);
|
||||
header.writeUInt32BE(size, 4);
|
||||
header.set([8, 6, 0, 0, 0], 8);
|
||||
return Buffer.concat([
|
||||
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
|
||||
pngChunk('IHDR', header),
|
||||
pngChunk('IDAT', deflateSync(rows)),
|
||||
pngChunk('IEND', Buffer.alloc(0)),
|
||||
]);
|
||||
}
|
||||
|
||||
/** A call named by host, path shape and parameter names. */
|
||||
export function describeKartaViewCall(method, url) {
|
||||
const keys = [...url.searchParams.keys()].sort();
|
||||
const path = url.pathname
|
||||
.replace(/\/tiles\/\d+\/\d+\/\d+\.json$/, '/tiles/{x}/{y}/{z}.json')
|
||||
.replace(/\/tiles\/\d+\/\d+\/\d+\.png$/, '/tiles/{x}/{y}/{z}.png')
|
||||
.replace(/\/photo\/\d+$/, '/photo/{id}')
|
||||
.replace(/\/sequence\/\d+\/photos$/, '/sequence/{id}/photos')
|
||||
.replace(/\/sequence\/\d+$/, '/sequence/{id}')
|
||||
.replace(/^\/files\/photo\/.*$/, '/files/photo/{path}')
|
||||
.replace(
|
||||
/^\/pr:sharp\/rs:fit:\d+:\d+\/.*$/,
|
||||
'/pr:sharp/rs:fit:{w}:{h}/{address}',
|
||||
);
|
||||
return `${method} ${url.hostname}${path}${keys.length ? `?${keys.join('&')}` : ''}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Answer one KartaView request; `known` is false for a call no fixture
|
||||
* covers, so the gate fails on a new one instead of missing it.
|
||||
*/
|
||||
export function answerKartaViewRequest({ method, url }) {
|
||||
const address = new URL(url);
|
||||
const photos = kartaViewPhotos();
|
||||
if (method === 'OPTIONS')
|
||||
return { status: 204, headers: CORS, body: '', known: true };
|
||||
const image = {
|
||||
status: 200,
|
||||
contentType: 'image/jpeg',
|
||||
headers: CORS,
|
||||
body: photoJpeg(),
|
||||
known: true,
|
||||
};
|
||||
if (address.hostname === 'cdn.kartaview.org')
|
||||
return /^\/pr:sharp\/rs:fit:\d+:\d+\/[\w-]+$/.test(address.pathname)
|
||||
? image
|
||||
: { ...raw(404, {}), known: false };
|
||||
if (address.hostname === 'storage2.openstreetcam.org') {
|
||||
if (!address.pathname.includes(KV_SEQUENCE_ID))
|
||||
return { ...raw(404, {}), known: false };
|
||||
const archived = photos.some(
|
||||
(photo) =>
|
||||
photo.archived && address.pathname.endsWith(`_${photo.id}.jpg`),
|
||||
);
|
||||
// An archived photo's file is gone from storage: KartaView answers 409.
|
||||
return archived ? { ...raw(409, {}), known: true } : image;
|
||||
}
|
||||
if (/^storage\d+\.openstreetcam\.org$/.test(address.hostname))
|
||||
// The wrong host the API names: KartaView answers 404 there.
|
||||
return { ...raw(404, {}), known: false };
|
||||
if (address.hostname !== 'api.kartaview.org')
|
||||
return { ...raw(404, {}), known: false };
|
||||
const path = address.pathname;
|
||||
const tile = path.match(
|
||||
/^\/2\.0\/sequence\/tiles\/(\d+)\/(\d+)\/(\d+)\.json$/,
|
||||
);
|
||||
if (tile)
|
||||
return {
|
||||
...raw(200, {
|
||||
lines: tileLines(Number(tile[3]), Number(tile[1]), Number(tile[2])),
|
||||
}),
|
||||
known: true,
|
||||
};
|
||||
// Filtered coverage: KartaView filters only its PNG tiles.
|
||||
const mask = path.match(
|
||||
/^\/2\.0\/sequence\/tiles\/(\d+)\/(\d+)\/(\d+)\.png$/,
|
||||
);
|
||||
if (mask)
|
||||
return {
|
||||
status: 200,
|
||||
contentType: 'image/png',
|
||||
headers: CORS,
|
||||
body: maskTilePng(
|
||||
Number(mask[3]),
|
||||
Number(mask[1]),
|
||||
Number(mask[2]),
|
||||
address.searchParams,
|
||||
),
|
||||
known: true,
|
||||
};
|
||||
if (path === '/2.0/photo/') {
|
||||
if (nearestLookups++ === 0)
|
||||
return {
|
||||
...raw(429, {}),
|
||||
headers: {
|
||||
...CORS,
|
||||
'Retry-After': '0',
|
||||
'Access-Control-Expose-Headers': 'Retry-After',
|
||||
},
|
||||
known: true,
|
||||
};
|
||||
const q = address.searchParams;
|
||||
const at = { lat: Number(q.get('lat')), lon: Number(q.get('lng')) };
|
||||
// Its own filter, as KartaView's: projection, and upload date.
|
||||
const projection = q.get('projection');
|
||||
const since = q.get('startDate');
|
||||
const near = photos
|
||||
.filter(
|
||||
(photo) =>
|
||||
(!projection || projection === (photo.isPano ? 'SPHERE' : 'PLANE')) &&
|
||||
(!since || SHOT.slice(0, 10) >= since),
|
||||
)
|
||||
.map((photo) => ({ photo, metres: metresApart(at, photo) }))
|
||||
.filter(({ metres }) => metres < 200)
|
||||
.sort((a, b) => a.metres - b.metres)
|
||||
.slice(0, Number(q.get('itemsPerPage')) || 10)
|
||||
.map(({ photo, metres }) => ({
|
||||
...record(photo),
|
||||
distance: metres.toFixed(2),
|
||||
}));
|
||||
return { ...ok({ data: near, hasMoreData: false }), known: true };
|
||||
}
|
||||
const one = path.match(/^\/2\.0\/photo\/(\d+)$/);
|
||||
if (one) {
|
||||
const photo = photos.find((p) => p.id === one[1]);
|
||||
return photo
|
||||
? { ...ok({ data: record(photo) }), known: true }
|
||||
: { ...raw(200, { status: { apiCode: 601 }, result: {} }), known: true };
|
||||
}
|
||||
const sequence = path.match(/^\/2\.0\/sequence\/(\d+)$/);
|
||||
if (sequence)
|
||||
return {
|
||||
...ok({
|
||||
data: {
|
||||
id: sequence[1],
|
||||
countActivePhotos:
|
||||
sequence[1] === KV_SEQUENCE_ID ? String(photos.length) : '0',
|
||||
},
|
||||
}),
|
||||
known: true,
|
||||
};
|
||||
const page = path.match(/^\/2\.0\/sequence\/(\d+)\/photos$/);
|
||||
if (page) {
|
||||
const q = address.searchParams;
|
||||
const size = Number(q.get('itemsPerPage')) || 100;
|
||||
const number = Number(q.get('page')) || 1;
|
||||
const rows = page[1] === KV_SEQUENCE_ID ? photos : [];
|
||||
return {
|
||||
...ok({
|
||||
data: rows.slice((number - 1) * size, number * size).map(record),
|
||||
hasMoreData: number * size < rows.length,
|
||||
}),
|
||||
known: true,
|
||||
};
|
||||
}
|
||||
return { ...raw(404, {}), known: false };
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
/**
|
||||
* Mapillary Graph API and image CDN answered from fixtures, covering every
|
||||
* request the provider and MapillaryJS make while opening, stepping and
|
||||
* closing a photo, so the gate needs no network and no real token.
|
||||
*
|
||||
* No `merge_cc` or `sfm_cluster` means MapillaryJS never asks for a mesh or
|
||||
* cluster, and empty S2 cells mean no spatial edges, which the gate does not
|
||||
* use. The photo is generated for this repository, so it has no third-party
|
||||
* licence.
|
||||
*/
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** The one sequence with photos: a north-south line through the parked view. */
|
||||
export const PHOTO_SEQUENCE_ID = 'fx-photo';
|
||||
export const PHOTO_LINE = Object.freeze({
|
||||
lon: -121.4944,
|
||||
south: 38.579,
|
||||
north: 38.591,
|
||||
});
|
||||
/** Metres between consecutive photos on the line (≥ the 3 m cone thinning). */
|
||||
export const PHOTO_SPACING_M = 30;
|
||||
/** Fixture thumbnail host (answered, never reached). */
|
||||
export const THUMB_HOST = 'qa-fixture.mapillary.com';
|
||||
|
||||
const METRES_PER_DEG_LAT = 110_540;
|
||||
|
||||
/** Ground distance in metres, as the app's metresBetween. */
|
||||
export function metresApart(a, b) {
|
||||
const lat = (((a.lat + b.lat) / 2) * Math.PI) / 180;
|
||||
return Math.hypot(
|
||||
(b.lon - a.lon) * 111_320 * Math.cos(lat),
|
||||
(b.lat - a.lat) * METRES_PER_DEG_LAT,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Photos along PHOTO_LINE, south to north; every third one is a panorama.
|
||||
* @returns {Array<{id: string, lon: number, lat: number, isPano: boolean, capturedAt: number, compassAngle: number}>}
|
||||
*/
|
||||
export function photoImages(now = Date.now()) {
|
||||
const step = PHOTO_SPACING_M / METRES_PER_DEG_LAT;
|
||||
const count =
|
||||
Math.floor((PHOTO_LINE.north - PHOTO_LINE.south) / step + 1e-9) + 1;
|
||||
return Array.from({ length: count }, (_, i) => ({
|
||||
id: String(9_100_000_000_000 + i),
|
||||
lon: PHOTO_LINE.lon,
|
||||
lat: Number((PHOTO_LINE.south + i * step).toFixed(7)),
|
||||
isPano: i % 3 === 0,
|
||||
capturedAt: now - 30 * DAY_MS - i * 1000,
|
||||
compassAngle: 0,
|
||||
}));
|
||||
}
|
||||
|
||||
const point = (image) => ({
|
||||
type: 'Point',
|
||||
coordinates: [image.lon, image.lat],
|
||||
});
|
||||
|
||||
/** Every field either caller asks for; extra fields are harmless to both. */
|
||||
function imageRecord(image) {
|
||||
const thumb = `https://${THUMB_HOST}/thumb/${image.id}.jpg`;
|
||||
return {
|
||||
id: image.id,
|
||||
sequence: PHOTO_SEQUENCE_ID,
|
||||
geometry: point(image),
|
||||
computed_geometry: point(image),
|
||||
altitude: 12,
|
||||
computed_altitude: 12,
|
||||
atomic_scale: 1,
|
||||
camera_parameters: image.isPano ? [] : [0.85, 0, 0],
|
||||
camera_type: image.isPano ? 'spherical' : 'perspective',
|
||||
captured_at: image.capturedAt,
|
||||
compass_angle: image.compassAngle,
|
||||
computed_compass_angle: image.compassAngle,
|
||||
// Angle-axis world-to-camera rotation: +90° about east looks north, level.
|
||||
computed_rotation: [Math.PI / 2, 0, 0],
|
||||
creator: { id: '1', username: 'qa-fixture' },
|
||||
exif_orientation: 1,
|
||||
height: 320,
|
||||
width: 640,
|
||||
is_pano: image.isPano,
|
||||
merge_cc: null,
|
||||
mesh: null,
|
||||
organization: null,
|
||||
quality_score: 0.9,
|
||||
sfm_cluster: null,
|
||||
thumb_256_url: thumb,
|
||||
thumb_1024_url: thumb,
|
||||
thumb_2048_url: thumb,
|
||||
};
|
||||
}
|
||||
|
||||
/** The requested fields plus id, as the Graph API answers. */
|
||||
function withFields(record, fields) {
|
||||
if (!fields) return record;
|
||||
const out = { id: record.id };
|
||||
for (const field of fields.split(','))
|
||||
if (field in record) out[field] = record[field];
|
||||
return out;
|
||||
}
|
||||
|
||||
const CORS = Object.freeze({
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Accept, Authorization, Content-Type',
|
||||
'Access-Control-Max-Age': '600',
|
||||
});
|
||||
|
||||
const json = (status, body) => ({
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
headers: CORS,
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
let photoBytes = null;
|
||||
export function photoJpeg() {
|
||||
photoBytes ??= readFileSync(new URL('./street-640x320.jpg', import.meta.url));
|
||||
return photoBytes;
|
||||
}
|
||||
|
||||
/** Name a call by its parameter names only, so the token is never echoed. */
|
||||
export function describeCall(method, url) {
|
||||
const keys = [...url.searchParams.keys()]
|
||||
.filter((key) => key !== 'access_token')
|
||||
.sort();
|
||||
const path = url.pathname
|
||||
.replace(/^\/\d{6,}(?=\/|$)/, '/{imageId}')
|
||||
.replace(/^\/thumb\/[\w-]+\.jpg$/, '/thumb/{imageId}.jpg');
|
||||
return `${method} ${url.hostname}${path}${keys.length ? `?${keys.join('&')}` : ''}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Answer one Mapillary request. `known` is false for a request no fixture
|
||||
* covers, so the gate fails on a new MapillaryJS call instead of missing it.
|
||||
* @param {{method: string, url: string, images?: ReturnType<typeof photoImages>}} request
|
||||
* @returns {{status: number, contentType?: string, headers?: object, body: string|Buffer, known: boolean}}
|
||||
*/
|
||||
export function answerMapillaryRequest({ method, url, images }) {
|
||||
const address = new URL(url);
|
||||
const photos = images ?? photoImages();
|
||||
const byId = new Map(photos.map((image) => [image.id, image]));
|
||||
if (method === 'OPTIONS')
|
||||
return { status: 204, headers: CORS, body: '', known: true };
|
||||
if (address.hostname === THUMB_HOST) {
|
||||
if (/^\/thumb\/[\w-]+\.jpg$/.test(address.pathname))
|
||||
return {
|
||||
status: 200,
|
||||
contentType: 'image/jpeg',
|
||||
headers: CORS,
|
||||
body: photoJpeg(),
|
||||
known: true,
|
||||
};
|
||||
return { ...json(404, { error: 'no such fixture' }), known: false };
|
||||
}
|
||||
if (address.hostname !== 'graph.mapillary.com')
|
||||
return { ...json(404, { error: 'no such fixture host' }), known: false };
|
||||
const q = address.searchParams;
|
||||
const fields = q.get('fields');
|
||||
const path = address.pathname;
|
||||
if (path === '/images') {
|
||||
if (q.has('image_ids')) {
|
||||
const data = q
|
||||
.get('image_ids')
|
||||
.split(',')
|
||||
.map((id) => byId.get(id))
|
||||
.filter(Boolean)
|
||||
.map((image) => withFields(imageRecord(image), fields));
|
||||
return { ...json(200, { data }), known: true };
|
||||
}
|
||||
if (q.has('sequence_ids')) {
|
||||
const ids = q.get('sequence_ids').split(',');
|
||||
const limit = Number(q.get('limit')) || photos.length;
|
||||
const data = ids.includes(PHOTO_SEQUENCE_ID)
|
||||
? photos
|
||||
.slice(0, limit)
|
||||
.map((image) => withFields(imageRecord(image), fields))
|
||||
: [];
|
||||
return { ...json(200, { data }), known: true };
|
||||
}
|
||||
if (q.has('lat') && q.has('lng')) {
|
||||
const at = { lat: Number(q.get('lat')), lon: Number(q.get('lng')) };
|
||||
const radius = Math.min(50, Number(q.get('radius')) || 50);
|
||||
const limit = Number(q.get('limit')) || 10;
|
||||
// In no particular order, as the real API answers.
|
||||
const data = photos
|
||||
.filter((image) => metresApart(at, image) <= radius)
|
||||
.reverse()
|
||||
.slice(0, limit)
|
||||
.map((image) => withFields(imageRecord(image), fields));
|
||||
return { ...json(200, { data }), known: true };
|
||||
}
|
||||
if (q.has('s2')) return { ...json(200, { data: [] }), known: true };
|
||||
return {
|
||||
...json(400, { error: 'unsupported images query' }),
|
||||
known: false,
|
||||
};
|
||||
}
|
||||
if (path === '/image_ids' && q.has('sequence_id')) {
|
||||
const data =
|
||||
q.get('sequence_id') === PHOTO_SEQUENCE_ID
|
||||
? photos.map((image) => ({ id: image.id }))
|
||||
: [];
|
||||
return { ...json(200, { data }), known: true };
|
||||
}
|
||||
const tiles = path.match(/^\/([\w-]+)\/tiles$/);
|
||||
if (tiles) return { ...json(200, { data: [] }), known: true };
|
||||
const entity = path.match(/^\/([\w-]+)$/);
|
||||
if (entity && byId.has(entity[1]))
|
||||
return {
|
||||
...json(200, withFields(imageRecord(byId.get(entity[1])), fields)),
|
||||
known: true,
|
||||
};
|
||||
return { ...json(404, { error: 'no such fixture' }), known: false };
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
+156
-1
@@ -26,11 +26,24 @@
|
||||
"src/annotations/drawMode.test.mjs",
|
||||
"src/annotations/drawTool.js",
|
||||
"src/annotations/drawTool.test.mjs",
|
||||
"src/annotations/adminOutline.test.mjs",
|
||||
"src/annotations/regionRing.test.mjs",
|
||||
"src/data/adminBoundaries.test.mjs",
|
||||
"scripts/build-admin-packs.mjs",
|
||||
"scripts/qa-admin-outlines.mjs",
|
||||
"src/data/inputOwnership.test.mjs",
|
||||
"docs/CODE-BOUNDARIES.md",
|
||||
"src/main.js",
|
||||
"docs/APPLICATION.md",
|
||||
"scripts/qa-application.mjs",
|
||||
"scripts/qa-voice-auth.mjs",
|
||||
"scripts/qa-voice-auth-focus.mjs",
|
||||
"src/codexOauthRealtime.test.mjs",
|
||||
"src/voice/cloudVoiceAuth.test.mjs",
|
||||
"scripts/qa-panel-resize.mjs",
|
||||
"scripts/qa-panelDrag.mjs",
|
||||
"src/tooling/panelResizeQa.test.mjs",
|
||||
"scripts/qa-terrain-429.mjs",
|
||||
"src/app/application.js",
|
||||
"src/app/application.test.mjs",
|
||||
"src/app/mapLifecycle.test.mjs",
|
||||
@@ -49,6 +62,7 @@
|
||||
"server/standalone/vite.config.js",
|
||||
"vite.config.js",
|
||||
"src/tooling/viteBuild.test.mjs",
|
||||
"src/tooling/allowedHosts.test.mjs",
|
||||
"server/providers/common/http.js",
|
||||
"server/providers/common/query.js",
|
||||
"server/providers/aircraft/opensky.js",
|
||||
@@ -70,6 +84,7 @@
|
||||
"server/providers/places/routes.js",
|
||||
"src/data/placeProviderPayloads.js",
|
||||
"src/tooling/placeProviders.test.mjs",
|
||||
"server/providers/common/same-site.js",
|
||||
"server/providers/space.js",
|
||||
"server/providers/space/celestrak.js",
|
||||
"server/providers/space/launch-library.js",
|
||||
@@ -157,6 +172,7 @@
|
||||
"src/ui/panelRails.js",
|
||||
"src/ui/leftPanelRail.js",
|
||||
"src/ui/rightPanelRail.js",
|
||||
"src/ui/panelResize.test.mjs",
|
||||
"src/ui/panelMeasurement.js",
|
||||
"src/ui/panelRailGeometry.js",
|
||||
"src/ui/panelRails.test.mjs",
|
||||
@@ -263,6 +279,7 @@
|
||||
"src/ui/styles.js",
|
||||
"src/ui/shellLifecycle.test.mjs",
|
||||
"src/testSupport/readStylesheet.mjs",
|
||||
"src/testSupport/streetLevelFakes.mjs",
|
||||
"src/splitFlap.js",
|
||||
"style.css",
|
||||
"src/ui/styles/overlays.css",
|
||||
@@ -514,6 +531,8 @@
|
||||
"src/data/cctvEstonia.test.mjs",
|
||||
"src/data/cctvWarendorf.test.mjs",
|
||||
"src/data/cctvCalgary.test.mjs",
|
||||
"src/data/cctvVegvesen.test.mjs",
|
||||
"src/data/cctvNyc.test.mjs",
|
||||
"src/data/cctvNswSource.test.mjs",
|
||||
"src/data/cctvTxdotSource.test.mjs",
|
||||
"src/layers/cctv/source.test.mjs",
|
||||
@@ -581,6 +600,8 @@
|
||||
"src/maps/imagery.js",
|
||||
"src/maps/registry.js",
|
||||
"src/maps/terrain.js",
|
||||
"src/maps/terrainRetry.js",
|
||||
"src/maps/terrainRetry.test.mjs",
|
||||
"src/mapStackController.js",
|
||||
"src/mapStartup.js",
|
||||
"src/mapStackChips.test.mjs",
|
||||
@@ -844,5 +865,139 @@
|
||||
"src/ui/imageryBoxTool.test.mjs",
|
||||
"src/ui/recentImagery.test.mjs",
|
||||
"src/layers/recentImagery/testDoubles.mjs",
|
||||
"scripts/qa-recent-imagery.mjs"
|
||||
"scripts/qa-recent-imagery.mjs",
|
||||
"src/ui/streetLevelControls.js",
|
||||
"src/ui/streetLevelPresentation.js",
|
||||
"src/ui/streetLevelPresentation.test.mjs",
|
||||
"src/app/layers/streetLevel.js",
|
||||
"src/tooling/mapillaryProvider.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/source.test.mjs",
|
||||
"src/streetLevelMarkup.test.mjs",
|
||||
"src/layers/streetLevel/index.js",
|
||||
"src/layers/streetLevel/policy.js",
|
||||
"src/layers/streetLevel/filter.js",
|
||||
"src/layers/streetLevel/registry.js",
|
||||
"src/layers/streetLevel/params.js",
|
||||
"src/layers/streetLevel/pickRouter.js",
|
||||
"src/layers/streetLevel/uiState.js",
|
||||
"src/layers/streetLevel/groundCast.js",
|
||||
"src/layers/streetLevel/meshSampler.js",
|
||||
"src/layers/streetLevel/view.js",
|
||||
"src/layers/streetLevel/marker.js",
|
||||
"src/layers/streetLevel/cameraFollow.js",
|
||||
"src/layers/streetLevel/credits.js",
|
||||
"src/layers/streetLevel/viewerHost.js",
|
||||
"src/layers/streetLevel/providers/mapillary/index.js",
|
||||
"src/layers/streetLevel/state.js",
|
||||
"src/layers/streetLevel/providers/mapillary/policy.js",
|
||||
"src/layers/streetLevel/providers/mapillary/coverage.js",
|
||||
"src/layers/streetLevel/providers/mapillary/sequences.js",
|
||||
"src/layers/streetLevel/glyphs.js",
|
||||
"src/layers/streetLevel/providers/mapillary/decode.js",
|
||||
"src/layers/streetLevel/tileMath.js",
|
||||
"src/layers/streetLevel/selection.js",
|
||||
"src/layers/streetLevel/providers/mapillary/viewer.js",
|
||||
"src/layers/streetLevel/providers/mapillary/source.js",
|
||||
"src/layers/streetLevel/groundClick.js",
|
||||
"src/layers/streetLevel/providers/google/index.js",
|
||||
"src/layers/streetLevel/providers/google/policy.js",
|
||||
"src/layers/streetLevel/providers/google/viewer.js",
|
||||
"src/layers/streetLevel/providers/google/mapsLoader.js",
|
||||
"src/layers/streetLevel/groundClick.test.mjs",
|
||||
"src/app/layers/streetLevel.test.mjs",
|
||||
"src/layers/streetLevel/providers/google/provider.test.mjs",
|
||||
"src/layers/streetLevel/providers/google/viewer.test.mjs",
|
||||
"src/layers/streetLevel/providers/google/mapsLoader.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/index.js",
|
||||
"src/layers/streetLevel/providers/kartaview/policy.js",
|
||||
"src/layers/streetLevel/providers/kartaview/source.js",
|
||||
"src/layers/streetLevel/providers/kartaview/decode.js",
|
||||
"src/layers/streetLevel/providers/kartaview/viewer.js",
|
||||
"src/layers/streetLevel/providers/kartaview/panorama.js",
|
||||
"src/layers/streetLevel/lineCoverage.js",
|
||||
"src/layers/streetLevel/sequenceCones.js",
|
||||
"src/layers/streetLevel/providers/kartaview/policy.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/decode.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/source.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/panorama.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/viewer.test.mjs",
|
||||
"src/layers/streetLevel/providers/kartaview/provider.test.mjs",
|
||||
"src/layers/streetLevel/lineCoverage.test.mjs",
|
||||
"src/testSupport/kartaViewFakes.mjs",
|
||||
"src/testSupport/googleStreetViewFakes.mjs",
|
||||
"server/providers/mapillary.js",
|
||||
"server/providers/mapillary/routes.js",
|
||||
"server/providers/mapillary/constants.js",
|
||||
"server/providers/mapillary/tiles.js",
|
||||
"server/providers/mapillary/trim.js",
|
||||
"src/layers/streetLevel/tileMath.test.mjs",
|
||||
"scripts/qa-street-level.mjs",
|
||||
"src/tooling/streetLevelQa.test.mjs",
|
||||
"scripts/qa-browserEvidence.mjs",
|
||||
"scripts/fixtures/street-level/mapillaryGraph.mjs",
|
||||
"scripts/fixtures/street-level/kartaView.mjs",
|
||||
"src/layers/streetLevel/filter.test.mjs",
|
||||
"src/layers/streetLevel/registry.test.mjs",
|
||||
"src/layers/streetLevel/params.test.mjs",
|
||||
"src/layers/streetLevel/pickRouter.test.mjs",
|
||||
"src/layers/streetLevel/uiState.test.mjs",
|
||||
"src/layers/streetLevel/groundCast.test.mjs",
|
||||
"src/layers/streetLevel/meshSampler.test.mjs",
|
||||
"src/layers/streetLevel/view.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/provider.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/decode.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/coverage.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/coverageFixture.mjs",
|
||||
"src/layers/streetLevel/cameraFollow.test.mjs",
|
||||
"src/ui/streetLevelControls.test.mjs",
|
||||
"src/ui/panelDock.test.mjs",
|
||||
"src/layers/streetLevel/marker.test.mjs",
|
||||
"src/layers/streetLevel/viewerHost.test.mjs",
|
||||
"src/layers/streetLevel/providers/mapillary/sequences.test.mjs",
|
||||
"src/layers/streetLevel/selection.test.mjs",
|
||||
"src/layers/streetLevel/index.test.mjs",
|
||||
"scripts/qa-overpass-offload.mjs",
|
||||
"src/overpassOffload.test.mjs",
|
||||
"src/sources/openFreeMap.test.mjs",
|
||||
"src/sources/vectorTiles.test.mjs",
|
||||
"src/sources/militaryTileGeometry.test.mjs",
|
||||
"src/layers/traffic/surface.test.mjs",
|
||||
"src/layers/traffic/roadModes.test.mjs",
|
||||
"scripts/qa-traffic-motion.mjs",
|
||||
"src/layers/traffic/motion.test.mjs",
|
||||
"scripts/qa-journey-recorder.mjs",
|
||||
"scripts/qa-alpr-journey.mjs",
|
||||
"scripts/qa-context-arrivals.mjs",
|
||||
"src/tools/catalog.test.mjs",
|
||||
"src/tools/area.test.mjs",
|
||||
"src/tools/queries/queries.test.mjs",
|
||||
"src/tools/mcp/mcp.test.mjs",
|
||||
"src/tools/localServer.test.mjs",
|
||||
"src/tools/mcpPanelKey.test.mjs",
|
||||
"src/tools/queries/aviation.test.mjs",
|
||||
"src/tools/queries/space.test.mjs",
|
||||
"src/tools/queries/media.test.mjs",
|
||||
"src/tools/queries/places.test.mjs",
|
||||
"src/layers/transit/source.test.mjs",
|
||||
"src/tools/queries/mobility.test.mjs",
|
||||
"src/tools/queries/environment.test.mjs",
|
||||
"src/tools/queries/maritime.test.mjs",
|
||||
"src/tools/queries/share.test.mjs",
|
||||
"src/tools/localRoute.test.mjs",
|
||||
"src/tools/queries/atmosphere.test.mjs",
|
||||
"src/tools/queries/imagery.test.mjs",
|
||||
"src/tools/queries/cables.test.mjs",
|
||||
"src/tools/queries/surveillance.test.mjs",
|
||||
"src/tools/queries/infrastructure.test.mjs",
|
||||
"src/tools/queries/events.test.mjs",
|
||||
"src/tools/functions.test.mjs",
|
||||
"src/tools/surfaces.test.mjs",
|
||||
"src/tools/places.test.mjs",
|
||||
"src/view/index.test.mjs",
|
||||
"src/tools/views.test.mjs",
|
||||
"src/app/embed.test.mjs",
|
||||
"src/tools/globePanel.test.mjs",
|
||||
"src/costEndpointThrottle.test.mjs",
|
||||
"src/sources/gbfsStations.test.mjs",
|
||||
"scripts/qa-gbfs-names.mjs"
|
||||
]
|
||||
@@ -0,0 +1,66 @@
|
||||
[
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00000-18dcb239-f29e-5e57-b36f-4bd46c12fb86-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00001-4a0ec645-f2e2-5fd1-aeaf-a2210290fe76-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00002-0bfb9ec4-5947-583a-9b20-5b39a6635e7d-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00003-0aa711cc-b475-5914-9670-7295640f184f-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00004-9a1af7ec-89dd-5aa1-a419-d6325719c330-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00005-e85508c1-3f80-5883-8539-35302da710f3-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00006-184886ec-67b4-5a65-852f-f26f757aaeea-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00007-adbd870a-4116-50a2-9554-57064af19a18-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00008-6362555a-8b0c-5589-a6fe-b71ddca82083-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00009-19bb41f3-6c3b-579a-adae-8f69030c37c7-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00010-0e372a9c-a60d-5b06-89e2-866989d612dd-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00011-2ae04990-7b00-5def-bf72-12b2fdc13b48-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00012-dcfe8859-26a4-544b-ad61-dfbc7e7f272a-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00013-a369ed98-5e3b-5bda-94c9-460f06e8746b-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00014-d03d3cde-6ae3-5938-abe9-b2018f7ab0e3-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00015-57a736a1-7d58-5851-877a-2fca7de484ba-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00016-141b8ad5-f6d4-5f5e-8353-450a53188e27-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00017-14133e77-297b-57f3-b028-c5133bb56578-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00018-65485e6e-8939-51ab-8566-47dd3887b6c3-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00019-64fb52b8-5ba4-5f00-860f-4fe71e1c9f3a-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00020-0feea924-6adb-5118-9a2b-e0a0d3ef6963-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00021-a9115b0c-a202-5468-bfae-0a1e6f953be3-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00022-440ee3b6-f6ae-5bf9-b5d1-cc81fcb0a430-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00023-53a4d462-62b9-589c-9650-f7c9652354e7-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00024-54089af7-32f4-5ad8-8970-cea7350cbb20-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00025-59cc63a7-e7da-58e2-8d06-1df15f3a0636-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00026-ba0e8b28-756c-5b5e-aa01-0939a15e45c8-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00027-8237ebb3-2fcb-50d7-884b-81c8e81bda1e-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00028-6fb209d3-9fbe-5155-9983-896b1bb9e84e-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00029-e86227ec-f5f7-51c6-b537-a10334796cc4-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00030-a2779b0d-003a-5f5d-812c-02eb4a5db71f-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00031-554320d6-a62d-5be7-80b0-d32fb67d87ca-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00032-a49d2756-469a-571f-86ef-be60690cfaea-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00033-0b36146b-6963-5133-8a67-ced0bbb37d6a-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00034-b676aef2-1e90-5966-8531-40f42c5d01d6-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00035-c9b04728-fc59-50f2-bfa8-5f8692ea731e-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00036-22a0fbd7-fd24-5832-ac6c-85e82b56ced6-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00037-d157ddb7-8962-515b-9ec3-d54333fcf836-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00038-39d77c44-55f2-5792-83e1-cac6416d0630-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00039-f6871cf2-753b-53f1-aacf-645ea035826d-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00040-3d142e7a-4189-5e50-9f68-0f77e9a51047-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00041-9e6d1cdd-f130-5089-9d0f-703626552100-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00042-4cefe8fe-4966-5e1e-9cf5-19fd90384db8-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00043-65360671-af63-58dd-9784-bceb07abe219-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00044-b423bff1-dd2b-52c4-90f6-cd1605e4d83f-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00045-1f4d97be-99fc-50de-b9cb-f3624c9a1817-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00046-32a9756a-e3a0-5980-a8d9-88dd475f25b9-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00047-cf80d668-e9f1-5f69-8e21-0a73f1732e5f-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00048-8d932283-4f3f-533d-895b-684faaa9cc34-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00049-996168f3-0c3a-5b58-8444-168c7d859a04-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00050-f9c3c526-3ea5-5dab-a46d-05c623c3e905-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00051-72b4d98d-4b3b-5b3b-a565-62c206d4fa29-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00052-7385fa90-2fa9-53c8-9ed4-96d5703f3a37-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00053-96804709-94fa-56d2-bafd-69b2cbce3881-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00054-520c3c50-82f8-5f3d-97b5-fa8f5f4e65e7-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00055-8742f073-69cd-579e-bf00-b16b6e6a7669-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00056-58083e97-7020-5522-ac19-d8a5b4dcbc98-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00057-bf201c73-1fb3-5dbc-8cec-d344d999820e-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00058-9e1561c3-ba15-528b-8db3-62ce04b1570b-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00059-dd8cd7ca-410f-54f8-9b50-66beb004d640-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00060-203c7177-ec4a-55c2-8003-d828b144a6d6-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00061-0da76969-7fb1-56b9-bb83-4e40f3acea8d-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00062-13344b87-db6c-5415-a48e-afd95a928a85-c000.zstd.parquet",
|
||||
"https://overturemaps-us-west-2.s3.us-west-2.amazonaws.com/release/2026-09-23.1/theme=base/type=land_use/part-00063-a61ab643-210f-518d-b6b8-894f72e5c094-c000.zstd.parquet"
|
||||
]
|
||||
@@ -0,0 +1,11 @@
|
||||
SELECT names.primary AS name, bbox, sources,
|
||||
source_tags['military'] AS military,
|
||||
ST_X(ST_PointOnSurface(geometry)) AS lon,
|
||||
ST_Y(ST_PointOnSurface(geometry)) AS lat,
|
||||
ST_Area_Spheroid(geometry) AS area
|
||||
FROM read_parquet(?)
|
||||
WHERE subtype = 'military'
|
||||
AND nullif(trim(names.primary), '') IS NOT NULL
|
||||
AND (source_tags['landuse'] = 'military'
|
||||
OR source_tags['military'] IN
|
||||
('airfield', 'naval_base', 'range', 'barracks', 'base', 'training_area'))
|
||||
+583
-27
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,655 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* qa-admin-outlines.mjs — bundled state/province/county outlines, driven like a
|
||||
* user would drive them.
|
||||
*
|
||||
* Run: node scripts/qa-admin-outlines.mjs http://localhost:4173 [--headful] [--no-video]
|
||||
*
|
||||
* Journey: frame Texas at state scale and outline "Texas"; pan and zoom toward
|
||||
* Austin in smooth 2–3 s moves; outline "Travis County, Texas"; pan north and
|
||||
* outline the neighbouring "Williamson County, Texas"; zoom out to see both;
|
||||
* then outline "Bavaria" (framed, as a voice request for a distant place is),
|
||||
* and finally Brandenburg (Berlin is a hole) and Hawaii (eight islands).
|
||||
*
|
||||
* Asserts for every outline: it resolves from the bundled packs (no geocode,
|
||||
* boundary or Overpass request), is drawn as a draped outline within the
|
||||
* budget, and holds still — no rebuilt primitives and no new entities while
|
||||
* the camera is still. The whole journey must log zero console/page errors and
|
||||
* send zero requests to any Overpass host or nominatim.openstreetmap.org.
|
||||
*
|
||||
* Evidence in qa-shots/ (gitignored): admin-outlines-<step>.png screenshots,
|
||||
* admin-outlines-journey.mp4 (CDP screencast at ~24 fps, frames re-timed from
|
||||
* their capture timestamps), and admin-outlines-result.json with per-step
|
||||
* timings, frame-time p50/p95 and still-camera frame-difference statistics.
|
||||
* Exits nonzero on any failed assertion.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import sharp from 'sharp';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-admin-outlines.mjs <dev-server-url> [--headful] [--no-video]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const url = args.find((arg) => !arg.startsWith('--'));
|
||||
if (!url || !['http:', 'https:'].includes(new URL(url).protocol))
|
||||
throw new Error('Supply a running dev server URL; see --help');
|
||||
const recordVideo = !args.includes('--no-video');
|
||||
const shots = path.resolve('qa-shots');
|
||||
const framesDir = path.join(shots, 'admin-outlines-frames');
|
||||
await fs.mkdir(shots, { recursive: true });
|
||||
await fs.rm(framesDir, { recursive: true, force: true });
|
||||
await fs.mkdir(framesDir, { recursive: true });
|
||||
|
||||
const OUTLINE_BUDGET_MS = 1500;
|
||||
const STILL_MS = 3000;
|
||||
const FRAME_INTERVAL_S = 1 / 24;
|
||||
|
||||
const result = {
|
||||
url,
|
||||
steps: [],
|
||||
forbiddenRequests: [],
|
||||
lookups: [],
|
||||
packRequests: [],
|
||||
consoleErrors: [],
|
||||
pageErrors: [],
|
||||
screenshots: [],
|
||||
video: null,
|
||||
};
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: !args.includes('--headful'),
|
||||
protocolTimeout: 300_000,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--use-gl=angle',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
],
|
||||
});
|
||||
|
||||
let page;
|
||||
const frames = [];
|
||||
let lastKept = -Infinity;
|
||||
let recording = false;
|
||||
let marker = 'startup';
|
||||
|
||||
try {
|
||||
page = await browser.newPage();
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed');
|
||||
});
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
const client = await page.createCDPSession();
|
||||
await client.send('Network.enable');
|
||||
client.on('Network.requestWillBeSent', ({ request }) => {
|
||||
const target = new URL(request.url);
|
||||
const where = `${request.method} ${target.origin}${target.pathname}`;
|
||||
if (
|
||||
target.hostname.includes('overpass') ||
|
||||
target.hostname === 'nominatim.openstreetmap.org'
|
||||
)
|
||||
result.forbiddenRequests.push({ step: marker, request: where });
|
||||
// Lookups an outline could make: a forward geocode, a place search, an
|
||||
// Overpass boundary query. The HUD's reverse geocode of the camera centre
|
||||
// (latlng=) is not one of them.
|
||||
const local = target.origin === new URL(url).origin;
|
||||
const reverse = target.searchParams.has('latlng');
|
||||
const lookup =
|
||||
!reverse &&
|
||||
((local &&
|
||||
/^\/api\/(overpass|geocode|google\/text-search|nominatim|photon)/.test(
|
||||
target.pathname,
|
||||
)) ||
|
||||
(target.hostname === 'maps.googleapis.com' &&
|
||||
target.pathname.includes('/geocode/')) ||
|
||||
target.hostname === 'photon.komoot.io');
|
||||
if (lookup) result.lookups.push({ step: marker, request: where });
|
||||
if (/\/(states_provinces|countries|counties)\.json$/.test(target.pathname))
|
||||
result.packRequests.push({ step: marker, pack: target.pathname });
|
||||
});
|
||||
page.on('pageerror', (error) =>
|
||||
result.pageErrors.push({ step: marker, message: error.message }),
|
||||
);
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error')
|
||||
result.consoleErrors.push({ step: marker, message: message.text() });
|
||||
});
|
||||
|
||||
client.on('Page.screencastFrame', async ({ data, metadata, sessionId }) => {
|
||||
client.send('Page.screencastFrameAck', { sessionId }).catch(() => {});
|
||||
if (!recording) return;
|
||||
const t = metadata.timestamp;
|
||||
if (t - lastKept < FRAME_INTERVAL_S * 0.9) return;
|
||||
lastKept = t;
|
||||
const file = path.join(
|
||||
framesDir,
|
||||
`${String(frames.length).padStart(6, '0')}.jpg`,
|
||||
);
|
||||
frames.push({ file, t, step: marker });
|
||||
await fs.writeFile(file, Buffer.from(data, 'base64'));
|
||||
});
|
||||
|
||||
console.log('Loading viewer...');
|
||||
const navigationUrl = new URL(url);
|
||||
navigationUrl.searchParams.set('welcome', '0');
|
||||
await page.goto(navigationUrl.href, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
timeout: 60_000,
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => window.__godsEyeView?.viewer && window.__godsEyeView?.annotations,
|
||||
{ timeout: 90_000, polling: 500 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 90_000, polling: 250 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector('#first-run-launcher:not([hidden])'),
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
result.renderer = await page.evaluate(() => {
|
||||
const gl = window.__godsEyeView.viewer.scene.context._gl;
|
||||
const debug = gl.getExtension('WEBGL_debug_renderer_info');
|
||||
return debug
|
||||
? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL)
|
||||
: gl.getParameter(gl.RENDERER);
|
||||
});
|
||||
console.log('Renderer:', result.renderer);
|
||||
|
||||
// In-page helpers: camera moves, frame timing and annotation state.
|
||||
await page.evaluate(() => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
const Cesium = window.__CESIUM__;
|
||||
window.__qaOutlines = {
|
||||
fly(view, duration) {
|
||||
return new Promise((resolve) => {
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.flyTo({
|
||||
destination: Cesium.Cartesian3.fromDegrees(
|
||||
view.lon,
|
||||
view.lat,
|
||||
view.height,
|
||||
),
|
||||
orientation: {
|
||||
heading: Cesium.Math.toRadians(view.heading ?? 0),
|
||||
pitch: Cesium.Math.toRadians(view.pitch ?? -70),
|
||||
roll: 0,
|
||||
},
|
||||
duration,
|
||||
easingFunction: Cesium.EasingFunction.QUADRATIC_IN_OUT,
|
||||
complete: resolve,
|
||||
cancel: resolve,
|
||||
});
|
||||
});
|
||||
},
|
||||
frameTimes(ms) {
|
||||
return new Promise((resolve) => {
|
||||
const deltas = [];
|
||||
let last = performance.now();
|
||||
const end = last + ms;
|
||||
const step = (now) => {
|
||||
deltas.push(now - last);
|
||||
last = now;
|
||||
if (now < end) requestAnimationFrame(step);
|
||||
else resolve(deltas.slice(1));
|
||||
};
|
||||
requestAnimationFrame(step);
|
||||
});
|
||||
},
|
||||
annotationSource() {
|
||||
for (let i = 0; i < viewer.dataSources.length; i++) {
|
||||
const source = viewer.dataSources.get(i);
|
||||
if (source.name === 'gev-annotations') return source;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
groundPrimitives() {
|
||||
const list = [];
|
||||
const walk = (collection) => {
|
||||
for (let i = 0; i < collection.length; i++) {
|
||||
const item = collection.get(i);
|
||||
if (item?.length !== undefined && typeof item.get === 'function')
|
||||
walk(item);
|
||||
else list.push(item);
|
||||
}
|
||||
};
|
||||
walk(viewer.scene.groundPrimitives);
|
||||
return list;
|
||||
},
|
||||
snapshot() {
|
||||
const source = this.annotationSource();
|
||||
const entities = source ? source.entities.values : [];
|
||||
const primitives = this.groundPrimitives();
|
||||
window.__qaPrimitiveIds ??= new WeakMap();
|
||||
let next = window.__qaPrimitiveNext ?? 1;
|
||||
const ids = primitives.map((primitive) => {
|
||||
if (!window.__qaPrimitiveIds.has(primitive))
|
||||
window.__qaPrimitiveIds.set(primitive, next++);
|
||||
return window.__qaPrimitiveIds.get(primitive);
|
||||
});
|
||||
window.__qaPrimitiveNext = next;
|
||||
return {
|
||||
entities: entities.length,
|
||||
polygons: entities.filter((e) => e.polygon).length,
|
||||
polylines: entities.filter((e) => e.polyline).length,
|
||||
draped:
|
||||
entities.every(
|
||||
(e) =>
|
||||
!e.polygon ||
|
||||
e.polygon.classificationType?.getValue() ===
|
||||
Cesium.ClassificationType.BOTH,
|
||||
) &&
|
||||
entities.every(
|
||||
(e) => !e.polyline || e.polyline.clampToGround?.getValue(),
|
||||
),
|
||||
primitiveIds: ids,
|
||||
ready: viewer.dataSourceDisplay.ready,
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const settle = () =>
|
||||
page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const { scene } = window.__godsEyeView.viewer;
|
||||
for (let i = 0; i < scene.primitives.length; i++) {
|
||||
const p = scene.primitives.get(i);
|
||||
if (p?.show && typeof p.tilesLoaded === 'boolean' && !p.tilesLoaded)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{ timeout: 60_000, polling: 500 },
|
||||
)
|
||||
.catch(() => {});
|
||||
|
||||
async function shot(name) {
|
||||
await page.evaluate(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(resolve)),
|
||||
),
|
||||
);
|
||||
const filename = `admin-outlines-${name}.png`;
|
||||
await page.screenshot({ path: path.join(shots, filename) });
|
||||
result.screenshots.push(filename);
|
||||
}
|
||||
|
||||
async function move(name, view, duration = 2.5) {
|
||||
marker = `move:${name}`;
|
||||
const [, times] = await Promise.all([
|
||||
page.evaluate((v, d) => window.__qaOutlines.fly(v, d), view, duration),
|
||||
page.evaluate((ms) => window.__qaOutlines.frameTimes(ms), duration * 900),
|
||||
]);
|
||||
result.steps.push({ step: `move:${name}`, frameTimes: stats(times) });
|
||||
}
|
||||
|
||||
/** Annotate like the voice tool does, then watch the mark while still. */
|
||||
async function outline(name, target, { flyTo = false } = {}) {
|
||||
marker = `outline:${name}`;
|
||||
const lookupsBefore = result.lookups.length;
|
||||
const errorsBefore = result.consoleErrors.length + result.pageErrors.length;
|
||||
const timing = await page.evaluate(
|
||||
async (target, flyTo) => {
|
||||
const started = performance.now();
|
||||
const response = await window.__godsEyeView.annotations.annotate(
|
||||
[{ type: 'area', target, label: target, footprint: true }],
|
||||
{ flyTo },
|
||||
);
|
||||
const resolvedMs = performance.now() - started;
|
||||
// "Drawn" = the first rendered frame after the entity visualizers have
|
||||
// built the mark's primitives (each tick updates them before render).
|
||||
const { viewer } = window.__godsEyeView;
|
||||
await new Promise((resolve) => {
|
||||
const off = viewer.scene.postRender.addEventListener(() => {
|
||||
if (!viewer.dataSourceDisplay.ready) return;
|
||||
off();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const drawnMs = performance.now() - started;
|
||||
const mark = window.__godsEyeView.annotations
|
||||
.list()
|
||||
.find((m) => m.id === response.ids?.[0]);
|
||||
return {
|
||||
response: response.results?.[0] ?? null,
|
||||
resolvedMs,
|
||||
drawnMs,
|
||||
parts: mark?.polygons?.length ?? (mark?.ring ? 1 : 0),
|
||||
holes: (mark?.polygons ?? []).reduce(
|
||||
(n, part) => n + part.length - 1,
|
||||
0,
|
||||
),
|
||||
ringVertices: mark?.ring?.length ?? 0,
|
||||
pending: Boolean(mark?.pendingOutline),
|
||||
};
|
||||
},
|
||||
target,
|
||||
flyTo,
|
||||
);
|
||||
if (flyTo) await new Promise((r) => setTimeout(r, 2200));
|
||||
await settle();
|
||||
// Hold still: the drawn mark must not rebuild or grow.
|
||||
marker = `still:${name}`;
|
||||
const before = await page.evaluate(() => window.__qaOutlines.snapshot());
|
||||
const stillStart = frames.length;
|
||||
const times = await page.evaluate(
|
||||
(ms) => window.__qaOutlines.frameTimes(ms),
|
||||
STILL_MS,
|
||||
);
|
||||
const after = await page.evaluate(() => window.__qaOutlines.snapshot());
|
||||
const step = {
|
||||
step: `outline:${name}`,
|
||||
target,
|
||||
...timing,
|
||||
entities: after.entities,
|
||||
polygons: after.polygons,
|
||||
polylines: after.polylines,
|
||||
draped: after.draped,
|
||||
stableWhileStill:
|
||||
before.entities === after.entities &&
|
||||
JSON.stringify(before.primitiveIds) ===
|
||||
JSON.stringify(after.primitiveIds),
|
||||
stillFrameTimes: stats(times),
|
||||
stillFrames: [stillStart, frames.length],
|
||||
lookups: result.lookups.slice(lookupsBefore),
|
||||
errors:
|
||||
result.consoleErrors.length + result.pageErrors.length - errorsBefore,
|
||||
};
|
||||
result.steps.push(step);
|
||||
console.log(
|
||||
`${name}: resolved ${timing.resolvedMs.toFixed(0)} ms, drawn ${timing.drawnMs.toFixed(0)} ms, ` +
|
||||
`${timing.parts} part(s), ${timing.holes} hole(s), via ${timing.response?.resolvedVia}; ` +
|
||||
`still p50/p95 ${step.stillFrameTimes.p50}/${step.stillFrameTimes.p95} ms; ` +
|
||||
`stable=${step.stableWhileStill}`,
|
||||
);
|
||||
assert.equal(timing.response?.ok, true, `${name}: annotation drawn`);
|
||||
assert.equal(timing.response.outline, true, `${name}: outline, not a pin`);
|
||||
assert.equal(timing.response.resolvedVia, 'bundled', `${name}: bundled`);
|
||||
assert.equal(timing.pending, false, `${name}: nothing pending`);
|
||||
assert.ok(
|
||||
timing.drawnMs < OUTLINE_BUDGET_MS,
|
||||
`${name}: drawn in ${timing.drawnMs} ms`,
|
||||
);
|
||||
assert.equal(step.draped, true, `${name}: draped on terrain/tiles`);
|
||||
assert.equal(step.stableWhileStill, true, `${name}: no redraw when still`);
|
||||
assert.deepEqual(step.lookups, [], `${name}: no network lookups`);
|
||||
assert.equal(step.errors, 0, `${name}: no console errors`);
|
||||
return step;
|
||||
}
|
||||
|
||||
// Start from a clean board and the default layers, as a user would.
|
||||
await page.evaluate(() => window.__godsEyeView.annotations.clear?.());
|
||||
await page.evaluate(() =>
|
||||
window.__qaOutlines.fly(
|
||||
{ lat: 25.5, lon: -99.6, height: 2_600_000, pitch: -80 },
|
||||
0,
|
||||
),
|
||||
);
|
||||
await settle();
|
||||
if (recordVideo) {
|
||||
await client.send('Page.startScreencast', {
|
||||
format: 'jpeg',
|
||||
quality: 72,
|
||||
maxWidth: 1280,
|
||||
maxHeight: 800,
|
||||
everyNthFrame: 1,
|
||||
});
|
||||
recording = true;
|
||||
}
|
||||
|
||||
await outline('texas', 'Texas');
|
||||
await shot('texas');
|
||||
await move('pan-west', {
|
||||
lat: 25.2,
|
||||
lon: -101.5,
|
||||
height: 2_300_000,
|
||||
pitch: -75,
|
||||
});
|
||||
await move('zoom-central-texas', {
|
||||
lat: 28.8,
|
||||
lon: -98.2,
|
||||
height: 700_000,
|
||||
pitch: -65,
|
||||
});
|
||||
await move('zoom-austin', {
|
||||
lat: 29.75,
|
||||
lon: -97.8,
|
||||
height: 120_000,
|
||||
pitch: -55,
|
||||
});
|
||||
await settle();
|
||||
await outline('travis', 'Travis County, Texas');
|
||||
await shot('travis');
|
||||
await move('pan-williamson', {
|
||||
lat: 30.05,
|
||||
lon: -97.65,
|
||||
height: 110_000,
|
||||
pitch: -55,
|
||||
});
|
||||
await settle();
|
||||
await outline('williamson', 'Williamson County, Texas');
|
||||
// Close over the shared Travis/Williamson line near Pflugerville: the
|
||||
// outline must lie on the photoreal mesh, not float above or sink into it.
|
||||
const border = await page.evaluate(() => {
|
||||
const travis = window.__godsEyeView.annotations
|
||||
.list()
|
||||
.find((m) => m.label === 'Travis County, Texas');
|
||||
let best = null;
|
||||
for (const [lon, lat] of travis.ring) {
|
||||
const d = (lat - 30.46) ** 2 + (lon + 97.62) ** 2;
|
||||
if (!best || d < best.d) best = { lon, lat, d };
|
||||
}
|
||||
return best;
|
||||
});
|
||||
await move(
|
||||
'drape-close',
|
||||
{ lat: border.lat - 0.035, lon: border.lon, height: 2500, pitch: -40 },
|
||||
3,
|
||||
);
|
||||
await settle();
|
||||
await page.evaluate((ms) => new Promise((r) => setTimeout(r, ms)), 2500);
|
||||
await shot('drape-close');
|
||||
await move('zoom-out-both', {
|
||||
lat: 29.55,
|
||||
lon: -97.7,
|
||||
height: 260_000,
|
||||
pitch: -60,
|
||||
});
|
||||
await settle();
|
||||
await page.evaluate((ms) => new Promise((r) => setTimeout(r, ms)), 1500);
|
||||
await shot('both-counties');
|
||||
await outline('bavaria', 'Bavaria', { flyTo: true });
|
||||
await shot('bavaria');
|
||||
await move(
|
||||
'orbit-bavaria',
|
||||
{ lat: 46.6, lon: 11.9, height: 650_000, heading: 20, pitch: -55 },
|
||||
3,
|
||||
);
|
||||
await settle();
|
||||
await shot('bavaria-orbit');
|
||||
// A hole (Berlin inside Brandenburg) and islands (Hawaii) draw as such.
|
||||
const brandenburg = await outline('brandenburg', 'Brandenburg', {
|
||||
flyTo: true,
|
||||
});
|
||||
assert.ok(brandenburg.holes >= 1, 'Brandenburg keeps the Berlin hole');
|
||||
await shot('brandenburg');
|
||||
const hawaii = await outline('hawaii', 'Hawaii', { flyTo: true });
|
||||
assert.ok(hawaii.parts >= 7, 'Hawaii keeps its islands');
|
||||
await shot('hawaii');
|
||||
marker = 'end';
|
||||
|
||||
for (const [name, target, lat, lon, height] of [
|
||||
['scotland', 'Scotland', 57, -4, 800000],
|
||||
['england', 'England', 52.5, -1.5, 800000],
|
||||
['wales', 'Wales', 52.3, -3.7, 420000],
|
||||
['northern-ireland', 'Northern Ireland', 54.7, -6.7, 300000],
|
||||
['switzerland', 'Switzerland', 46.8, 8.2, 500000],
|
||||
['iran', 'Iran', 32, 54, 2200000],
|
||||
['france', 'France', 46.6, 2.2, 1600000],
|
||||
['japan', 'Japan', 37, 138, 2200000],
|
||||
['georgia-state', 'Georgia', 32.6, -83.4, 650000],
|
||||
['georgia-country', 'Georgia', 42, 43.6, 700000],
|
||||
['georgia-qualified', 'the country of Georgia', 42, 43.6, 700000],
|
||||
]) {
|
||||
await page.evaluate(() => window.__godsEyeView.annotations.clear());
|
||||
await move(name, { lat, lon, height, pitch: -90 }, 1);
|
||||
await outline(name, target);
|
||||
await shot(name);
|
||||
const rendered = await page.evaluate(() => window.__qaOutlines.snapshot());
|
||||
assert.ok(rendered.polygons > 0, `${target} polygon renders`);
|
||||
}
|
||||
|
||||
if (recordVideo) {
|
||||
recording = false;
|
||||
await client.send('Page.stopScreencast');
|
||||
}
|
||||
result.totals = {
|
||||
forbiddenRequests: result.forbiddenRequests.length,
|
||||
consoleErrors: result.consoleErrors.length,
|
||||
pageErrors: result.pageErrors.length,
|
||||
};
|
||||
assert.deepEqual(
|
||||
result.forbiddenRequests,
|
||||
[],
|
||||
'no Overpass or nominatim.openstreetmap.org requests',
|
||||
);
|
||||
// The packs load on the first outline that needs them, never at start-up.
|
||||
assert.deepEqual(
|
||||
result.packRequests.map((r) => [r.step, path.basename(r.pack)]),
|
||||
[
|
||||
['outline:texas', 'states_provinces.json'],
|
||||
['outline:travis', 'counties.json'],
|
||||
['outline:scotland', 'countries.json'],
|
||||
],
|
||||
'packs load lazily, once each',
|
||||
);
|
||||
assert.deepEqual(result.pageErrors, [], 'no page errors');
|
||||
assert.deepEqual(result.consoleErrors, [], 'no console errors');
|
||||
} catch (error) {
|
||||
result.failure = error.message;
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
if (page)
|
||||
await page
|
||||
.screenshot({ path: path.join(shots, 'admin-outlines-failure.png') })
|
||||
.catch(() => {});
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
if (recordVideo && frames.length > 1) {
|
||||
// Frame-to-frame change while the camera is still: steady pulses read as
|
||||
// small, even values; a flicker (a part vanishing and returning) as a spike.
|
||||
for (const step of result.steps.filter((s) => s.stillFrames)) {
|
||||
const [start, end] = step.stillFrames;
|
||||
const diffs = [];
|
||||
let previous = null;
|
||||
for (const frame of frames.slice(start, end)) {
|
||||
const pixels = await sharp(frame.file)
|
||||
.resize(160, 100, { fit: 'fill' })
|
||||
.greyscale()
|
||||
.raw()
|
||||
.toBuffer();
|
||||
if (previous) {
|
||||
let sum = 0;
|
||||
let max = 0;
|
||||
for (let i = 0; i < pixels.length; i++) {
|
||||
const d = Math.abs(pixels[i] - previous[i]);
|
||||
sum += d;
|
||||
if (d > max) max = d;
|
||||
}
|
||||
diffs.push(sum / pixels.length);
|
||||
}
|
||||
previous = pixels;
|
||||
}
|
||||
step.stillFrameDiff = {
|
||||
frames: end - start,
|
||||
meanAbsDiff: stats(diffs, 3),
|
||||
};
|
||||
}
|
||||
const list = [];
|
||||
for (let i = 0; i < frames.length; i++) {
|
||||
const duration =
|
||||
i + 1 < frames.length ? frames[i + 1].t - frames[i].t : FRAME_INTERVAL_S;
|
||||
list.push(`file '${frames[i].file}'`, `duration ${duration.toFixed(4)}`);
|
||||
}
|
||||
list.push(`file '${frames.at(-1).file}'`);
|
||||
const listFile = path.join(framesDir, 'frames.txt');
|
||||
await fs.writeFile(listFile, list.join('\n') + '\n');
|
||||
const video = path.join(shots, 'admin-outlines-journey.mp4');
|
||||
execFileSync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-f',
|
||||
'concat',
|
||||
'-safe',
|
||||
'0',
|
||||
'-i',
|
||||
listFile,
|
||||
'-vf',
|
||||
'fps=24,format=yuv420p',
|
||||
'-c:v',
|
||||
'libx264',
|
||||
'-crf',
|
||||
'23',
|
||||
video,
|
||||
]);
|
||||
const span = frames.at(-1).t - frames[0].t;
|
||||
result.video = {
|
||||
file: path.relative(process.cwd(), video),
|
||||
frames: frames.length,
|
||||
seconds: Number(span.toFixed(1)),
|
||||
capturedFps: Number(((frames.length - 1) / span).toFixed(1)),
|
||||
};
|
||||
console.log('Video:', result.video);
|
||||
await fs.rm(framesDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(shots, 'admin-outlines-result.json'),
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
console.table(
|
||||
result.steps.map((s) => ({
|
||||
step: s.step,
|
||||
resolvedMs: s.resolvedMs?.toFixed(0),
|
||||
drawnMs: s.drawnMs?.toFixed(0),
|
||||
parts: s.parts,
|
||||
stable: s.stableWhileStill,
|
||||
p50: (s.stillFrameTimes || s.frameTimes)?.p50,
|
||||
p95: (s.stillFrameTimes || s.frameTimes)?.p95,
|
||||
stillDiffP95: s.stillFrameDiff?.meanAbsDiff.p95,
|
||||
stillDiffMax: s.stillFrameDiff?.meanAbsDiff.max,
|
||||
})),
|
||||
);
|
||||
console.log(process.exitCode ? 'FAILED' : 'PASSED');
|
||||
|
||||
function stats(values, digits = 1) {
|
||||
if (!values.length) return { n: 0 };
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const at = (q) =>
|
||||
Number(
|
||||
sorted[
|
||||
Math.min(sorted.length - 1, Math.floor(q * sorted.length))
|
||||
].toFixed(digits),
|
||||
);
|
||||
return { n: values.length, p50: at(0.5), p95: at(0.95), max: at(1) };
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* qa-alpr-journey.mjs — ALPR cameras on a realistic Austin camera journey.
|
||||
*
|
||||
* Run: node scripts/qa-alpr-journey.mjs http://localhost:4173 [--headful]
|
||||
* [--passes off,on,on] [--out qa-shots/alpr-journey] [--no-assert]
|
||||
*
|
||||
* Drives the same eased camera path several times: a neighbourhood by Camp
|
||||
* Mabry (~2 km across), two 600 m pans, a small zoom out and back, downtown
|
||||
* (~6 km), the whole city (~30 km) and back to the neighbourhood. The first
|
||||
* pass runs with Mapped ALPR Cameras off (it also warms the photoreal tiles),
|
||||
* then ALPR is switched on mid-journey from its layer row and the path repeats
|
||||
* cold and warm.
|
||||
*
|
||||
* Per stop it records: time from the camera stopping to cameras shown (or an
|
||||
* explicit "zoom in" state), the camera count and row status, rAF interval and
|
||||
* Cesium render CPU time (mean / p50 / p95) while moving and while holding, and
|
||||
* marker stability while the camera is still (camera-set changes after the stop
|
||||
* has settled). Asserts, unless --no-assert: neighbourhood and downtown show
|
||||
* cameras within 2 s on the warm pass; the whole-city stop shows cameras or a
|
||||
* zoom-in prompt and never ends a hold still loading; at most one visible
|
||||
* marker update per stop and none after 3 s while still; warm-pass frame p50/p95 within 10% of the ALPR-off pass (plus a
|
||||
* 2 ms floor for timer noise); zero console errors.
|
||||
*
|
||||
* Rendered-state checks read the recorded frames, not entity state: ALPR-cyan
|
||||
* pixels per frame during each hold, minus the ALPR-off pass at the same stop,
|
||||
* give when markers actually reached the screen. Cameras projected on screen
|
||||
* must be painted, no later than 700 ms after the layer state settled (a
|
||||
* missing render request fails here), within 2 s of the stop on the warm pass;
|
||||
* a view whose loaded cameras are all off screen must say "None on screen".
|
||||
* At the end of each hold the screen is compared with the same scene after
|
||||
* forcing four frames; any marker difference is a missing render request.
|
||||
* --self-test-drop-renders disables explicit render requests during the ALPR
|
||||
* passes to prove those checks fail.
|
||||
*
|
||||
* Writes <out>/result.json, <out>/clip.mp4 (CDP screencast with real frame
|
||||
* timing) and per-stop screenshots. Needs ffmpeg on PATH.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import sharp from 'sharp';
|
||||
import {
|
||||
createScreencast,
|
||||
frameStats,
|
||||
installFrameProbe,
|
||||
moveCamera,
|
||||
pageNow,
|
||||
readFrames,
|
||||
} from './qa-journey-recorder.mjs';
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes('--help') || argv.includes('-h')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-alpr-journey.mjs <dev-server-url> [--headful] [--passes off,on,on] [--out dir] [--no-assert] [--self-test-drop-renders]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const option = (name, fallback) => {
|
||||
const i = argv.indexOf(name);
|
||||
return i >= 0 && argv[i + 1] ? argv[i + 1] : fallback;
|
||||
};
|
||||
const appUrl =
|
||||
argv.find((arg) => /^https?:/.test(arg)) || 'http://localhost:4173';
|
||||
const outDir = path.resolve(option('--out', 'qa-shots/alpr-journey'));
|
||||
const passes = option('--passes', 'off,on,on').split(',');
|
||||
const assertOn = !argv.includes('--no-assert');
|
||||
const dropRenders = argv.includes('--self-test-drop-renders');
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const cyanCount = async (file) => {
|
||||
const { data, info } = await sharp(file)
|
||||
.resize({ width: 640 })
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
let n = 0;
|
||||
for (let i = 0; i < data.length; i += info.channels) {
|
||||
const r = data[i];
|
||||
const g = data[i + 1];
|
||||
const b = data[i + 2];
|
||||
if (r < 140 && g > 160 && b > 205 && b - r > 80) n += 1;
|
||||
}
|
||||
return n;
|
||||
};
|
||||
|
||||
/** Cyan pixels (at 640 px width) that mean at least one badge is painted. */
|
||||
const MIN_MARKER_PIXELS = 12;
|
||||
|
||||
// Camp Mabry neighbourhood (Tarrytown / Bryker Woods), downtown, whole city.
|
||||
const N = {
|
||||
lat: 30.2985,
|
||||
lon: -97.7655,
|
||||
height: 1500,
|
||||
heading: 20,
|
||||
pitch: -55,
|
||||
};
|
||||
const east = (view, m) => ({
|
||||
...view,
|
||||
lon: view.lon + m / (111_320 * Math.cos((view.lat * Math.PI) / 180)),
|
||||
});
|
||||
const north = (view, m) => ({ ...view, lat: view.lat + m / 111_320 });
|
||||
const STOPS = [
|
||||
{ name: 'neighbourhood', view: N, move: 0, hold: 6, scale: 'neighbourhood' },
|
||||
{
|
||||
name: 'pan-east-600m',
|
||||
view: east(N, 600),
|
||||
move: 2.5,
|
||||
hold: 5,
|
||||
scale: 'neighbourhood',
|
||||
},
|
||||
{
|
||||
name: 'pan-north-600m',
|
||||
view: north(east(N, 600), 600),
|
||||
move: 2.5,
|
||||
hold: 5,
|
||||
scale: 'neighbourhood',
|
||||
},
|
||||
{
|
||||
name: 'zoom-out-3km',
|
||||
view: { ...north(east(N, 600), 600), height: 3000 },
|
||||
move: 2.5,
|
||||
hold: 5,
|
||||
scale: 'neighbourhood',
|
||||
},
|
||||
{
|
||||
name: 'zoom-back-in',
|
||||
view: north(east(N, 600), 600),
|
||||
move: 2.5,
|
||||
hold: 5,
|
||||
scale: 'neighbourhood',
|
||||
},
|
||||
{
|
||||
name: 'downtown-6km',
|
||||
view: { lat: 30.262, lon: -97.7431, height: 4500, heading: 0, pitch: -60 },
|
||||
move: 3,
|
||||
hold: 7,
|
||||
scale: 'downtown',
|
||||
},
|
||||
{
|
||||
name: 'city-30km',
|
||||
view: { lat: 30.2, lon: -97.75, height: 24000, heading: 0, pitch: -65 },
|
||||
move: 3,
|
||||
hold: 8,
|
||||
scale: 'city',
|
||||
},
|
||||
{
|
||||
name: 'back-to-neighbourhood',
|
||||
view: N,
|
||||
move: 3,
|
||||
hold: 6,
|
||||
scale: 'neighbourhood',
|
||||
},
|
||||
];
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: !argv.includes('--headful'),
|
||||
protocolTimeout: 300_000,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
...(process.platform === 'darwin'
|
||||
? ['--use-angle=metal', '--enable-gpu']
|
||||
: ['--use-gl=angle', '--use-angle=swiftshader']),
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
],
|
||||
});
|
||||
const result = {
|
||||
url: appUrl,
|
||||
startedAt: new Date().toISOString(),
|
||||
passes: [],
|
||||
consoleErrors: [],
|
||||
failures: [],
|
||||
};
|
||||
const fail = (message) => {
|
||||
result.failures.push(message);
|
||||
console.log(`[FAIL] ${message}`);
|
||||
};
|
||||
let page;
|
||||
try {
|
||||
page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') result.consoleErrors.push(message.text());
|
||||
});
|
||||
page.on('pageerror', (error) => result.consoleErrors.push(error.message));
|
||||
const url = new URL(appUrl);
|
||||
url.searchParams.set('welcome', '0');
|
||||
await page.goto(url.href, { waitUntil: 'domcontentloaded', timeout: 90_000 });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120_000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await installFrameProbe(page);
|
||||
// ALPR watcher: row state every 100 ms and the rendered camera set (ids and
|
||||
// visibility) as a cheap hash, so structural churn while still is visible.
|
||||
await page.evaluate(() => {
|
||||
const samples = [];
|
||||
window.__gevAlprSamples = samples;
|
||||
setInterval(() => {
|
||||
const entry = window.__godsEyeView.dataManager.layers.get('alpr-cameras');
|
||||
const stats = entry?.module?.getStats?.() || {};
|
||||
// Layers that publish a render revision are measured by it; otherwise
|
||||
// hash the rendered entity ids and their visibility.
|
||||
let hash = 0;
|
||||
let shown = 0;
|
||||
if (Number.isFinite(stats.renderRevision)) {
|
||||
hash = stats.renderRevision;
|
||||
shown = stats.shown ?? stats.count ?? 0;
|
||||
} else {
|
||||
const entities =
|
||||
entry?.module && entry.enabled
|
||||
? window.__godsEyeView.viewer.dataSources.getByName(
|
||||
'alpr-cameras',
|
||||
)[0]?.entities.values || []
|
||||
: [];
|
||||
for (const entity of entities) {
|
||||
if (entity.show === false) continue;
|
||||
shown += 1;
|
||||
const id = String(entity.id);
|
||||
for (let i = 0; i < id.length; i++)
|
||||
hash = (hash * 31 + id.charCodeAt(i)) | 0;
|
||||
}
|
||||
}
|
||||
samples.push({
|
||||
t: performance.now(),
|
||||
enabled: Boolean(entry?.enabled),
|
||||
count: stats.count ?? 0,
|
||||
shown,
|
||||
hash,
|
||||
loading: Boolean(stats.loading),
|
||||
status: stats.status || null,
|
||||
label: stats.loadingLabel || '',
|
||||
onScreen: stats.onScreen ?? null,
|
||||
});
|
||||
if (samples.length > 20_000) samples.splice(0, 5_000);
|
||||
}, 100);
|
||||
});
|
||||
const recorder = await createScreencast(page, outDir);
|
||||
await recorder.start();
|
||||
|
||||
async function waitTiles(maxMs = 20_000) {
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const { scene } = window.__godsEyeView.viewer;
|
||||
if (scene.globe.show) return scene.globe.tilesLoaded;
|
||||
for (let i = 0; i < scene.primitives.length; i++) {
|
||||
const p = scene.primitives.get(i);
|
||||
if (p.show && p.tilesLoaded === false) return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{ timeout: maxMs, polling: 250 },
|
||||
)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
for (const [passIndex, mode] of passes.entries()) {
|
||||
const alpr = mode === 'on';
|
||||
const passName = `${passIndex + 1}-${mode}${alpr && passes.slice(0, passIndex).includes('on') ? '-warm' : alpr ? '-cold' : ''}`;
|
||||
console.log(`Pass ${passName}`);
|
||||
await moveCamera(page, STOPS[0].view, 0);
|
||||
await waitTiles();
|
||||
if (
|
||||
alpr !==
|
||||
(await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.isEnabled('alpr-cameras'),
|
||||
))
|
||||
) {
|
||||
// The user's switch: the layer row, not a private API.
|
||||
await page.evaluate(
|
||||
(enabled) =>
|
||||
window.__godsEyeView.dataManager.setEnabled('alpr-cameras', enabled, {
|
||||
origin: 'user',
|
||||
}),
|
||||
alpr,
|
||||
);
|
||||
}
|
||||
if (alpr && dropRenders)
|
||||
// Self-test: drop explicit render requests so markers reach the screen
|
||||
// only with the next camera move. The rendered-pixel checks must fail.
|
||||
await page.evaluate(() => {
|
||||
window.__godsEyeView.viewer.scene.requestRender = () => {};
|
||||
});
|
||||
const pass = { name: passName, alpr, stops: [] };
|
||||
for (const stop of STOPS) {
|
||||
const moveStart = await pageNow(page);
|
||||
if (stop.move) await moveCamera(page, stop.view, stop.move);
|
||||
else await moveCamera(page, stop.view, 0);
|
||||
const holdStart = await pageNow(page);
|
||||
const holdStartEpoch = await page.evaluate(
|
||||
() => performance.timeOrigin + performance.now(),
|
||||
);
|
||||
await sleep(stop.hold * 1000);
|
||||
const holdEnd = await pageNow(page);
|
||||
// Render truth: what the screen shows at the end of the hold versus the
|
||||
// same scene after forcing a few frames. A difference means the layer
|
||||
// changed state without asking for a render.
|
||||
let staleFrame = null;
|
||||
if (alpr) {
|
||||
const shown = await cyanCount(await page.screenshot({ type: 'png' }));
|
||||
await page.evaluate(async () => {
|
||||
const { scene } = window.__godsEyeView.viewer;
|
||||
const mode = scene.requestRenderMode;
|
||||
scene.requestRenderMode = false;
|
||||
for (let i = 0; i < 4; i++)
|
||||
await new Promise((resolve) => requestAnimationFrame(resolve));
|
||||
scene.requestRenderMode = mode;
|
||||
});
|
||||
const forced = await cyanCount(await page.screenshot({ type: 'png' }));
|
||||
staleFrame = {
|
||||
shown,
|
||||
forced,
|
||||
stale: Math.abs(forced - shown) > Math.max(8, forced * 0.15),
|
||||
};
|
||||
}
|
||||
const moving = frameStats(await readFrames(page, moveStart, holdStart));
|
||||
const holding = frameStats(await readFrames(page, holdStart, holdEnd));
|
||||
const samples = await page.evaluate(
|
||||
(a, b) => window.__gevAlprSamples.filter((s) => s.t >= a && s.t < b),
|
||||
holdStart,
|
||||
holdEnd,
|
||||
);
|
||||
// Time to cameras: the last visible change (or the end of loading) after
|
||||
// the camera stopped. More than one visible change, or any change after
|
||||
// 3 s, is flicker: a still camera must see one clean update at most.
|
||||
let visualChanges = 0;
|
||||
const changeLog = [];
|
||||
let lastChangeAt = null;
|
||||
let loadingEndAt = null;
|
||||
let wasLoading = false;
|
||||
let previous = null;
|
||||
for (const s of samples) {
|
||||
if (
|
||||
previous &&
|
||||
(s.hash !== previous.hash || s.shown !== previous.shown)
|
||||
) {
|
||||
visualChanges += 1;
|
||||
lastChangeAt = s.t;
|
||||
changeLog.push({
|
||||
atMs: Math.round(s.t - holdStart),
|
||||
shown: s.shown,
|
||||
loading: s.loading,
|
||||
});
|
||||
}
|
||||
if (s.loading) wasLoading = true;
|
||||
else if (wasLoading && loadingEndAt === null) loadingEndAt = s.t;
|
||||
previous = s;
|
||||
}
|
||||
const first = samples[0] || {};
|
||||
const settledAt = Math.max(
|
||||
lastChangeAt ?? holdStart,
|
||||
loadingEndAt ?? (first.loading ? holdEnd : holdStart),
|
||||
);
|
||||
const end = samples.at(-1) || {};
|
||||
const row = {
|
||||
stop: stop.name,
|
||||
scale: stop.scale,
|
||||
holdEpochMs: [holdStartEpoch, holdStartEpoch + (holdEnd - holdStart)],
|
||||
onScreen: end.onScreen ?? null,
|
||||
staleFrame,
|
||||
timeToCamerasMs: alpr ? Math.round(settledAt - holdStart) : null,
|
||||
cameras: end.count ?? 0,
|
||||
shown: end.shown ?? 0,
|
||||
status: end.status,
|
||||
label: end.label,
|
||||
loadingAtHoldEnd: Boolean(end.loading),
|
||||
visualChanges,
|
||||
changeLog,
|
||||
lateChanges: samples.filter(
|
||||
(sample, i) =>
|
||||
i > 0 &&
|
||||
sample.t > holdStart + 3000 &&
|
||||
(sample.hash !== samples[i - 1].hash ||
|
||||
sample.shown !== samples[i - 1].shown),
|
||||
).length,
|
||||
moving,
|
||||
holding,
|
||||
};
|
||||
pass.stops.push(row);
|
||||
console.log(
|
||||
` ${stop.name.padEnd(22)} cams ${String(row.cameras).padStart(5)} t ${String(row.timeToCamerasMs ?? '-').padStart(5)} ms frame p50/p95 ${holding.frameP50}/${holding.frameP95} render p50/p95 ${holding.renderP50}/${holding.renderP95} changes ${visualChanges}/${row.lateChanges} ${row.status || ''} ${row.label || ''}`,
|
||||
);
|
||||
if (passIndex === passes.length - 1) {
|
||||
await page.screenshot({
|
||||
path: path.join(outDir, `${passName}-${stop.name}.png`),
|
||||
});
|
||||
}
|
||||
}
|
||||
result.passes.push(pass);
|
||||
}
|
||||
const clip = await recorder.stop();
|
||||
result.clip = clip;
|
||||
// Clip-relative hold windows, so a reviewer can jump to each stop.
|
||||
const clipStart = recorder.frames[0]?.timestamp * 1000;
|
||||
for (const pass of result.passes)
|
||||
for (const stop of pass.stops)
|
||||
stop.clipSeconds = stop.holdEpochMs.map(
|
||||
(ms) => Math.round((ms - clipStart) / 100) / 10,
|
||||
);
|
||||
console.log('clip', clip);
|
||||
|
||||
// Rendered-state check from the recorded frames themselves: count ALPR-cyan
|
||||
// pixels per frame during each hold, subtract the ALPR-off pass at the same
|
||||
// stop, and find when the markers actually became visible on screen. Entity
|
||||
// state can be ready while nothing is painted (a missing render request),
|
||||
// which this catches and the in-page samples cannot.
|
||||
const framesIn = ([a, b]) =>
|
||||
recorder.frames.filter(
|
||||
(frame) => frame.timestamp * 1000 >= a && frame.timestamp * 1000 < b,
|
||||
);
|
||||
const median = (list) =>
|
||||
list.length ? [...list].sort((x, y) => x - y)[list.length >> 1] : 0;
|
||||
const offPass = result.passes.find((p) => !p.alpr);
|
||||
for (const pass of result.passes) {
|
||||
for (const [index, stop] of pass.stops.entries()) {
|
||||
const frames = framesIn(stop.holdEpochMs);
|
||||
const counts = [];
|
||||
for (const frame of frames)
|
||||
counts.push({
|
||||
atMs: Math.round(frame.timestamp * 1000 - stop.holdEpochMs[0]),
|
||||
cyan: await cyanCount(frame.file),
|
||||
});
|
||||
stop.pixelTimeline = counts;
|
||||
if (!pass.alpr) {
|
||||
stop.cyanBaseline = median(counts.map((c) => c.cyan));
|
||||
continue;
|
||||
}
|
||||
const baseline = offPass?.stops[index]?.cyanBaseline ?? 0;
|
||||
const tail = counts.filter(
|
||||
(c) => c.atMs >= stop.holdEpochMs[1] - stop.holdEpochMs[0] - 1000,
|
||||
);
|
||||
const final = median(tail.map((c) => c.cyan)) - baseline;
|
||||
stop.renderedMarkerPixels = final;
|
||||
let visibleAt = null;
|
||||
if (final >= MIN_MARKER_PIXELS) {
|
||||
for (let i = counts.length - 1; i >= 0; i--) {
|
||||
if (counts[i].cyan - baseline >= final * 0.8)
|
||||
visibleAt = counts[i].atMs;
|
||||
else break;
|
||||
}
|
||||
}
|
||||
stop.renderedVisibleAtMs = visibleAt;
|
||||
console.log(
|
||||
` ${pass.name} ${stop.stop.padEnd(22)} on screen ${stop.onScreen ?? '-'} marker px ${final} visible at ${visibleAt ?? '-'} ms (state ${stop.timeToCamerasMs} ms) end-of-hold px ${stop.staleFrame?.shown} / forced ${stop.staleFrame?.forced}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Aggregate and assert.
|
||||
const aggregate = (pass, phase) => {
|
||||
const frames = pass.stops.map((s) => s[phase]);
|
||||
const weighted = (key) => {
|
||||
const total = frames.reduce((n, f) => n + (f.frames || 0), 0);
|
||||
return (
|
||||
Math.round(
|
||||
(frames.reduce((n, f) => n + (f[key] || 0) * (f.frames || 0), 0) /
|
||||
Math.max(1, total)) *
|
||||
10,
|
||||
) / 10
|
||||
);
|
||||
};
|
||||
return {
|
||||
frameP50: weighted('frameP50'),
|
||||
frameP95: weighted('frameP95'),
|
||||
renderP50: weighted('renderP50'),
|
||||
renderP95: weighted('renderP95'),
|
||||
longFrames: frames.reduce((n, f) => n + f.longFrames, 0),
|
||||
};
|
||||
};
|
||||
result.summary = result.passes.map((pass) => ({
|
||||
pass: pass.name,
|
||||
moving: aggregate(pass, 'moving'),
|
||||
holding: aggregate(pass, 'holding'),
|
||||
}));
|
||||
console.table(
|
||||
result.summary.map((s) => ({
|
||||
pass: s.pass,
|
||||
'move p50/p95': `${s.moving.frameP50}/${s.moving.frameP95}`,
|
||||
'hold p50/p95': `${s.holding.frameP50}/${s.holding.frameP95}`,
|
||||
'render hold p50/p95': `${s.holding.renderP50}/${s.holding.renderP95}`,
|
||||
'long frames': s.moving.longFrames + s.holding.longFrames,
|
||||
})),
|
||||
);
|
||||
if (assertOn) {
|
||||
const off = result.summary.find((s) => s.pass.includes('off'));
|
||||
const warm = result.summary.find((s) => s.pass.endsWith('warm'));
|
||||
const warmPass = result.passes.find((p) => p.name.endsWith('warm'));
|
||||
const onPasses = result.passes.filter((p) => p.alpr);
|
||||
if (off && warm) {
|
||||
for (const phase of ['moving', 'holding'])
|
||||
for (const key of ['frameP50', 'frameP95'])
|
||||
if (warm[phase][key] > off[phase][key] * 1.1 + 2)
|
||||
fail(
|
||||
`${phase} ${key} ${warm[phase][key]} ms with ALPR vs ${off[phase][key]} ms without`,
|
||||
);
|
||||
}
|
||||
for (const stop of warmPass?.stops || []) {
|
||||
if (
|
||||
stop.scale !== 'city' &&
|
||||
!(
|
||||
stop.timeToCamerasMs !== null &&
|
||||
stop.timeToCamerasMs <= 2000 &&
|
||||
stop.cameras > 0
|
||||
)
|
||||
)
|
||||
fail(
|
||||
`${stop.stop}: cameras after ${stop.timeToCamerasMs} ms (${stop.cameras}) on the warm pass`,
|
||||
);
|
||||
}
|
||||
for (const pass of onPasses)
|
||||
for (const stop of pass.stops) {
|
||||
if (stop.staleFrame?.stale)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: screen showed ${stop.staleFrame.shown} marker px, ${stop.staleFrame.forced} after forcing a render`,
|
||||
);
|
||||
// Cameras on screen must be painted, and painted when state says so.
|
||||
if (stop.onScreen > 0 && stop.renderedMarkerPixels < MIN_MARKER_PIXELS)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: ${stop.onScreen} cameras on screen but no markers rendered`,
|
||||
);
|
||||
if (
|
||||
stop.renderedVisibleAtMs !== null &&
|
||||
stop.renderedVisibleAtMs > stop.timeToCamerasMs + 700
|
||||
)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: markers rendered ${stop.renderedVisibleAtMs} ms after the stop, ${stop.timeToCamerasMs} ms after state`,
|
||||
);
|
||||
if (
|
||||
pass.name.endsWith('warm') &&
|
||||
stop.scale !== 'city' &&
|
||||
stop.renderedVisibleAtMs !== null &&
|
||||
stop.renderedVisibleAtMs > 2000
|
||||
)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: markers visible only after ${stop.renderedVisibleAtMs} ms`,
|
||||
);
|
||||
if (
|
||||
stop.cameras > 0 &&
|
||||
stop.onScreen === 0 &&
|
||||
!/none on screen/i.test(stop.label)
|
||||
)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: no camera on screen and no explicit state`,
|
||||
);
|
||||
if (stop.loadingAtHoldEnd)
|
||||
fail(`${pass.name} ${stop.stop}: still loading when the hold ended`);
|
||||
if (
|
||||
stop.scale === 'city' &&
|
||||
!(stop.cameras > 0 || /zoom in/i.test(stop.label))
|
||||
)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: neither cameras nor a zoom-in prompt`,
|
||||
);
|
||||
if (stop.visualChanges > 1 || stop.lateChanges > 0)
|
||||
fail(
|
||||
`${pass.name} ${stop.stop}: ${stop.visualChanges} marker changes (${stop.lateChanges} after 3 s) while the camera was still`,
|
||||
);
|
||||
}
|
||||
const errors = result.consoleErrors.filter(
|
||||
(text) => !/favicon/i.test(text),
|
||||
);
|
||||
if (errors.length)
|
||||
fail(`console errors: ${errors.slice(0, 3).join(' | ')}`);
|
||||
}
|
||||
} catch (error) {
|
||||
fail(error.stack || error.message);
|
||||
} finally {
|
||||
fs.writeFileSync(
|
||||
path.join(outDir, 'result.json'),
|
||||
JSON.stringify(result, null, 1),
|
||||
);
|
||||
await browser.close();
|
||||
}
|
||||
console.log(
|
||||
`${result.failures.length ? 'FAIL' : 'PASS'} — ${path.join(outDir, 'result.json')}`,
|
||||
);
|
||||
process.exit(result.failures.length ? 1 : 0);
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* Evidence the browser QA gates collect while they run and save under
|
||||
* `qa-artifacts/<gate>/` when they fail (CI uploads that directory). Tokens
|
||||
* and keys are masked before anything is printed or saved.
|
||||
*/
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
/** Console messages and failed requests kept per page. */
|
||||
export const EVIDENCE_LIMIT = 200;
|
||||
|
||||
/** Cesium's console line when the render loop throws (CesiumWidget). */
|
||||
export const RENDER_ERROR_PATTERN = /An error occurred while rendering/i;
|
||||
|
||||
/** Mask tokens in a URL or message before it is printed or saved. */
|
||||
export function redact(text) {
|
||||
return String(text ?? '')
|
||||
.replace(
|
||||
/([?&](?:access_token|token|key|api_key|apikey|client_secret|signature)=)[^&\s"']*/gi,
|
||||
'$1REDACTED',
|
||||
)
|
||||
.replace(/MLY\|[^\s&"',)]+/g, 'MLY|REDACTED')
|
||||
.replace(/AIza[\w-]{30,}/g, 'AIza…REDACTED');
|
||||
}
|
||||
|
||||
function push(list, entry) {
|
||||
list.push(entry);
|
||||
if (list.length > EVIDENCE_LIMIT)
|
||||
list.splice(0, list.length - EVIDENCE_LIMIT);
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe to `viewer.scene.renderError` in every document the page loads
|
||||
* (reloads included), into `window.__qaRenderErrors`. Call before `goto`.
|
||||
*/
|
||||
export function hookRenderErrors(page) {
|
||||
return page.evaluateOnNewDocument(() => {
|
||||
window.__qaRenderErrors = [];
|
||||
const hook = () => {
|
||||
const scene = window.__godsEyeView?.viewer?.scene;
|
||||
if (!scene?.renderError) return false;
|
||||
scene.renderError.addEventListener((_, error) =>
|
||||
window.__qaRenderErrors.push(
|
||||
String(error?.stack || error?.message || error),
|
||||
),
|
||||
);
|
||||
return true;
|
||||
};
|
||||
const timer = setInterval(() => {
|
||||
if (hook()) clearInterval(timer);
|
||||
}, 250);
|
||||
});
|
||||
}
|
||||
|
||||
export async function readRenderErrors(page) {
|
||||
try {
|
||||
return await page.evaluate(() => window.__qaRenderErrors ?? []);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch one page's console, errors and failed requests. `errors` is shared
|
||||
* across pages; `isError(text)` adds console lines a gate treats as errors.
|
||||
* @returns {{name: string, page: object, console: Array<object>, failed: Array<object>, renderErrors: Array<string>}}
|
||||
*/
|
||||
export function watchPage(page, { name, errors, isError = () => false }) {
|
||||
const monitor = { name, page, console: [], failed: [], renderErrors: [] };
|
||||
page.on('console', (message) => {
|
||||
const text = redact(message.text());
|
||||
push(monitor.console, { type: message.type(), text });
|
||||
if (message.type() === 'error' && RENDER_ERROR_PATTERN.test(text)) {
|
||||
monitor.renderErrors.push(text);
|
||||
errors.push(`[${name}] render loop: ${text}`);
|
||||
} else if (isError(text)) errors.push(`[${name}] ${text}`);
|
||||
});
|
||||
page.on('pageerror', (error) => {
|
||||
const text = redact(error?.stack || error?.message || error);
|
||||
push(monitor.console, { type: 'pageerror', text });
|
||||
errors.push(`[${name}] ${text}`);
|
||||
});
|
||||
page.on('requestfailed', (request) =>
|
||||
push(monitor.failed, {
|
||||
url: redact(request.url()),
|
||||
method: request.method(),
|
||||
failure: request.failure()?.errorText ?? 'failed',
|
||||
}),
|
||||
);
|
||||
page.on('response', (response) => {
|
||||
if (response.status() < 400) return;
|
||||
push(monitor.failed, {
|
||||
url: redact(response.url()),
|
||||
method: response.request().method(),
|
||||
status: response.status(),
|
||||
});
|
||||
});
|
||||
return monitor;
|
||||
}
|
||||
|
||||
const withTimeout = (promise, ms, label) =>
|
||||
Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`${label} timed out`)), ms),
|
||||
),
|
||||
]);
|
||||
|
||||
function streetLevelState(page) {
|
||||
return page.evaluate(() => {
|
||||
const module =
|
||||
window.__godsEyeView?.dataManager?.layers?.get('street-level')?.module;
|
||||
return module?.getUIState ? module.getUIState() : null;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Save a screenshot and JSON record per page, plus the error. A page that
|
||||
* cannot be captured records why instead.
|
||||
* @returns {Promise<string>} the directory written
|
||||
*/
|
||||
export async function saveFailureArtifacts({ dir, browser, monitors, error }) {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(dir, 'error.txt'),
|
||||
redact(error?.stack || error?.message || String(error)),
|
||||
);
|
||||
const known = new Map(monitors.map((monitor) => [monitor.page, monitor]));
|
||||
let pages = [];
|
||||
try {
|
||||
pages = await browser.pages();
|
||||
} catch {
|
||||
pages = monitors.map((monitor) => monitor.page);
|
||||
}
|
||||
let index = 0;
|
||||
for (const page of pages) {
|
||||
const monitor = known.get(page);
|
||||
// The browser's first blank tab holds nothing worth saving.
|
||||
if (!monitor && page.url() === 'about:blank') continue;
|
||||
const base = `${String(index++).padStart(2, '0')}-${monitor?.name ?? 'page'}`;
|
||||
const record = { url: redact(page.url()) };
|
||||
try {
|
||||
await withTimeout(
|
||||
page.screenshot({
|
||||
path: path.join(dir, `${base}.png`),
|
||||
fullPage: true,
|
||||
}),
|
||||
30_000,
|
||||
'screenshot',
|
||||
);
|
||||
} catch (shotError) {
|
||||
record.screenshotError = String(shotError?.message || shotError);
|
||||
}
|
||||
try {
|
||||
record.streetLevel = await withTimeout(
|
||||
streetLevelState(page),
|
||||
10_000,
|
||||
'getUIState',
|
||||
);
|
||||
} catch (stateError) {
|
||||
record.streetLevelError = String(stateError?.message || stateError);
|
||||
}
|
||||
record.renderErrors = [
|
||||
...(monitor?.renderErrors ?? []),
|
||||
...(await withTimeout(readRenderErrors(page), 10_000, 'render errors')
|
||||
.then((list) => list.map((text) => `scene.renderError: ${text}`))
|
||||
.catch(() => [])),
|
||||
].map(redact);
|
||||
record.console = monitor?.console ?? [];
|
||||
record.failedRequests = monitor?.failed ?? [];
|
||||
await fs.writeFile(
|
||||
path.join(dir, `${base}.json`),
|
||||
redact(JSON.stringify(record, null, 2)),
|
||||
);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
@@ -253,19 +253,19 @@ async function main() {
|
||||
const url = new URL(request.url());
|
||||
if (url.origin !== APP_ORIGIN) return void request.continue();
|
||||
// Silence the live feeds: the injected field is the only population.
|
||||
if (url.pathname === '/api/ais-live') {
|
||||
if (url.pathname === '/api/vessels') {
|
||||
return void request.respond(stubJson({
|
||||
rows: [], source: 'QA', status: 'open', error: null, refreshing: false,
|
||||
newestPositionAt: null, lastMessageAt: null,
|
||||
}));
|
||||
}
|
||||
if (url.pathname === '/api/adsblol/mil') {
|
||||
if (url.pathname === '/api/military') {
|
||||
return void request.respond(stubJson({ msg: 'No error', now: Date.now(), ac: [] }));
|
||||
}
|
||||
if (url.pathname === '/api/opensky') {
|
||||
if (url.pathname === '/api/flights') {
|
||||
return void request.respond(stubJson({ time: Math.floor(Date.now() / 1000), states: [] }));
|
||||
}
|
||||
if (url.pathname === '/api/opensky-track') {
|
||||
if (url.pathname === '/api/flights/track') {
|
||||
return void request.respond(stubJson({ path: [] }));
|
||||
}
|
||||
if (url.pathname === '/api/openai/hud-summary') {
|
||||
|
||||
+103
-39
@@ -42,7 +42,7 @@ page.on('pageerror', (error) => consoleErrors.push(error.message));
|
||||
page.on('response', (response) => {
|
||||
const url = new URL(response.url());
|
||||
const expectedOptionalTrackMiss = response.status() === 404
|
||||
&& url.pathname === '/api/opensky-track';
|
||||
&& url.pathname === '/api/flights/track';
|
||||
if (
|
||||
url.origin === new URL(appUrl).origin
|
||||
&& response.status() >= 400
|
||||
@@ -108,7 +108,7 @@ try {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/ais-live') {
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/vessels') {
|
||||
request.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
@@ -124,7 +124,7 @@ try {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/adsblol/mil') {
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/military') {
|
||||
request.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
@@ -132,7 +132,7 @@ try {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/opensky-track') {
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/flights/track') {
|
||||
request.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
@@ -832,7 +832,7 @@ try {
|
||||
await readinessPage.setRequestInterception(true);
|
||||
readinessPage.on('request', (request) => {
|
||||
const url = new URL(request.url());
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/opensky') {
|
||||
if (url.origin === new URL(appUrl).origin && url.pathname === '/api/flights') {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
request.respond({
|
||||
status: 200, contentType: 'application/json',
|
||||
@@ -1572,6 +1572,10 @@ try {
|
||||
const cockpit = manager.cockpitView;
|
||||
const current = document.getElementById('cockpit-vision-current');
|
||||
const label = document.getElementById('cockpit-vision-current-label');
|
||||
const settle = (ms = 900) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const intensities = () => Object.fromEntries(
|
||||
Object.entries(manager.stages || {}).map(([name, stage]) => [name, stage.uniforms?.intensity]),
|
||||
);
|
||||
const read = () => ({
|
||||
mode: current?.dataset.cockpitVision,
|
||||
label: label?.textContent?.trim(),
|
||||
@@ -1582,62 +1586,106 @@ try {
|
||||
parametersCollapsed: document.getElementById('param-slider-panel')?.classList.contains('collapsed'),
|
||||
signalCollapsed: cockpit.signalCollapsed,
|
||||
focusRetained: document.activeElement === current,
|
||||
activeStyle: manager.activeStyle,
|
||||
intensities: intensities(),
|
||||
});
|
||||
const reenter = async () => {
|
||||
cockpit.exit({ restoreTracking: true });
|
||||
await settle();
|
||||
for (let attempt = 0; attempt < 40; attempt += 1) {
|
||||
if (cockpit.enter() === true) return true;
|
||||
await settle(200);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
manager._setCockpitDisclosure('display', false);
|
||||
manager._setCockpitDisclosure('radio', false);
|
||||
cockpit.setVisionMode('optical');
|
||||
current.focus({ preventScroll: true });
|
||||
const states = [read()];
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
for (let index = 0; index < 7; index += 1) {
|
||||
current.click();
|
||||
states.push(read());
|
||||
}
|
||||
const originalStyle = manager.activeStyle;
|
||||
// A Noir map preset: Cockpit enters on Noir, Normal clears every shader
|
||||
// without changing the map preset, and Exit Cockpit restores Noir.
|
||||
cockpit.exit({ restoreTracking: true });
|
||||
manager.setStyle('noir');
|
||||
await new Promise((resolve) => setTimeout(resolve, 900));
|
||||
await settle();
|
||||
const enteredOnNoir = await reenter();
|
||||
const entryNoir = read();
|
||||
cockpit.setVisionMode('optical');
|
||||
const inheritedNoir = read();
|
||||
cockpit.setVisionMode('noir');
|
||||
const explicitNoir = read();
|
||||
cockpit.setVisionMode('optical');
|
||||
const restoredNoir = {
|
||||
...read(),
|
||||
activeStyle: manager.activeStyle,
|
||||
intensity: manager.stages.noir?.uniforms?.intensity,
|
||||
};
|
||||
const normalInsideNoir = read();
|
||||
cockpit.setVisionMode('crt');
|
||||
const cockpitOnlyCrt = read();
|
||||
const exitedNoir = cockpit.exit({ restoreTracking: true }) === true;
|
||||
await settle();
|
||||
const restoredNoir = { ...read(), cockpitActive: cockpit.active };
|
||||
manager.setStyle(originalStyle);
|
||||
await new Promise((resolve) => setTimeout(resolve, 900));
|
||||
await settle();
|
||||
const reentered = await reenter();
|
||||
cockpit.setVisionMode('optical');
|
||||
return { states, inheritedNoir, explicitNoir, restoredNoir };
|
||||
return {
|
||||
states,
|
||||
enteredOnNoir,
|
||||
entryNoir,
|
||||
normalInsideNoir,
|
||||
cockpitOnlyCrt,
|
||||
exitedNoir,
|
||||
restoredNoir,
|
||||
reentered,
|
||||
};
|
||||
});
|
||||
const cycleModes = ['optical', 'crt', 'nvg', 'thermal', 'anime', 'noir', 'snow', 'optical'];
|
||||
const cycleLabels = ['NORMAL', 'CRT', 'NVG', 'FLIR', 'ANIME', 'NOIR', 'SNOW', 'NORMAL'];
|
||||
check(
|
||||
'Cockpit vision control exposes inherited, CRT, NVG, FLIR, and NOIR before wrapping',
|
||||
JSON.stringify(visionCycle.states.map((state) => state.mode))
|
||||
=== JSON.stringify(['optical', 'crt', 'nvg', 'thermal', 'noir', 'optical'])
|
||||
&& visionCycle.states[4]?.label === 'NOIR'
|
||||
&& visionCycle.states[4]?.aria === 'Current cockpit vision style: Noir. Activate for next style.',
|
||||
JSON.stringify(visionCycle),
|
||||
'Cockpit vision control exposes Normal, CRT, NVG, FLIR, Anime, Noir and Snow before wrapping',
|
||||
JSON.stringify(visionCycle.states.map((state) => state.mode)) === JSON.stringify(cycleModes)
|
||||
&& JSON.stringify(visionCycle.states.map((state) => state.label)) === JSON.stringify(cycleLabels)
|
||||
&& visionCycle.states[5]?.aria === 'Current cockpit vision style: Noir. Activate for next style.'
|
||||
&& visionCycle.states[7]?.aria === 'Current cockpit vision style: Normal. Activate for next style.',
|
||||
JSON.stringify(visionCycle.states.map(({ mode, label, aria }) => ({ mode, label, aria }))),
|
||||
);
|
||||
check(
|
||||
'user vision changes open Display for every temporary treatment and retain selector focus',
|
||||
visionCycle.states[1]?.mode === 'crt'
|
||||
&& visionCycle.states.slice(1, 5).every((state) => state.displayOpen === 'true')
|
||||
&& visionCycle.states.slice(1, 5).every((state) => state.radioOpen === 'false')
|
||||
&& visionCycle.states.slice(1, 5).every((state) => state.parametersActive)
|
||||
&& visionCycle.states.slice(1, 5).every((state) => !state.parametersCollapsed)
|
||||
&& visionCycle.states.slice(1, 5).every((state) => state.signalCollapsed)
|
||||
visionCycle.states.slice(1, 7).every((state) => state.displayOpen === 'true')
|
||||
&& visionCycle.states.slice(1, 7).every((state) => state.radioOpen === 'false')
|
||||
&& visionCycle.states.slice(1, 7).every((state) => state.parametersActive)
|
||||
&& visionCycle.states.slice(1, 7).every((state) => !state.parametersCollapsed)
|
||||
&& visionCycle.states.slice(1, 7).every((state) => state.signalCollapsed)
|
||||
&& visionCycle.states.every((state) => state.focusRetained),
|
||||
JSON.stringify(visionCycle.states),
|
||||
);
|
||||
check(
|
||||
'inherited Noir and explicit Noir keep distinct ownership and restore the Noir map preset',
|
||||
visionCycle.inheritedNoir?.mode === 'optical'
|
||||
&& visionCycle.inheritedNoir?.label === 'NOIR'
|
||||
&& visionCycle.explicitNoir?.mode === 'noir'
|
||||
&& visionCycle.explicitNoir?.label === 'NOIR'
|
||||
&& visionCycle.restoredNoir?.mode === 'optical'
|
||||
'each Cockpit vision mode drives only its own shader and Normal clears them all',
|
||||
visionCycle.states.slice(1, 7).every((state) => {
|
||||
const target = {
|
||||
crt: 'retro', nvg: 'surveillance', thermal: 'thermal', anime: 'anime', noir: 'noir', snow: 'snow',
|
||||
}[state.mode];
|
||||
return Object.entries(state.intensities).every(([name, value]) =>
|
||||
name === target ? value > 0 : value === 0);
|
||||
})
|
||||
&& Object.values(visionCycle.states[7].intensities).every((value) => value === 0),
|
||||
JSON.stringify(visionCycle.states.map(({ mode, intensities }) => ({ mode, intensities }))),
|
||||
);
|
||||
check(
|
||||
'Cockpit enters on the Noir map preset, Normal clears it, and Exit Cockpit restores it',
|
||||
visionCycle.enteredOnNoir
|
||||
&& visionCycle.entryNoir?.mode === 'noir'
|
||||
&& visionCycle.entryNoir?.label === 'NOIR'
|
||||
&& visionCycle.normalInsideNoir?.mode === 'optical'
|
||||
&& visionCycle.normalInsideNoir?.label === 'NORMAL'
|
||||
&& visionCycle.normalInsideNoir?.activeStyle === 'noir'
|
||||
&& Object.values(visionCycle.normalInsideNoir?.intensities || {}).every((value) => value === 0)
|
||||
&& visionCycle.cockpitOnlyCrt?.mode === 'crt'
|
||||
&& visionCycle.cockpitOnlyCrt?.activeStyle === 'noir'
|
||||
&& visionCycle.exitedNoir
|
||||
&& !visionCycle.restoredNoir?.cockpitActive
|
||||
&& visionCycle.restoredNoir?.activeStyle === 'noir'
|
||||
&& visionCycle.restoredNoir?.intensity === 1,
|
||||
&& visionCycle.restoredNoir?.intensities?.noir === 1
|
||||
&& visionCycle.restoredNoir?.intensities?.retro === 0
|
||||
&& visionCycle.reentered,
|
||||
JSON.stringify(visionCycle),
|
||||
);
|
||||
await page.evaluate(() => window.__godsEyeView.styleManager.cockpitView.setVisionMode('noir'));
|
||||
@@ -2266,10 +2314,16 @@ try {
|
||||
// Both the UI and compatibility facade delegate to this navigation owner.
|
||||
const manager = window.__godsEyeView.styleManager._locationNavigation;
|
||||
const awareness = window.__godsEyeView.dataManager.layers.get('military-awareness')?.module;
|
||||
const styleManager = window.__godsEyeView.styleManager;
|
||||
const styleBefore = styleManager.activeStyle;
|
||||
// A Cockpit-only choice that Reset must undo.
|
||||
styleManager.cockpitView.setVisionMode('snow');
|
||||
window.__qaCockpitReset = {
|
||||
calls: 0,
|
||||
original: manager.resetToGlobeView,
|
||||
subjectId: awareness?.getContextSnapshot?.()?.subject?.id || null,
|
||||
styleBefore,
|
||||
snowDuringCockpit: styleManager.stages?.snow?.uniforms?.intensity,
|
||||
};
|
||||
manager.resetToGlobeView = function qaCountedCockpitReset(...args) {
|
||||
window.__qaCockpitReset.calls += 1;
|
||||
@@ -2305,16 +2359,26 @@ try {
|
||||
resetHidden: document.getElementById('cockpit-reset-globe')?.hidden,
|
||||
height,
|
||||
subjectPreserved: Boolean(qa.subjectId && subjectId === qa.subjectId),
|
||||
styleRestored: gev.styleManager.activeStyle === qa.styleBefore,
|
||||
snowDuringCockpit: qa.snowDuringCockpit,
|
||||
// Cockpit-time intensities are Cockpit's own, so check against the map preset.
|
||||
visionRestored: Object.entries(gev.styleManager.stages || {}).every(([name, stage]) =>
|
||||
name === gev.styleManager.activeStyle
|
||||
? stage.uniforms?.intensity > 0
|
||||
: stage.uniforms?.intensity === 0),
|
||||
};
|
||||
});
|
||||
check(
|
||||
'keyboard Cockpit Reset uses one canonical route and preserves Contact selection',
|
||||
'keyboard Cockpit Reset uses one canonical route, preserves Contact selection and restores the map style',
|
||||
resetState.calls === 1
|
||||
&& !resetState.cockpitActive
|
||||
&& !resetState.trackedEntity
|
||||
&& resetState.resetHidden
|
||||
&& Math.abs(resetState.height - 18_000_000) < 150_000
|
||||
&& resetState.subjectPreserved,
|
||||
&& resetState.subjectPreserved
|
||||
&& resetState.snowDuringCockpit > 0
|
||||
&& resetState.styleRestored
|
||||
&& resetState.visionRestored,
|
||||
JSON.stringify(resetState),
|
||||
);
|
||||
check('runtime console remains clean', consoleErrors.length === 0 && localHttpErrors.length === 0,
|
||||
|
||||
@@ -0,0 +1,590 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* qa-context-arrivals.mjs — search arrivals above the surface, and Contacts
|
||||
* listing mapped installations around a moving tracked aircraft.
|
||||
*
|
||||
* Run: node scripts/qa-context-arrivals.mjs http://localhost:4173 [--headful]
|
||||
* [--out qa-shots/context-arrivals] [--only arrivals|contacts]
|
||||
*
|
||||
* Arrivals: types "Camp Mabry" and "39.7392, -104.9903" (downtown Denver)
|
||||
* into the location search, waits for the flight and the photoreal tiles,
|
||||
* orbits a little like a user looking around, and asserts the camera sits at
|
||||
* least 60 m above the rendered surface under it and under the view centre.
|
||||
*
|
||||
* Contacts: serves two synthetic aircraft through the page's own
|
||||
* /api/flights response (one flying north over Camp Mabry at 2,500 m, one
|
||||
* near Fort Cavazos) and an empty military feed, starts from a wide Texas
|
||||
* view, and presses CONTACTS as a user would. Asserts the Mapped
|
||||
* installations row lists at least one site with a distance within 15 s,
|
||||
* never falls back to "?" while the aircraft moves (30 s), keeps its site
|
||||
* entities (no rebuilds while the view is still), still lists sites in
|
||||
* Cockpit, and that SEARCH NEARBY SITES reports what it found. Then re-enters
|
||||
* Contacts on the Fort Cavazos aircraft and asserts a site within 30 km.
|
||||
* Requires zero console errors.
|
||||
*
|
||||
* Writes <out>/result.json, screenshots and <out>/clip.mp4 (CDP screencast
|
||||
* with real frame timing; needs ffmpeg).
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { createScreencast, moveCamera } from './qa-journey-recorder.mjs';
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
if (argv.includes('--help') || argv.includes('-h')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-context-arrivals.mjs <dev-server-url> [--headful] [--out dir] [--only arrivals|contacts]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const option = (name, fallback) => {
|
||||
const i = argv.indexOf(name);
|
||||
return i >= 0 && argv[i + 1] ? argv[i + 1] : fallback;
|
||||
};
|
||||
const appUrl =
|
||||
argv.find((arg) => /^https?:/.test(arg)) || 'http://localhost:4173';
|
||||
const outDir = path.resolve(option('--out', 'qa-shots/context-arrivals'));
|
||||
const only = option('--only', null);
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const result = {
|
||||
url: appUrl,
|
||||
arrivals: [],
|
||||
contacts: {},
|
||||
consoleErrors: [],
|
||||
failures: [],
|
||||
};
|
||||
const check = (name, ok, detail = '') => {
|
||||
console.log(
|
||||
`[${ok ? 'PASS' : 'FAIL'}] ${name}${detail ? ` — ${detail}` : ''}`,
|
||||
);
|
||||
if (!ok) result.failures.push(`${name}${detail ? ` — ${detail}` : ''}`);
|
||||
};
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: !argv.includes('--headful'),
|
||||
protocolTimeout: 300_000,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
...(process.platform === 'darwin'
|
||||
? ['--use-angle=metal', '--enable-gpu']
|
||||
: ['--use-gl=angle', '--use-angle=swiftshader']),
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
],
|
||||
});
|
||||
|
||||
async function openPage({ aircraft = false } = {}) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') result.consoleErrors.push(message.text());
|
||||
});
|
||||
page.on('pageerror', (error) => result.consoleErrors.push(error.message));
|
||||
page.on('response', (response) => {
|
||||
if (response.status() < 400) return;
|
||||
const url = new URL(response.url());
|
||||
// Track/enrichment lookups for the synthetic ICAOs have no upstream record.
|
||||
const synthetic = /aaa05\d/i.test(url.search + url.pathname);
|
||||
(synthetic
|
||||
? (result.expectedMisses ??= [])
|
||||
: (result.httpErrors ??= [])
|
||||
).push(`${response.status()} ${url.pathname}${url.search}`);
|
||||
});
|
||||
await page.evaluateOnNewDocument((aircraft) => {
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed');
|
||||
if (!aircraft) return;
|
||||
// Synthetic traffic through the app's own feed endpoints: one aircraft
|
||||
// flying north over Camp Mabry, one loitering near Fort Cavazos.
|
||||
const started = Date.now();
|
||||
const realFetch = window.fetch.bind(window);
|
||||
window.fetch = (input, init) => {
|
||||
const raw =
|
||||
typeof input === 'string' || input instanceof URL
|
||||
? String(input)
|
||||
: input?.url;
|
||||
let url;
|
||||
try {
|
||||
url = new URL(raw, location.href);
|
||||
} catch {
|
||||
return realFetch(input, init);
|
||||
}
|
||||
if (url.origin === location.origin && url.pathname === '/api/flights') {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const seconds = (Date.now() - started) / 1000;
|
||||
const lat = 30.29 + (seconds * 120) / 111_320;
|
||||
const json = {
|
||||
time: now,
|
||||
states: [
|
||||
[
|
||||
'aaa051',
|
||||
'QA051',
|
||||
'Synthetic',
|
||||
now,
|
||||
now,
|
||||
-97.765,
|
||||
lat,
|
||||
2500,
|
||||
false,
|
||||
120,
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
2500,
|
||||
null,
|
||||
false,
|
||||
0,
|
||||
],
|
||||
[
|
||||
'aaa052',
|
||||
'QA052',
|
||||
'Synthetic',
|
||||
now,
|
||||
now,
|
||||
-97.72,
|
||||
31.12,
|
||||
3000,
|
||||
false,
|
||||
90,
|
||||
270,
|
||||
0,
|
||||
null,
|
||||
3000,
|
||||
null,
|
||||
false,
|
||||
0,
|
||||
],
|
||||
],
|
||||
};
|
||||
return Promise.resolve(Response.json(json));
|
||||
}
|
||||
if (url.origin === location.origin && url.pathname === '/api/military')
|
||||
return Promise.resolve(
|
||||
Response.json({ ac: [], now: Date.now(), total: 0 }),
|
||||
);
|
||||
return realFetch(input, init);
|
||||
};
|
||||
}, aircraft);
|
||||
const url = new URL(appUrl);
|
||||
url.searchParams.set('welcome', '0');
|
||||
await page.goto(url.href, { waitUntil: 'domcontentloaded', timeout: 90_000 });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120_000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
return page;
|
||||
}
|
||||
|
||||
async function waitTiles(page, maxMs = 30_000) {
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const { scene } = window.__godsEyeView.viewer;
|
||||
if (scene.globe.show) return scene.globe.tilesLoaded;
|
||||
for (let i = 0; i < scene.primitives.length; i++) {
|
||||
const p = scene.primitives.get(i);
|
||||
if (p.show && p.tilesLoaded === false) return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{ timeout: maxMs, polling: 250 },
|
||||
)
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
/** Camera height above the rendered surface under it and under the view centre. */
|
||||
function clearance(page) {
|
||||
return page.evaluate(() => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
const { scene, camera } = viewer;
|
||||
const carto = camera.positionCartographic.clone();
|
||||
const sample = (c) => {
|
||||
try {
|
||||
const h = scene.sampleHeight(c);
|
||||
return Number.isFinite(h) ? h : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const under = sample(carto);
|
||||
let centre = null;
|
||||
const canvas = scene.canvas;
|
||||
const hit = scene.pickPosition?.({
|
||||
x: canvas.clientWidth / 2,
|
||||
y: canvas.clientHeight / 2,
|
||||
});
|
||||
if (hit) centre = scene.globe.ellipsoid.cartesianToCartographic(hit);
|
||||
const centreSurface = centre ? sample(centre) : null;
|
||||
const surface = Math.max(under ?? -Infinity, centreSurface ?? -Infinity);
|
||||
return {
|
||||
lat: (carto.latitude * 180) / Math.PI,
|
||||
lon: (carto.longitude * 180) / Math.PI,
|
||||
cameraM: Math.round(carto.height),
|
||||
surfaceUnderM: under === null ? null : Math.round(under),
|
||||
surfaceCentreM: centreSurface === null ? null : Math.round(centreSurface),
|
||||
clearanceM: Number.isFinite(surface)
|
||||
? Math.round(carto.height - surface)
|
||||
: null,
|
||||
pitchDeg: Math.round((camera.pitch * 180) / Math.PI),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function arrivals() {
|
||||
const page = await openPage();
|
||||
const dir = path.join(outDir, 'arrivals');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const recorder = await createScreencast(page, dir);
|
||||
await recorder.start();
|
||||
for (const query of ['Camp Mabry', '39.7392, -104.9903']) {
|
||||
await page.evaluate(() =>
|
||||
document.getElementById('search-toggle')?.click(),
|
||||
);
|
||||
await page.focus('#location-search');
|
||||
await page.evaluate(() => {
|
||||
document.getElementById('location-search').value = '';
|
||||
});
|
||||
await page.type('#location-search', query, { delay: 40 });
|
||||
await page.keyboard.press('Enter');
|
||||
// Wait for the flight to land: the camera still for 1.5 s.
|
||||
const landed = Date.now();
|
||||
let last = null;
|
||||
let stillSince = null;
|
||||
while (Date.now() - landed < 30_000) {
|
||||
const now = await page.evaluate(() => {
|
||||
const c = window.__godsEyeView.viewer.camera.positionWC;
|
||||
return [c.x, c.y, c.z];
|
||||
});
|
||||
const moved =
|
||||
!last ||
|
||||
Math.hypot(now[0] - last[0], now[1] - last[1], now[2] - last[2]) > 0.5;
|
||||
last = now;
|
||||
if (moved) stillSince = null;
|
||||
else stillSince ??= Date.now();
|
||||
if (
|
||||
Date.now() - landed > 4000 &&
|
||||
stillSince &&
|
||||
Date.now() - stillSince > 1500
|
||||
)
|
||||
break;
|
||||
await sleep(250);
|
||||
}
|
||||
await waitTiles(page);
|
||||
await sleep(2500); // the ground guard measures after tiles stream in
|
||||
const arrived = await clearance(page);
|
||||
const slug = query.replace(/\W+/g, '-').replace(/^-|-$/g, '').toLowerCase();
|
||||
await page.screenshot({ path: path.join(dir, `arrival-${slug}.png`) });
|
||||
// Look around: a gentle orbit, then check the eye still clears the ground.
|
||||
const view = await page.evaluate(() => {
|
||||
const c = window.__godsEyeView.viewer.camera;
|
||||
return {
|
||||
lat: (c.positionCartographic.latitude * 180) / Math.PI,
|
||||
lon: (c.positionCartographic.longitude * 180) / Math.PI,
|
||||
height: c.positionCartographic.height,
|
||||
heading: (c.heading * 180) / Math.PI,
|
||||
pitch: (c.pitch * 180) / Math.PI,
|
||||
};
|
||||
});
|
||||
await moveCamera(page, { ...view, heading: view.heading + 25 }, 2.5);
|
||||
await sleep(1500);
|
||||
const lookedAround = await clearance(page);
|
||||
result.arrivals.push({ query, arrived, lookedAround });
|
||||
check(
|
||||
`${query}: arrival clears the rendered surface`,
|
||||
arrived.clearanceM !== null && arrived.clearanceM >= 60,
|
||||
JSON.stringify(arrived),
|
||||
);
|
||||
}
|
||||
result.arrivalClip = await recorder.stop();
|
||||
await page.close();
|
||||
}
|
||||
|
||||
async function contacts() {
|
||||
const page = await openPage({ aircraft: true });
|
||||
const dir = path.join(outDir, 'contacts');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('flights', true, {
|
||||
origin: 'user',
|
||||
}),
|
||||
);
|
||||
// A wide Texas view: installation viewports are unbounded here.
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat: 30.35, lon: -97.76, height: 900_000, pitch: -90 },
|
||||
0,
|
||||
);
|
||||
await sleep(3000);
|
||||
// In-page probe: row text and installation entity churn every 250 ms.
|
||||
await page.evaluate(() => {
|
||||
const samples = [];
|
||||
window.__gevContextSamples = samples;
|
||||
let created = 0;
|
||||
const seen = new WeakSet();
|
||||
setInterval(() => {
|
||||
const row = [
|
||||
...document.querySelectorAll(
|
||||
'#military-awareness-panel .military-awareness-row',
|
||||
),
|
||||
].find(
|
||||
(candidate) =>
|
||||
candidate.querySelector('strong')?.textContent?.trim() ===
|
||||
'Mapped installations',
|
||||
);
|
||||
const ds = window.__godsEyeView.viewer.dataSources.getByName(
|
||||
'military-installations',
|
||||
)[0];
|
||||
for (const entity of ds?.entities.values || [])
|
||||
if (!seen.has(entity)) {
|
||||
seen.add(entity);
|
||||
created += 1;
|
||||
}
|
||||
samples.push({
|
||||
t: performance.now(),
|
||||
count: row?.querySelector('b')?.textContent?.trim() ?? null,
|
||||
text: row?.innerText?.replace(/\s+/g, ' ').trim() ?? null,
|
||||
installations: [
|
||||
...(row?.querySelectorAll('button[data-awareness-id]') || []),
|
||||
].map((button) => ({
|
||||
id: button.dataset.awarenessId,
|
||||
name: button.childNodes[0]?.textContent?.trim(),
|
||||
distance: button.querySelector('span')?.textContent?.trim(),
|
||||
})),
|
||||
entities: ds?.entities.values.length ?? 0,
|
||||
created,
|
||||
tracked:
|
||||
window.__godsEyeView.viewer.trackedEntity?.gevTrackedId ?? null,
|
||||
cockpit: document.body.classList.contains('cockpit-mode'),
|
||||
});
|
||||
if (samples.length > 5000) samples.splice(0, 1000);
|
||||
}, 250);
|
||||
});
|
||||
const recorder = await createScreencast(page, dir);
|
||||
await recorder.start();
|
||||
const t0 = await page.evaluate(() => performance.now());
|
||||
await page.evaluate(() =>
|
||||
document.getElementById('global-context-flights-btn').click(),
|
||||
);
|
||||
await page
|
||||
.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.viewer.trackedEntity?.gevTrackedId ===
|
||||
'flights:aaa051',
|
||||
{ timeout: 60_000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
const readSamples = (from) =>
|
||||
page.evaluate(
|
||||
(a) => window.__gevContextSamples.filter((s) => s.t >= a),
|
||||
from,
|
||||
);
|
||||
const numeric = (s) => /^\d+/.test(String(s.count ?? ''));
|
||||
// Up to 15 s for the first answer, then watch 30 s of flight.
|
||||
const deadline = Date.now() + 15_000;
|
||||
let first = null;
|
||||
while (Date.now() < deadline && !first) {
|
||||
first = (await readSamples(t0)).find(
|
||||
(s) =>
|
||||
numeric(s) &&
|
||||
Number.parseInt(s.count, 10) >= 1 &&
|
||||
/\d+(\.\d+)? k?m\b/.test(s.text),
|
||||
);
|
||||
if (!first) await sleep(500);
|
||||
}
|
||||
check(
|
||||
'Contacts lists a mapped installation with a distance',
|
||||
Boolean(first),
|
||||
first
|
||||
? `${Math.round(first.t - t0)} ms: ${first.text}`
|
||||
: 'row never answered',
|
||||
);
|
||||
await page.screenshot({ path: path.join(dir, 'contacts-camp-mabry.png') });
|
||||
await sleep(30_000);
|
||||
const flight = (await readSamples(first?.t ?? t0)).filter(
|
||||
(s) => s.tracked === 'flights:aaa051',
|
||||
);
|
||||
const reverted = flight.filter((s) => !numeric(s));
|
||||
const createdDuring = flight.length
|
||||
? flight.at(-1).created - flight[0].created
|
||||
: 0;
|
||||
result.contacts.mabry = {
|
||||
firstAnswerMs: first ? Math.round(first.t - t0) : null,
|
||||
firstText: first?.text ?? null,
|
||||
lastText: flight.at(-1)?.text ?? null,
|
||||
samples: flight.length,
|
||||
revertedSamples: reverted.length,
|
||||
entities: flight.at(-1)?.entities ?? 0,
|
||||
entitiesCreatedWhileTracking: createdDuring,
|
||||
};
|
||||
check(
|
||||
'the row never falls back to "?" while the aircraft moves',
|
||||
reverted.length === 0,
|
||||
`${reverted.length}/${flight.length}`,
|
||||
);
|
||||
check(
|
||||
'site entities are not rebuilt while tracking',
|
||||
createdDuring === 0,
|
||||
`${createdDuring} created`,
|
||||
);
|
||||
|
||||
// Cockpit: the same context must hold with a horizon view.
|
||||
const cockpit = await page.evaluate(() => {
|
||||
const view = window.__godsEyeView.styleManager.cockpitView;
|
||||
view.syncEntry();
|
||||
return view.enter();
|
||||
});
|
||||
await sleep(12_000);
|
||||
const inCockpit = (
|
||||
await readSamples((await page.evaluate(() => performance.now())) - 8000)
|
||||
).filter((s) => s.cockpit);
|
||||
const scope = await page.evaluate(
|
||||
() =>
|
||||
document.getElementById('cockpit-context-subject')?.textContent || null,
|
||||
);
|
||||
await page.screenshot({ path: path.join(dir, 'cockpit-camp-mabry.png') });
|
||||
result.contacts.cockpit = {
|
||||
entered: cockpit,
|
||||
samples: inCockpit.length,
|
||||
last: inCockpit.at(-1)?.text ?? null,
|
||||
scope,
|
||||
};
|
||||
check(
|
||||
'Cockpit scope names the installation window',
|
||||
/INSTALLATIONS WITHIN \d+ KM/.test(scope || ''),
|
||||
scope || 'no scope',
|
||||
);
|
||||
check(
|
||||
'Cockpit keeps mapped installations listed',
|
||||
cockpit !== false &&
|
||||
inCockpit.length > 0 &&
|
||||
inCockpit.every((s) => numeric(s) && Number.parseInt(s.count, 10) >= 1),
|
||||
inCockpit.at(-1)?.text || 'no cockpit samples',
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.cockpitView.exit?.(),
|
||||
);
|
||||
await sleep(2000);
|
||||
|
||||
// SEARCH NEARBY SITES, as the user presses it.
|
||||
await page.evaluate(() => {
|
||||
document.getElementById('toast').textContent = '';
|
||||
document.getElementById('installations-search-btn').click();
|
||||
});
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => document.getElementById('toast').textContent.trim().length > 0,
|
||||
{ timeout: 20_000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
const toast = await page.evaluate(() =>
|
||||
document.getElementById('toast').textContent.trim(),
|
||||
);
|
||||
await sleep(1500);
|
||||
const afterSearch = (
|
||||
await readSamples((await page.evaluate(() => performance.now())) - 1000)
|
||||
).at(-1);
|
||||
result.contacts.search = { toast, row: afterSearch?.text ?? null };
|
||||
check(
|
||||
'SEARCH NEARBY SITES reports what it found and the row still lists sites',
|
||||
/mapped sites? within \d+ km/i.test(toast) && numeric(afterSearch || {}),
|
||||
`toast "${toast}" · ${afterSearch?.text}`,
|
||||
);
|
||||
|
||||
// Fort Cavazos: leave Contacts, track the other aircraft, re-enter.
|
||||
await page.evaluate(() =>
|
||||
document.getElementById('global-context-flights-btn').click(),
|
||||
);
|
||||
await sleep(2000);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('flights')
|
||||
.module.trackById('aaa052'),
|
||||
);
|
||||
await sleep(3000);
|
||||
const t1 = await page.evaluate(() => performance.now());
|
||||
await page.evaluate(() =>
|
||||
document.getElementById('global-context-flights-btn').click(),
|
||||
);
|
||||
let cavazos = null;
|
||||
const cavazosDeadline = Date.now() + 20_000;
|
||||
while (Date.now() < cavazosDeadline) {
|
||||
const latest = (await readSamples(t1))
|
||||
.filter((s) => s.tracked === 'flights:aaa052' && numeric(s))
|
||||
.at(-1);
|
||||
const site = latest?.installations.find(
|
||||
(record) => record.name === 'Fort Hood',
|
||||
);
|
||||
const nearest = site?.distance?.match(/^(\d+(?:\.\d+)?) (k?m)$/);
|
||||
const km = nearest
|
||||
? Number(nearest[1]) / (nearest[2] === 'm' ? 1000 : 1)
|
||||
: null;
|
||||
if (km !== null && km <= 30) {
|
||||
cavazos = {
|
||||
text: latest.text,
|
||||
nearestKm: km,
|
||||
site,
|
||||
afterMs: Math.round(latest.t - t1),
|
||||
};
|
||||
break;
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
await sleep(2000);
|
||||
await page.screenshot({ path: path.join(dir, 'contacts-fort-cavazos.png') });
|
||||
result.contacts.cavazos = cavazos;
|
||||
check(
|
||||
'Fort Cavazos contact lists Fort Hood within 30 km',
|
||||
Boolean(cavazos),
|
||||
JSON.stringify(cavazos),
|
||||
);
|
||||
result.contactsClip = await recorder.stop();
|
||||
await page.close();
|
||||
}
|
||||
|
||||
try {
|
||||
if (only !== 'contacts') await arrivals();
|
||||
if (only !== 'arrivals') await contacts();
|
||||
// A 404 for a synthetic aircraft's track logs "Failed to load resource";
|
||||
// count those only when every failed response was such an expected miss.
|
||||
const resourceErrors = result.consoleErrors.filter((text) =>
|
||||
/Failed to load resource/.test(text),
|
||||
).length;
|
||||
const errors = result.consoleErrors.filter(
|
||||
(text) =>
|
||||
!/favicon/i.test(text) &&
|
||||
!(
|
||||
/Failed to load resource/.test(text) &&
|
||||
!(result.httpErrors || []).length &&
|
||||
resourceErrors <= (result.expectedMisses || []).length
|
||||
),
|
||||
);
|
||||
check(
|
||||
'zero console errors',
|
||||
errors.length === 0,
|
||||
errors.slice(0, 3).join(' | '),
|
||||
);
|
||||
check(
|
||||
'zero failed app responses',
|
||||
!(result.httpErrors || []).length,
|
||||
(result.httpErrors || []).slice(0, 3).join(' | '),
|
||||
);
|
||||
} catch (error) {
|
||||
check('harness completed', false, error.stack || error.message);
|
||||
} finally {
|
||||
fs.writeFileSync(
|
||||
path.join(outDir, 'result.json'),
|
||||
JSON.stringify(result, null, 1),
|
||||
);
|
||||
await browser.close();
|
||||
}
|
||||
console.log(
|
||||
`${result.failures.length ? 'FAIL' : 'PASS'} — ${path.join(outDir, 'result.json')}`,
|
||||
);
|
||||
process.exit(result.failures.length ? 1 : 0);
|
||||
@@ -12,7 +12,7 @@
|
||||
* honoring the politeness bounds.
|
||||
*
|
||||
* Fetch shim (installed before app code, same pattern as qa-sprites-b5.mjs):
|
||||
* - /api/opensky → 12 straight-flying planes, category 0
|
||||
* - /api/flights → 12 straight-flying planes, category 0
|
||||
* - /api/adsbdb/type/:hex → varied REAL type codes (C172, B744, DH8D, H60,
|
||||
* F16, GLID, B77W, A320, B738, PC12, R44) + one
|
||||
* found:false miss; responses are HELD until the
|
||||
@@ -260,14 +260,14 @@ async function main() {
|
||||
});
|
||||
}
|
||||
if (url.includes('/api/adsbdb/')) return Promise.resolve(jsonResponse({ found: false }));
|
||||
if (url.includes('/api/opensky-track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/adsblol/trace')) {
|
||||
if (url.includes('/api/flights/track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/military/track')) {
|
||||
return Promise.resolve(jsonResponse({ timestamp: Math.floor(nowSec), trace: [] }));
|
||||
}
|
||||
if (url.includes('/api/adsblol/mil')) {
|
||||
if (url.includes('/api/military')) {
|
||||
return Promise.resolve(jsonResponse({ msg: 'No error', now: Date.now(), ac: [] }));
|
||||
}
|
||||
if (url.includes('/api/opensky')) {
|
||||
if (url.includes('/api/flights')) {
|
||||
const states = S.planes.map((f) => {
|
||||
const s = S.stateAt(f, tRel);
|
||||
return [
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
* upstream failures so the run is deterministic and never depends on live
|
||||
* AISStream / CelesTrak availability:
|
||||
*
|
||||
* (i) AIS invalid key — /api/ais-live is answered with
|
||||
* (i) AIS invalid key — /api/vessels is answered with
|
||||
* {rows:[],status:'error',error:'invalid key'}. Assert the layer's
|
||||
* getStats().error is set (feed down) — NOT a clean 'just now · 0'.
|
||||
*
|
||||
@@ -193,10 +193,12 @@ async function main() {
|
||||
page.on('request', (req) => {
|
||||
const url = req.url();
|
||||
// (i) AIS invalid-key: the exact payload the spec calls for.
|
||||
if (url.includes('/api/ais-live') && !url.includes('/track')) {
|
||||
if (url.includes('/api/vessels') && !url.includes('/track')) {
|
||||
req.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
// The bundled server names its vessel provider; attribution reads it.
|
||||
headers: { 'X-Feed-Source': 'AISStream' },
|
||||
body: JSON.stringify({ rows: [], status: 'error', error: 'invalid key' }),
|
||||
});
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env node
|
||||
/** Persistent inline attribution through actual OSM layer display lifetimes. */
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { moveCamera } from './qa-journey-recorder.mjs';
|
||||
const url = process.argv.find((a) => /^https?:/.test(a));
|
||||
if (!url) throw new Error('Supply the running server URL');
|
||||
const dir = process.env.QA_POLISH_DIR || 'qa-shots/final-polish';
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const result = { states: [], errors: [] };
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--use-angle=metal'],
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
page.on('pageerror', (e) => result.errors.push(e.message));
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') result.errors.push(m.text());
|
||||
});
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
await page.goto(`${url}/?welcome=0`, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await page.evaluate(async () => {
|
||||
const g = window.__godsEyeView;
|
||||
for (const [id] of g.dataManager.layers)
|
||||
await g.dataManager.setEnabled(id, false);
|
||||
g.styleManager.setDetection({ enabled: false });
|
||||
});
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lon: -97.765, lat: 30.3125, height: 3500, pitch: -80 },
|
||||
0,
|
||||
);
|
||||
const toggle = (id, value) =>
|
||||
page.evaluate(
|
||||
(id, value) => window.__godsEyeView.dataManager.setEnabled(id, value),
|
||||
id,
|
||||
value,
|
||||
);
|
||||
const text = () => page.$eval('#cesium-credits', (e) => e.innerText);
|
||||
const shot = async (name, osm, tiles) => {
|
||||
await sleep(300);
|
||||
const state = await page.$eval('#cesium-credits', (e) => {
|
||||
const nodes = [
|
||||
...e.querySelectorAll('a,img,.cesium-credit-expand-link'),
|
||||
].filter((n) =>
|
||||
n.checkVisibility({ visibilityProperty: true, opacityProperty: true }),
|
||||
);
|
||||
return {
|
||||
text: e.innerText,
|
||||
rects: nodes
|
||||
.map((n) => {
|
||||
const b = n.getBoundingClientRect();
|
||||
return {
|
||||
text: n.textContent || n.alt || 'image',
|
||||
x: b.x,
|
||||
y: b.y,
|
||||
width: b.width,
|
||||
height: b.height,
|
||||
reachable: n.contains(
|
||||
document.elementFromPoint(
|
||||
b.x + b.width / 2,
|
||||
b.y + b.height / 2,
|
||||
),
|
||||
),
|
||||
};
|
||||
})
|
||||
.filter((b) => b.width > 0 && b.height > 0),
|
||||
};
|
||||
});
|
||||
assert.equal(state.text.includes('© OpenStreetMap'), osm);
|
||||
assert.equal(state.text.includes('© OpenMapTiles'), tiles);
|
||||
assert.equal(
|
||||
(state.text.match(/© OpenStreetMap/g) || []).length,
|
||||
osm ? 1 : 0,
|
||||
);
|
||||
assert.ok(state.rects.length >= 3);
|
||||
const centers = state.rects.map((b) => b.y + b.height / 2);
|
||||
assert.ok(
|
||||
Math.max(...centers) - Math.min(...centers) < 12,
|
||||
`${name}: credits stay on one desktop row`,
|
||||
);
|
||||
assert.ok(
|
||||
state.rects.every(
|
||||
(b) => b.x >= 0 && b.x + b.width <= page.viewport().width,
|
||||
),
|
||||
'no credit clipped',
|
||||
);
|
||||
assert.ok(
|
||||
state.rects.every((b) => b.reachable),
|
||||
'all credits and the attribution link remain clear of controls',
|
||||
);
|
||||
await page.screenshot({ path: `${dir}/after-${name}.png` });
|
||||
await page.screenshot({
|
||||
path: `${dir}/after-${name}-row.png`,
|
||||
clip: { x: 16, y: 750, width: 1150, height: 150 },
|
||||
});
|
||||
result.states.push({ name, ...state });
|
||||
};
|
||||
await sleep(1600);
|
||||
await shot('nothing', false, false);
|
||||
await toggle('alpr-cameras', true);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document
|
||||
.querySelector('#cesium-credits')
|
||||
.innerText.includes('© OpenStreetMap'),
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await shot('alpr', true, false);
|
||||
await sleep(6000);
|
||||
await shot('alpr-persistent', true, false);
|
||||
await toggle('traffic', true);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document
|
||||
.querySelector('#cesium-credits')
|
||||
.innerText.includes('© OpenMapTiles'),
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await shot('alpr-and-traffic', true, true);
|
||||
await toggle('alpr-cameras', false);
|
||||
await shot('traffic', true, true);
|
||||
for (const width of [1280, 1920]) {
|
||||
await page.setViewport({ width, height: 900 });
|
||||
await sleep(500);
|
||||
await shot(`traffic-${width}`, true, true);
|
||||
}
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.setCleanView(true),
|
||||
);
|
||||
await shot('traffic-clean-ui', true, true);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.setCleanView(false),
|
||||
);
|
||||
await toggle('traffic', false);
|
||||
await shot('nothing-restored', false, false);
|
||||
assert.deepEqual(result.errors, []);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} finally {
|
||||
await fs.writeFile(`${dir}/credits.json`, JSON.stringify(result, null, 2));
|
||||
await browser.close();
|
||||
}
|
||||
@@ -89,7 +89,7 @@ await page.evaluateOnNewDocument((site, icao) => {
|
||||
window.__TERRAIN.ok += 1;
|
||||
return realFetch(input, init);
|
||||
}
|
||||
if (app && url.pathname === '/api/opensky') {
|
||||
if (app && url.pathname === '/api/flights') {
|
||||
const c = window.__CONTACT;
|
||||
const t = Math.floor(Date.now() / 1000);
|
||||
// baro_altitude (7) and geo_altitude (13) both null, on_ground true.
|
||||
@@ -99,7 +99,7 @@ await page.evaluateOnNewDocument((site, icao) => {
|
||||
null, true, 1.5, 90, 0, null, null, null, false, 0]],
|
||||
}));
|
||||
}
|
||||
if (app && url.pathname === '/api/opensky-track') return Promise.resolve(json({ path: [] }));
|
||||
if (app && url.pathname === '/api/flights/track') return Promise.resolve(json({ path: [] }));
|
||||
if (app && /^\/api\/adsbdb\//.test(url.pathname)) return Promise.resolve(json({}));
|
||||
return realFetch(input, init);
|
||||
};
|
||||
|
||||
@@ -268,11 +268,11 @@ async function installSyntheticFetches(page) {
|
||||
}));
|
||||
window.fetch = (input, init) => {
|
||||
const url = typeof input === 'string' ? input : input?.url || '';
|
||||
if (url.includes('/api/opensky-track')) return json({ path: [] });
|
||||
if (url.includes('/api/adsblol/trace')) return json({ timestamp: Date.now() / 1000, trace: [] });
|
||||
if (url.includes('/api/opensky')) return json({ time: Math.floor(Date.now() / 1000), states: [] });
|
||||
if (url.includes('/api/flights/track')) return json({ path: [] });
|
||||
if (url.includes('/api/military/track')) return json({ timestamp: Date.now() / 1000, trace: [] });
|
||||
if (url.includes('/api/flights')) return json({ time: Math.floor(Date.now() / 1000), states: [] });
|
||||
if (url.includes('/api/adsbdb/')) return json({ found: false });
|
||||
if (url.includes('/api/ais-live')) return json({ status: 'open', rows: [] });
|
||||
if (url.includes('/api/vessels')) return json({ status: 'open', rows: [] });
|
||||
return realFetch(input, init);
|
||||
};
|
||||
});
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Keyless station-name regression against the real globe and bike-share tool.
|
||||
* Run with a dev or preview server: node scripts/qa-gbfs-names.mjs --url http://localhost:4173
|
||||
* GBFS documents are synthetic; no operator feed or provider key is required.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const urlIndex = process.argv.indexOf('--url');
|
||||
const appUrl =
|
||||
urlIndex < 0 ? 'http://localhost:4173' : process.argv[urlIndex + 1];
|
||||
const shots = path.resolve('qa-shots/gbfs-names');
|
||||
fs.mkdirSync(shots, { recursive: true });
|
||||
const information = {
|
||||
version: '3.0',
|
||||
data: {
|
||||
stations: [
|
||||
{
|
||||
station_id: 'english',
|
||||
name: [
|
||||
{ language: 'hu', text: 'Magyar tér' },
|
||||
{ language: 'en', text: 'QA English Square' },
|
||||
],
|
||||
},
|
||||
{
|
||||
station_id: 'short',
|
||||
name: [null, { language: 'en', text: ' ' }],
|
||||
short_name: [{ language: 'en-GB', text: 'QA Short Name' }],
|
||||
},
|
||||
{ station_id: 'unnamed', name: [null, {}, { language: 42, text: [] }] },
|
||||
{ station_id: 'legacy', name: ' QA Legacy Station ' },
|
||||
{
|
||||
station_id: 'fallback',
|
||||
name: [null, { language: 'hu', text: 'QA Magyar Station' }],
|
||||
},
|
||||
].map((station, index) => ({
|
||||
...station,
|
||||
lat: 30.2681 + index * 0.001,
|
||||
lon: -97.7469 + index * 0.001,
|
||||
capacity: 13,
|
||||
})),
|
||||
},
|
||||
};
|
||||
const status = {
|
||||
data: {
|
||||
stations: information.data.stations.map(({ station_id }) => ({
|
||||
station_id,
|
||||
num_bikes_available: 4,
|
||||
num_docks_available: 9,
|
||||
is_renting: true,
|
||||
is_returning: true,
|
||||
})),
|
||||
},
|
||||
};
|
||||
const browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
...(process.env.PUPPETEER_EXECUTABLE_PATH
|
||||
? { executablePath: process.env.PUPPETEER_EXECUTABLE_PATH }
|
||||
: {}),
|
||||
args:
|
||||
process.platform === 'darwin'
|
||||
? ['--use-angle=metal']
|
||||
: ['--use-gl=angle', '--use-angle=swiftshader'],
|
||||
});
|
||||
let checks = 0;
|
||||
const check = (name, actual, expected) => {
|
||||
assert.deepEqual(actual, expected, name);
|
||||
checks++;
|
||||
console.log(`[PASS] ${name}`);
|
||||
};
|
||||
async function open(mode = 'normal') {
|
||||
const page = await browser.newPage();
|
||||
await page.bringToFront();
|
||||
const errors = [];
|
||||
page.on('pageerror', (error) => errors.push(error.message));
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
await page.evaluateOnNewDocument(
|
||||
({ information, status, mode }) => {
|
||||
localStorage.clear();
|
||||
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
|
||||
window.__qaPainted = [];
|
||||
const paint = CanvasRenderingContext2D.prototype.fillText;
|
||||
CanvasRenderingContext2D.prototype.fillText = function (text, ...args) {
|
||||
if (
|
||||
String(text).startsWith('QA ') ||
|
||||
String(text) === 'Station unnamed'
|
||||
) {
|
||||
window.__qaPainted.push(String(text));
|
||||
if (window.__qaPainted.length > 100) window.__qaPainted.shift();
|
||||
}
|
||||
return paint.call(this, text, ...args);
|
||||
};
|
||||
const original = window.fetch.bind(window);
|
||||
window.fetch = async (input, init) => {
|
||||
const url = String(
|
||||
typeof input === 'string' || input instanceof URL ? input : input.url,
|
||||
);
|
||||
if (url.includes('/api/gbfs/')) {
|
||||
if (mode === 'failure') return new Response('{}', { status: 503 });
|
||||
const payload =
|
||||
mode === 'empty'
|
||||
? { data: { stations: [] } }
|
||||
: decodeURIComponent(url).includes('station_information')
|
||||
? information
|
||||
: status;
|
||||
return new Response(JSON.stringify(payload), {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
if (url.includes('/api/flights')) return new Response('{"states":[]}');
|
||||
if (url.includes('/api/military')) return new Response('{"ac":[]}');
|
||||
if (url.includes('/api/vessels'))
|
||||
return new Response('{"connected":false,"vessels":[]}');
|
||||
return original(input, init);
|
||||
};
|
||||
},
|
||||
{ information, status, mode },
|
||||
);
|
||||
await page.goto(appUrl, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager && window.__godsEyeView?.voiceCommands,
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
!document.getElementById('loading-screen') ||
|
||||
(document.getElementById('loading-screen').classList.contains('hidden') &&
|
||||
getComputedStyle(document.getElementById('loading-screen')).opacity ===
|
||||
'0'),
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await page.evaluate(async () => {
|
||||
const { dataManager, viewer } = window.__godsEyeView;
|
||||
for (const [id, layer] of dataManager.layers) {
|
||||
if (layer.enabled)
|
||||
await dataManager.setEnabled(id, false, { origin: 'user' });
|
||||
}
|
||||
const Cartesian3 = viewer.camera.position.constructor;
|
||||
viewer.camera.cancelFlight();
|
||||
viewer.camera.setView({
|
||||
destination: Cartesian3.fromDegrees(-97.7449, 30.2701, 3000),
|
||||
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
|
||||
});
|
||||
await dataManager.setEnabled('bikeshare', true, { origin: 'user' });
|
||||
window.__godsEyeView.requestRender('gbfs-name-qa');
|
||||
});
|
||||
return { page, errors };
|
||||
}
|
||||
try {
|
||||
const { page, errors } = await open();
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers.get('bikeshare').module.getStats()
|
||||
.count === 5,
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
const records = await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getAnalystRecords(),
|
||||
);
|
||||
check(
|
||||
'the layer retains all five stations and normalized names',
|
||||
records.map((r) => r.name),
|
||||
[
|
||||
'QA English Square',
|
||||
'QA Short Name',
|
||||
null,
|
||||
'QA Legacy Station',
|
||||
'QA Magyar Station',
|
||||
],
|
||||
);
|
||||
const tool = await page.evaluate(() =>
|
||||
window.__godsEyeView.voiceCommands.runner('get_bike_share', {
|
||||
area: { lat: 30.2701, lon: -97.7449, radius_km: 2 },
|
||||
}),
|
||||
);
|
||||
check('the real bike-share tool succeeds', tool.ok, true);
|
||||
check(
|
||||
'the tool preserves localized, short, unnamed, legacy and non-English names',
|
||||
tool.data.rows.map((r) => r.name).sort(),
|
||||
[
|
||||
null,
|
||||
'QA English Square',
|
||||
'QA Short Name',
|
||||
'QA Legacy Station',
|
||||
'QA Magyar Station',
|
||||
].sort(),
|
||||
);
|
||||
check(
|
||||
'availability remains attached to every station',
|
||||
tool.data.rows.every(
|
||||
(r) => r.bikes_available === 4 && r.docks_available === 9,
|
||||
),
|
||||
true,
|
||||
);
|
||||
const clickPoint = await page.evaluate(() => {
|
||||
window.__qaPainted = [];
|
||||
const { viewer } = window.__godsEyeView;
|
||||
const Cartesian3 = viewer.camera.position.constructor;
|
||||
const point = viewer.scene.cartesianToCanvasCoordinates(
|
||||
Cartesian3.fromDegrees(-97.7469, 30.2681),
|
||||
);
|
||||
const rect = viewer.scene.canvas.getBoundingClientRect();
|
||||
return { x: rect.left + point.x, y: rect.top + point.y };
|
||||
});
|
||||
await page.mouse.click(clickPoint.x, clickPoint.y);
|
||||
await page.waitForFunction(
|
||||
() => window.__qaPainted.includes('QA English Square'),
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
check(
|
||||
'a real pointer click selects the station with its localized name',
|
||||
await page.evaluate(() => window.__qaPainted.includes('QA English Square')),
|
||||
true,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
for (const [id, title] of [
|
||||
['english', 'QA English Square'],
|
||||
['short', 'QA Short Name'],
|
||||
['unnamed', 'Station unnamed'],
|
||||
]) {
|
||||
await page.evaluate((id) => {
|
||||
window.__qaPainted = [];
|
||||
const app = window.__godsEyeView;
|
||||
app.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module._selectBikeshareStationForTest(`austin-capmetro:${id}`);
|
||||
app.requestRender('gbfs-name-selection-qa');
|
||||
}, id);
|
||||
await page.waitForFunction(
|
||||
(title) => window.__qaPainted.includes(title),
|
||||
{ timeout: 10000 },
|
||||
title,
|
||||
);
|
||||
check(
|
||||
`the selected ${id} card paints the expected title`,
|
||||
await page.evaluate((title) => window.__qaPainted.includes(title), title),
|
||||
true,
|
||||
);
|
||||
await page.screenshot({ path: path.join(shots, `${id}.png`) });
|
||||
await page.keyboard.press('Escape');
|
||||
await page.waitForFunction(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getDetectableObjects()
|
||||
.every((record) => !record.skipLabel),
|
||||
);
|
||||
check(
|
||||
`Escape clears the ${id} card`,
|
||||
await page.evaluate(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getDetectableObjects()
|
||||
.filter((record) => record.skipLabel).length,
|
||||
),
|
||||
0,
|
||||
);
|
||||
}
|
||||
await page.setViewport({ width: 390, height: 844 });
|
||||
await page.evaluate(() => {
|
||||
window.__qaPainted = [];
|
||||
const app = window.__godsEyeView;
|
||||
// Use the supported clean view to inspect the mobile card clear of panels.
|
||||
app.styleManager.toggleCleanView(true);
|
||||
const Cartesian3 = app.viewer.camera.position.constructor;
|
||||
app.viewer.camera.cancelFlight();
|
||||
app.viewer.camera.setView({
|
||||
destination: Cartesian3.fromDegrees(-97.7449, 30.2701, 3000),
|
||||
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
|
||||
});
|
||||
app.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module._selectBikeshareStationForTest('austin-capmetro:english');
|
||||
app.requestRender('gbfs-name-mobile-qa');
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => window.__qaPainted.includes('QA English Square'),
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
check(
|
||||
'the mobile selected card paints its normalized name',
|
||||
await page.evaluate(() => window.__qaPainted.includes('QA English Square')),
|
||||
true,
|
||||
);
|
||||
await page.screenshot({ path: path.join(shots, 'mobile.png') });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('bikeshare', false, {
|
||||
origin: 'user',
|
||||
}),
|
||||
);
|
||||
check(
|
||||
'disabling clears selected cards and analyst records',
|
||||
await page.evaluate(() => ({
|
||||
count: window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getAnalystRecords().length,
|
||||
selected: window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getDetectableObjects()
|
||||
.filter((record) => record.skipLabel).length,
|
||||
})),
|
||||
{ count: 0, selected: 0 },
|
||||
);
|
||||
check('normal fixtures produce no page exceptions', errors, []);
|
||||
await page.close();
|
||||
for (const mode of ['empty', 'failure']) {
|
||||
const { page, errors } = await open(mode);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
!window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getStats().loading,
|
||||
);
|
||||
check(
|
||||
`${mode} leaves no rendered stations`,
|
||||
await page.evaluate(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('bikeshare')
|
||||
.module.getStats().count,
|
||||
),
|
||||
0,
|
||||
);
|
||||
check(`${mode} produces no page exceptions`, errors, []);
|
||||
await page.close();
|
||||
}
|
||||
console.log(`GBFS names: ${checks}/${checks} checks passed; ${appUrl}`);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
@@ -525,11 +525,11 @@ async function main() {
|
||||
const nowSec = Date.now() / 1000 + (T.timeOffsetSec || 0);
|
||||
const tRel = nowSec - T.epochMs / 1000;
|
||||
|
||||
if (url.includes('/api/opensky-track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/adsblol/trace')) {
|
||||
if (url.includes('/api/flights/track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/military/track')) {
|
||||
return Promise.resolve(jsonResponse({ timestamp: Math.floor(nowSec), trace: [] }));
|
||||
}
|
||||
if (url.includes('/api/opensky')) {
|
||||
if (url.includes('/api/flights')) {
|
||||
T.__hitsGuard = ++window.__TURN_HITS.opensky;
|
||||
const row = (id, callsign, s, altM, category) => [
|
||||
id, callsign, 'Synthetica',
|
||||
@@ -548,7 +548,7 @@ async function main() {
|
||||
if (T.heli65.active) states.push(row(T.heli65.icao, T.heli65.callsign, T.stateAt(T.heli65, tRel), T.heli65.altM, T.heli65.category));
|
||||
return Promise.resolve(jsonResponse({ time: Math.floor(nowSec), states }));
|
||||
}
|
||||
if (url.includes('/api/adsblol/mil')) {
|
||||
if (url.includes('/api/military')) {
|
||||
window.__TURN_HITS.mil++;
|
||||
const ac = T.military.map((m) => {
|
||||
const s = T.stateAt(m, tRel);
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env node
|
||||
/** Mapped installations: isolated camera navigation, cached revisits and Contacts release. */
|
||||
import fs from 'node:fs';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { moveCamera } from './qa-journey-recorder.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-installation-navigation.mjs <url> [--baseline]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const baseline = args.includes('--baseline');
|
||||
const result = { views: [], errors: [], forbidden: [], failures: [] };
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
args: ['--use-angle=metal', '--disable-background-timer-throttling'],
|
||||
protocolTimeout: 120000,
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
const tiles = [];
|
||||
page.on('request', (r) => {
|
||||
const u = new URL(r.url());
|
||||
if (
|
||||
u.hostname === 'tiles.openfreemap.org' &&
|
||||
/\/\d+\/\d+\/\d+(\.pbf)?$/.test(u.pathname)
|
||||
)
|
||||
tiles.push(u.pathname);
|
||||
if (
|
||||
/overpass/i.test(u.hostname) ||
|
||||
u.hostname === 'nominatim.openstreetmap.org'
|
||||
)
|
||||
result.forbidden.push(u.hostname);
|
||||
});
|
||||
page.on('pageerror', (e) => result.errors.push(e.message));
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') result.errors.push(m.text());
|
||||
});
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
await page.goto(
|
||||
args.find((a) => /^https?:/.test(a)),
|
||||
{ waitUntil: 'domcontentloaded' },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await page.evaluate(async () => {
|
||||
const g = window.__godsEyeView;
|
||||
for (const [id] of g.dataManager.layers)
|
||||
await g.dataManager.setEnabled(id, false);
|
||||
g.viewer.scene.screenSpaceCameraController.enableCollisionDetection = false;
|
||||
window.installationEvents = { changed: 0, end: 0 };
|
||||
window.installationEntityRefs = new Map();
|
||||
g.viewer.camera.changed.addEventListener(
|
||||
() => window.installationEvents.changed++,
|
||||
);
|
||||
g.viewer.camera.moveEnd.addEventListener(
|
||||
() => window.installationEvents.end++,
|
||||
);
|
||||
});
|
||||
let low = 60000,
|
||||
high = 400000;
|
||||
for (let i = 0; i < 9; i++) {
|
||||
const height = (low + high) / 2;
|
||||
await moveCamera(page, { lat: 32.72, lon: -117.16, height, pitch: -90 }, 0);
|
||||
const fits = await page.evaluate(async () => {
|
||||
const { installationTileZoom } =
|
||||
await import('/src/layers/installations/source.js');
|
||||
const r = window.__godsEyeView.viewer.camera.computeViewRectangle();
|
||||
const d = 180 / Math.PI;
|
||||
return (
|
||||
r &&
|
||||
installationTileZoom({
|
||||
west: r.west * d,
|
||||
east: r.east * d,
|
||||
south: r.south * d,
|
||||
north: r.north * d,
|
||||
}) !== null
|
||||
);
|
||||
});
|
||||
if (fits) low = height;
|
||||
else high = height;
|
||||
}
|
||||
const widestHeight = Math.floor(low);
|
||||
result.widestHeight = widestHeight;
|
||||
result.firstTooWideHeight = Math.ceil(high);
|
||||
let enabled = false;
|
||||
const places = [
|
||||
['san-diego', 32.72, -117.16],
|
||||
['pendleton', 33.33, -117.4],
|
||||
['miramar', 32.87, -117.14],
|
||||
['san-diego-return', 32.72, -117.16],
|
||||
];
|
||||
for (const height of [
|
||||
5000,
|
||||
20000,
|
||||
60000,
|
||||
150000,
|
||||
widestHeight,
|
||||
400000,
|
||||
5000,
|
||||
]) {
|
||||
for (const [name, lat, lon] of height >= 150000
|
||||
? places.slice(0, 1)
|
||||
: places) {
|
||||
const before = tiles.length;
|
||||
const previous = await page.evaluate(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats().lastUpdate,
|
||||
);
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat, lon, height, pitch: -90 },
|
||||
enabled ? 0.6 : 0,
|
||||
);
|
||||
const stop = Date.now();
|
||||
if (!enabled) {
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled(
|
||||
'military-installations',
|
||||
true,
|
||||
),
|
||||
);
|
||||
enabled = true;
|
||||
}
|
||||
await page
|
||||
.waitForFunction(
|
||||
(prev, wide) => {
|
||||
const s = window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats();
|
||||
return (
|
||||
!s.loading &&
|
||||
(s.lastUpdate > prev || (wide && s.status === 'zoom-in'))
|
||||
);
|
||||
},
|
||||
{ timeout: 6000 },
|
||||
previous,
|
||||
height === 400000,
|
||||
)
|
||||
.catch(() => {});
|
||||
const snapshot = await page.evaluate(() => {
|
||||
const g = window.__godsEyeView;
|
||||
const ds = g.viewer.dataSources.getByName('military-installations')[0];
|
||||
const points = ds.entities.values.filter((e) => e.point);
|
||||
const stable = points.every(
|
||||
(e) =>
|
||||
!window.installationEntityRefs.has(e.id) ||
|
||||
window.installationEntityRefs.get(e.id) === e,
|
||||
);
|
||||
window.installationEntityRefs = new Map(points.map((e) => [e.id, e]));
|
||||
return {
|
||||
...g.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats(),
|
||||
stable,
|
||||
ids: points.map((e) => e.id),
|
||||
events: { ...window.installationEvents },
|
||||
};
|
||||
});
|
||||
const row = {
|
||||
name,
|
||||
height,
|
||||
ms: Date.now() - stop,
|
||||
tiles: tiles.length - before,
|
||||
updated: snapshot.lastUpdate > previous,
|
||||
...snapshot,
|
||||
};
|
||||
if (
|
||||
(!row.stable && result.views.at(-1)?.height === height) ||
|
||||
new Set(row.ids).size !== row.ids.length
|
||||
)
|
||||
result.failures.push(
|
||||
`${name}/${height}: unstable or duplicate site entities`,
|
||||
);
|
||||
if (name.endsWith('return')) {
|
||||
const first = result.views.find(
|
||||
(v) => v.name === 'san-diego' && v.height === height,
|
||||
);
|
||||
if (
|
||||
first &&
|
||||
JSON.stringify([...first.ids].sort()) !==
|
||||
JSON.stringify([...row.ids].sort())
|
||||
)
|
||||
result.failures.push(
|
||||
`${name}/${height}: changed site ids on revisit`,
|
||||
);
|
||||
}
|
||||
result.views.push(row);
|
||||
console.log(JSON.stringify(row));
|
||||
if (!row.updated && row.status !== 'zoom-in')
|
||||
result.failures.push(`${name}/${height}: no reload`);
|
||||
if (row.ms > 1500 && row.status !== 'zoom-in')
|
||||
result.failures.push(`${name}/${height}: ${row.ms} ms`);
|
||||
if (name.endsWith('return') && row.tiles)
|
||||
result.failures.push(`${name}/${height}: revisit fetched ${row.tiles}`);
|
||||
}
|
||||
}
|
||||
for (const gesture of [
|
||||
'wheel',
|
||||
'keyboard',
|
||||
'contacts-close',
|
||||
'contacts-clear',
|
||||
]) {
|
||||
let previous = await page.evaluate(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats().lastUpdate,
|
||||
);
|
||||
if (gesture === 'wheel') {
|
||||
await page.mouse.move(640, 400);
|
||||
await page.mouse.wheel({ deltaY: -350 });
|
||||
} else if (gesture === 'keyboard') {
|
||||
await page.evaluate(() =>
|
||||
document.getElementById('search-toggle').click(),
|
||||
);
|
||||
await page.focus('#location-search');
|
||||
await page.evaluate(() => {
|
||||
document.getElementById('location-search').value = '';
|
||||
});
|
||||
await page.type('#location-search', '33.33, -117.40');
|
||||
await page.keyboard.press('Enter');
|
||||
await sleep(4500);
|
||||
} else {
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat: 32.87, lon: -117.14, height: 5000, pitch: -90 },
|
||||
0.6,
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.viewer.dataSources
|
||||
.getByName('military-installations')[0]
|
||||
.entities.values.some((e) => e.point),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
await page.evaluate(() => {
|
||||
const button = document.getElementById('global-context-flights-btn');
|
||||
if (button.getAttribute('aria-selected') !== 'true') button.click();
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const g = window.__godsEyeView;
|
||||
return (
|
||||
g.dataManager.isEffectivelyEnabled('military-awareness') &&
|
||||
!g.dataManager.layers.get('military-awareness').module.getParams()
|
||||
.passive
|
||||
);
|
||||
},
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
await sleep(1000);
|
||||
await page.evaluate(async () => {
|
||||
const g = window.__godsEyeView;
|
||||
const ds = g.viewer.dataSources.getByName('military-installations')[0];
|
||||
const id = ds.entities.values.find((e) => e.point).id;
|
||||
window.installationSubjectId = id;
|
||||
await g.dataManager.layers
|
||||
.get('military-awareness')
|
||||
.module.focusTarget('military-installations', id);
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const a =
|
||||
window.__godsEyeView.dataManager.layers.get(
|
||||
'military-awareness',
|
||||
).module;
|
||||
const subject = a.getContextSnapshot()?.subject;
|
||||
return (
|
||||
subject?.layerId === 'military-installations' &&
|
||||
subject.id === window.installationSubjectId &&
|
||||
!a._getAwarenessNavigationStateForTest().pendingSelectionKey
|
||||
);
|
||||
},
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() => !window.__godsEyeView.viewer.camera._currentFlight,
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats().coverage.kind === 'subject',
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document
|
||||
.getElementById('global-context-flights-btn')
|
||||
.getAttribute('aria-busy') === 'false',
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
const anchored = await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats(),
|
||||
);
|
||||
if (anchored.coverage.kind !== 'subject')
|
||||
result.failures.push(`${gesture}: Contacts did not acquire subject`);
|
||||
result.contextClear = await page.evaluate(async (clear) => {
|
||||
if (clear) {
|
||||
const { clearSelectedEntityContextForLayer, getContextStore } =
|
||||
await import('/src/data/contextStore.js');
|
||||
const store = getContextStore();
|
||||
const before = {
|
||||
selectedId: store.selectedEntityId,
|
||||
record: store.entities.get(store.selectedEntityId)?.id,
|
||||
subject: window.__godsEyeView.dataManager.layers
|
||||
.get('military-awareness')
|
||||
.module.getContextSnapshot()?.subject,
|
||||
};
|
||||
clearSelectedEntityContextForLayer('military-installations');
|
||||
return before;
|
||||
} else document.getElementById('global-context-flights-btn').click();
|
||||
}, gesture === 'contacts-clear');
|
||||
if (gesture === 'contacts-clear') {
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
!window.__godsEyeView.dataManager.layers
|
||||
.get('military-awareness')
|
||||
.module.getContextSnapshot(),
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
} else {
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const button = document.getElementById(
|
||||
'global-context-flights-btn',
|
||||
);
|
||||
return (
|
||||
button.getAttribute('aria-selected') === 'false' &&
|
||||
button.getAttribute('aria-busy') === 'false'
|
||||
);
|
||||
},
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
}
|
||||
await page.waitForFunction(
|
||||
() => !window.__godsEyeView.viewer.camera._currentFlight,
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
await sleep(500);
|
||||
previous = await page.evaluate(
|
||||
() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats().lastUpdate,
|
||||
);
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat: 33.33, lon: -117.4, height: 5000, pitch: -90 },
|
||||
0.6,
|
||||
);
|
||||
}
|
||||
await page
|
||||
.waitForFunction(
|
||||
(prev) => {
|
||||
const s = window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats();
|
||||
return (
|
||||
!s.loading && s.lastUpdate > prev && s.coverage.kind === 'viewport'
|
||||
);
|
||||
},
|
||||
{ timeout: 6000 },
|
||||
previous,
|
||||
)
|
||||
.catch(() => result.failures.push(`${gesture}: no viewport reload`));
|
||||
const row = await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats(),
|
||||
);
|
||||
if (gesture.startsWith('contacts') && row.count !== 1)
|
||||
result.failures.push(
|
||||
`${gesture}: destination count ${row.count}, expected 1`,
|
||||
);
|
||||
result.views.push({ name: gesture, ...row });
|
||||
console.log(JSON.stringify({ name: gesture, ...row }));
|
||||
}
|
||||
await page.screenshot({ path: 'qa-shots/installations-navigation.png' });
|
||||
} finally {
|
||||
fs.mkdirSync('qa-shots', { recursive: true });
|
||||
fs.writeFileSync(
|
||||
`qa-shots/installations-navigation-${baseline ? 'before' : 'after'}.json`,
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
await browser.close();
|
||||
}
|
||||
if (
|
||||
!baseline &&
|
||||
(result.failures.length || result.errors.length || result.forbidden.length)
|
||||
)
|
||||
process.exitCode = 1;
|
||||
@@ -0,0 +1,404 @@
|
||||
/** Shared browser assertions for installation label persistence and selection. */
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import { moveCamera, createScreencast } from './qa-journey-recorder.mjs';
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
export async function checkInstallationPan(page, dir) {
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('local-datacenters', true),
|
||||
);
|
||||
await sleep(3000);
|
||||
await page.evaluate(async () => {
|
||||
const { getOverlayPaintRect } =
|
||||
await import('/src/overlays/worldOverlay.js');
|
||||
const g = window.__godsEyeView;
|
||||
const installations = g.dataManager.layers.get(
|
||||
'military-installations',
|
||||
).module;
|
||||
const centerIds = [];
|
||||
for (let i = 0; i < g.viewer.dataSources.length; i++)
|
||||
for (const e of g.viewer.dataSources.get(i).entities.values)
|
||||
if (e.__localLayerId === 'local-datacenters') centerIds.push(e.id);
|
||||
window.__polishPanFrames = [];
|
||||
window.__stopPolishPan = g.viewer.scene.postRender.addEventListener(() => {
|
||||
const sites = [],
|
||||
centers = [];
|
||||
installations.visitNamedMarkers((id) => {
|
||||
if (getOverlayPaintRect('military-installations', id)) sites.push(id);
|
||||
});
|
||||
for (const id of centerIds)
|
||||
if (getOverlayPaintRect('local-datacenters', id)) centers.push(id);
|
||||
window.__polishPanFrames.push({ t: performance.now(), sites, centers });
|
||||
});
|
||||
});
|
||||
await moveCamera(page, { lon: -99, lat: 39, height: 6500000, pitch: -90 }, 8);
|
||||
const frames = await page.evaluate(() => {
|
||||
window.__stopPolishPan();
|
||||
return window.__polishPanFrames;
|
||||
});
|
||||
const count = (key) => {
|
||||
let appeared = 0,
|
||||
disappeared = 0,
|
||||
reappeared = 0;
|
||||
const seen = new Set(frames[0][key]);
|
||||
for (let i = 1; i < frames.length; i++) {
|
||||
const previous = new Set(frames[i - 1][key]),
|
||||
next = new Set(frames[i][key]);
|
||||
frames[i][`${key}Appeared`] = [...next].filter((id) => !previous.has(id));
|
||||
frames[i][`${key}Disappeared`] = [...previous].filter(
|
||||
(id) => !next.has(id),
|
||||
);
|
||||
for (const id of frames[i][`${key}Appeared`]) {
|
||||
appeared++;
|
||||
if (seen.has(id)) reappeared++;
|
||||
seen.add(id);
|
||||
}
|
||||
disappeared += frames[i][`${key}Disappeared`].length;
|
||||
}
|
||||
const mean =
|
||||
frames.reduce((sum, f) => sum + f[key].length, 0) / frames.length;
|
||||
return {
|
||||
appeared,
|
||||
disappeared,
|
||||
reappeared,
|
||||
mean,
|
||||
eventsPerLabel: (appeared + disappeared) / mean,
|
||||
};
|
||||
};
|
||||
const result = {
|
||||
frames: frames.length,
|
||||
installations: count('sites'),
|
||||
datacenters: count('centers'),
|
||||
};
|
||||
await fs.writeFile(
|
||||
`${dir}/slow-pan-frames.json`,
|
||||
JSON.stringify(frames, null, 2),
|
||||
);
|
||||
await fs.writeFile(`${dir}/slow-pan.json`, JSON.stringify(result, null, 2));
|
||||
assert.ok(frames.length > 100);
|
||||
assert.ok(
|
||||
result.installations.mean >= 5 && result.datacenters.mean >= 5,
|
||||
'both real layers contribute visible labels on the same pan',
|
||||
);
|
||||
assert.ok(
|
||||
frames.every((f) => f.sites.length <= 24),
|
||||
'shared-domain budget cannot exceed the installation title cap',
|
||||
);
|
||||
assert.ok(
|
||||
result.installations.eventsPerLabel <=
|
||||
result.datacenters.eventsPerLabel + 0.1,
|
||||
'installation label transitions per label match Data Centers on the same slow pan',
|
||||
);
|
||||
assert.ok(
|
||||
result.installations.reappeared <= result.datacenters.reappeared,
|
||||
'installation labels do not flicker off and back on more than Data Centers',
|
||||
);
|
||||
await page.screenshot({ path: `${dir}/slow-pan-side-by-side.png` });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('local-datacenters', false),
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function checkInstallationSelection(
|
||||
page,
|
||||
dir,
|
||||
{ video = true } = {},
|
||||
) {
|
||||
const result = [];
|
||||
const recorder = video
|
||||
? await createScreencast(page, `${dir}/selection-orbit`, { fps: 24 })
|
||||
: null;
|
||||
await recorder?.start();
|
||||
try {
|
||||
for (const site of [
|
||||
{
|
||||
key: 'mabry',
|
||||
name: 'Camp Mabry',
|
||||
lon: -97.765,
|
||||
lat: 30.3125,
|
||||
height: 3500,
|
||||
},
|
||||
{
|
||||
key: 'naval',
|
||||
name: 'Naval Base San Diego',
|
||||
lon: -117.119,
|
||||
lat: 32.684,
|
||||
height: 6500,
|
||||
},
|
||||
]) {
|
||||
await moveCamera(page, { ...site, pitch: -80 }, 1.5);
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const s = window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats();
|
||||
return !s.loading && s.count > 0;
|
||||
},
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await sleep(1500);
|
||||
const picked = await page.evaluate(async (site) => {
|
||||
const C = await import('/node_modules/cesium/Build/Cesium/index.js');
|
||||
const g = window.__godsEyeView,
|
||||
layer = g.dataManager.layers.get('military-installations').module;
|
||||
const record = layer
|
||||
.getNearby(C.Cartesian3.fromDegrees(site.lon, site.lat), 100000, 1000)
|
||||
.find((r) => r.name === site.name && r.footprints?.length);
|
||||
if (!record) throw new Error(`No polygon for ${site.name}`);
|
||||
let point;
|
||||
layer.visitNamedMarkers((id, p) => {
|
||||
if (id === record.id) point = p.position;
|
||||
});
|
||||
const screen = C.SceneTransforms.worldToWindowCoordinates(
|
||||
g.viewer.scene,
|
||||
point,
|
||||
);
|
||||
window.__polishSelectedId = record.id;
|
||||
window.__polishTitlePaints = [];
|
||||
const canvas = document.getElementById('world-overlay-canvas'),
|
||||
ctx = canvas.getContext('2d');
|
||||
const original = ctx.fillText;
|
||||
const matches = (text) =>
|
||||
String(text).toLowerCase() === site.name.toLowerCase();
|
||||
ctx.fillText = function (text, ...args) {
|
||||
if (matches(text))
|
||||
window.__polishTitlePaints.push({ t: performance.now(), text });
|
||||
return original.call(this, text, ...args);
|
||||
};
|
||||
window.__restorePolishPaint = () => {
|
||||
ctx.fillText = original;
|
||||
};
|
||||
window.__polishSelectionFrames = [];
|
||||
const { getSelectedEntityContext } =
|
||||
await import('/src/data/contextStore.js');
|
||||
const { getOverlayPaintRect, getWorldOverlayDiagnostics } =
|
||||
await import('/src/overlays/worldOverlay.js');
|
||||
const knownFills = new Set();
|
||||
const fillEntities = () => {
|
||||
const list = [];
|
||||
for (let i = 0; i < g.viewer.dataSources.length; i++)
|
||||
for (const e of g.viewer.dataSources.get(i).entities.values)
|
||||
if (e.id.startsWith(`${record.id}:selected-fill:`)) list.push(e);
|
||||
for (const e of list) knownFills.add(e);
|
||||
return list;
|
||||
};
|
||||
// Geometry-instance ids prove the ground primitive actually built, not just an entity declaration.
|
||||
const primitives = () => {
|
||||
let count = 0;
|
||||
const walk = (collection) => {
|
||||
if (collection?._collectionsArray) {
|
||||
for (const child of collection._collectionsArray) walk(child);
|
||||
return;
|
||||
}
|
||||
for (let i = 0; i < (collection?.length || 0); i++) {
|
||||
const p = collection.get(i);
|
||||
if (
|
||||
p.ready &&
|
||||
typeof p.getGeometryInstanceAttributes === 'function'
|
||||
) {
|
||||
for (const e of knownFills) {
|
||||
try {
|
||||
const a = p.getGeometryInstanceAttributes(e);
|
||||
if (a && (!a.show || a.show[0])) count++;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (typeof p.get === 'function' || p._collectionsArray) walk(p);
|
||||
}
|
||||
};
|
||||
walk(g.viewer.scene.groundPrimitives);
|
||||
return count;
|
||||
};
|
||||
window.__polishFillProbe = () => ({
|
||||
entities: fillEntities().length,
|
||||
primitives: primitives(),
|
||||
classification: fillEntities().map((e) =>
|
||||
e.polygon.classificationType.getValue(),
|
||||
),
|
||||
selected: getSelectedEntityContext()?.id || null,
|
||||
});
|
||||
window.__stopPolishSelection =
|
||||
g.viewer.scene.postRender.addEventListener(() => {
|
||||
window.__polishSelectionFrames.push({
|
||||
paints: window.__polishTitlePaints.splice(0).length,
|
||||
own: Boolean(
|
||||
getOverlayPaintRect('military-installations', record.id),
|
||||
),
|
||||
tracked: Boolean(
|
||||
getOverlayPaintRect('tracked', `installations:${record.id}`),
|
||||
),
|
||||
selected: getSelectedEntityContext()?.id === record.id,
|
||||
total:
|
||||
getWorldOverlayDiagnostics().paintedBySource[
|
||||
'military-installations'
|
||||
] || 0,
|
||||
});
|
||||
});
|
||||
return {
|
||||
id: record.id,
|
||||
x: screen.x,
|
||||
y: screen.y,
|
||||
parts: record.footprints.length,
|
||||
};
|
||||
}, site);
|
||||
await page.screenshot({
|
||||
path: `${dir}/after-${site.key}-unselected.png`,
|
||||
});
|
||||
await page.mouse.click(picked.x, picked.y);
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const p = window.__polishFillProbe();
|
||||
return p.selected === window.__polishSelectedId && p.primitives > 0;
|
||||
},
|
||||
{ timeout: 20000 },
|
||||
)
|
||||
.catch(async (error) => {
|
||||
await page.screenshot({ path: `${dir}/selection-failure.png` });
|
||||
const diagnostic = await page.evaluate(() => {
|
||||
const g = window.__godsEyeView;
|
||||
const describe = (p) => ({
|
||||
type: p.constructor.name,
|
||||
ready: p.ready,
|
||||
length: p.length,
|
||||
keys: Object.keys(p).filter((k) =>
|
||||
/primitive|instance|collection/i.test(k),
|
||||
),
|
||||
ids: p._instanceIds?.map((id) => id.id || String(id)),
|
||||
children:
|
||||
typeof p.get === 'function'
|
||||
? Array.from({ length: p.length }, (_, i) =>
|
||||
describe(p.get(i)),
|
||||
)
|
||||
: p._collectionsArray?.map(describe),
|
||||
});
|
||||
return {
|
||||
probe: window.__polishFillProbe(),
|
||||
stats: g.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats(),
|
||||
primitives: describe(g.viewer.scene.groundPrimitives),
|
||||
frames: window.__polishSelectionFrames.slice(-5),
|
||||
};
|
||||
});
|
||||
await fs.writeFile(
|
||||
`${dir}/selection-failure.json`,
|
||||
JSON.stringify({ picked, ...diagnostic }, null, 2),
|
||||
);
|
||||
throw error;
|
||||
});
|
||||
await sleep(500);
|
||||
const selected = await page.evaluate(() => window.__polishFillProbe());
|
||||
await page.screenshot({ path: `${dir}/after-selection-${site.key}.png` });
|
||||
// Close, oblique angles expose surface clipping and floating fills.
|
||||
for (const heading of [30, 120]) {
|
||||
await page.evaluate(
|
||||
async (site, heading) => {
|
||||
const C =
|
||||
await import('/node_modules/cesium/Build/Cesium/index.js');
|
||||
const g = window.__godsEyeView;
|
||||
const target = C.Cartesian3.fromDegrees(
|
||||
site.lon,
|
||||
site.lat,
|
||||
site.key === 'mabry' ? 180 : 10,
|
||||
);
|
||||
g.viewer.camera.lookAt(
|
||||
target,
|
||||
new C.HeadingPitchRange(
|
||||
C.Math.toRadians(heading),
|
||||
C.Math.toRadians(-30),
|
||||
800,
|
||||
),
|
||||
);
|
||||
g.viewer.camera.lookAtTransform(C.Matrix4.IDENTITY);
|
||||
},
|
||||
site,
|
||||
heading,
|
||||
);
|
||||
await sleep(1700);
|
||||
assert.ok(
|
||||
(await page.evaluate(() => window.__polishFillProbe())).selected,
|
||||
'orbit must preserve installation selection',
|
||||
);
|
||||
await page.screenshot({
|
||||
path: `${dir}/after-${site.key}-orbit-${heading}.png`,
|
||||
});
|
||||
}
|
||||
await moveCamera(page, { ...site, pitch: -80 }, 1);
|
||||
await sleep(900);
|
||||
const empty = await page.evaluate(() => {
|
||||
const g = window.__godsEyeView;
|
||||
for (let y = 250; y < 650; y += 75)
|
||||
for (let x = 400; x < 850; x += 75) {
|
||||
const pick = g.viewer.scene.pick({ x, y });
|
||||
if (!pick?.id?.installationId && !pick?.id?.gevTrackedId)
|
||||
return { x, y };
|
||||
}
|
||||
throw new Error('No empty map point');
|
||||
});
|
||||
await page.mouse.click(empty.x, empty.y);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__polishFillProbe().selected === null &&
|
||||
window.__polishFillProbe().entities === 0,
|
||||
);
|
||||
await sleep(600);
|
||||
const deselected = await page.evaluate(() => {
|
||||
window.__stopPolishSelection();
|
||||
window.__restorePolishPaint();
|
||||
return {
|
||||
...window.__polishFillProbe(),
|
||||
frames: window.__polishSelectionFrames,
|
||||
};
|
||||
});
|
||||
assert.equal(selected.entities, picked.parts);
|
||||
assert.equal(selected.primitives, picked.parts);
|
||||
assert.ok(
|
||||
selected.classification.every((c) => c === 1),
|
||||
'Google 3D tile classification',
|
||||
);
|
||||
assert.equal(deselected.entities, 0);
|
||||
assert.equal(deselected.primitives, 0);
|
||||
const selectedFrames = deselected.frames.filter(
|
||||
(f) => f.selected && f.own,
|
||||
);
|
||||
assert.ok(selectedFrames.length > 10);
|
||||
assert.ok(
|
||||
selectedFrames.every((f) => !f.tracked && f.paints === 1),
|
||||
'selected record has exactly one painted overlay element',
|
||||
);
|
||||
assert.ok(
|
||||
deselected.frames
|
||||
.slice(-5)
|
||||
.every((f) => !f.selected && f.own && !f.tracked && f.paints === 1),
|
||||
'deselect restores the one ambient entry',
|
||||
);
|
||||
await page.screenshot({
|
||||
path: `${dir}/after-${site.key}-deselected.png`,
|
||||
});
|
||||
result.push({ site, picked, selected, deselected });
|
||||
}
|
||||
} finally {
|
||||
await fs.writeFile(
|
||||
`${dir}/selection-checks.json`,
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
if (recorder)
|
||||
await fs.writeFile(
|
||||
`${dir}/selection-video.json`,
|
||||
JSON.stringify(await recorder.stop(), null, 2),
|
||||
);
|
||||
}
|
||||
return result.map(({ site, picked, selected, deselected }) => ({
|
||||
site: site.name,
|
||||
id: picked.id,
|
||||
selected,
|
||||
deselected: {
|
||||
entities: deselected.entities,
|
||||
primitives: deselected.primitives,
|
||||
},
|
||||
frames: deselected.frames.length,
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* qa-journey-recorder.mjs — shared support for journey QA gates (not a gate).
|
||||
*
|
||||
* - `createScreencast(page, dir)` records CDP `Page.startScreencast` frames
|
||||
* (every compositor frame) with their real timestamps and builds
|
||||
* `<dir>/clip.mp4` through ffmpeg's concat demuxer with per-frame
|
||||
* durations, so held frames last as long as they did on screen.
|
||||
* - `installFrameProbe(page)` measures every animation frame in the page:
|
||||
* the requestAnimationFrame interval and, for frames Cesium rendered, the
|
||||
* CPU time between `scene.preUpdate` and `scene.postRender`.
|
||||
* - `frameStats(frames)` reduces a probe window to count / mean / p50 / p95.
|
||||
* - `moveCamera(page, view, seconds)` eases the camera like a drag or wheel
|
||||
* zoom rather than teleporting it.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
/** Percentile of a numeric list (nearest rank); NaN for an empty list. */
|
||||
export function percentile(values, p) {
|
||||
if (!values.length) return Number.NaN;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[
|
||||
Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)
|
||||
];
|
||||
}
|
||||
|
||||
/** Summarize probe frames: rAF interval and Cesium render CPU time (ms). */
|
||||
export function frameStats(frames) {
|
||||
const intervals = frames.map((f) => f.dt).filter(Number.isFinite);
|
||||
const renders = frames.map((f) => f.renderMs).filter(Number.isFinite);
|
||||
const round = (v) => (Number.isFinite(v) ? Math.round(v * 10) / 10 : null);
|
||||
const mean = (list) =>
|
||||
list.length ? list.reduce((a, b) => a + b, 0) / list.length : Number.NaN;
|
||||
return {
|
||||
frames: intervals.length,
|
||||
frameMean: round(mean(intervals)),
|
||||
frameP50: round(percentile(intervals, 50)),
|
||||
frameP95: round(percentile(intervals, 95)),
|
||||
longFrames: intervals.filter((v) => v > 50).length,
|
||||
renderedFrames: renders.length,
|
||||
renderMean: round(mean(renders)),
|
||||
renderP50: round(percentile(renders, 50)),
|
||||
renderP95: round(percentile(renders, 95)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Install the in-page frame probe once per document; returns nothing. */
|
||||
export async function installFrameProbe(page) {
|
||||
await page.evaluate(() => {
|
||||
if (window.__gevFrameProbe) return;
|
||||
const { viewer } = window.__godsEyeView;
|
||||
const probe = { frames: [], started: 0 };
|
||||
window.__gevFrameProbe = probe;
|
||||
let last = performance.now();
|
||||
let renderStart = null;
|
||||
let pendingRender = null;
|
||||
viewer.scene.preUpdate.addEventListener(() => {
|
||||
renderStart = performance.now();
|
||||
});
|
||||
viewer.scene.postRender.addEventListener(() => {
|
||||
if (renderStart !== null) pendingRender = performance.now() - renderStart;
|
||||
renderStart = null;
|
||||
});
|
||||
const tick = (now) => {
|
||||
probe.frames.push({ t: now, dt: now - last, renderMs: pendingRender });
|
||||
if (probe.frames.length > 40_000) probe.frames.splice(0, 10_000);
|
||||
pendingRender = null;
|
||||
last = now;
|
||||
requestAnimationFrame(tick);
|
||||
};
|
||||
requestAnimationFrame(tick);
|
||||
});
|
||||
}
|
||||
|
||||
/** Read probe frames with t in [t0, t1) (page performance.now() times). */
|
||||
export async function readFrames(page, t0, t1 = Infinity) {
|
||||
return page.evaluate(
|
||||
(a, b) =>
|
||||
window.__gevFrameProbe.frames.filter(
|
||||
(f) => f.t >= a && f.t < (b ?? Infinity),
|
||||
),
|
||||
t0,
|
||||
Number.isFinite(t1) ? t1 : null,
|
||||
);
|
||||
}
|
||||
|
||||
/** Page clock in ms (performance.now()). */
|
||||
export function pageNow(page) {
|
||||
return page.evaluate(() => performance.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Ease the camera to a view over `seconds`, resolving when it lands.
|
||||
* @param {import('puppeteer').Page} page Page.
|
||||
* @param {{lat:number, lon:number, height:number, heading?:number, pitch?:number}} view
|
||||
* Destination (degrees, metres).
|
||||
* @param {number} seconds Flight duration; 0 jumps.
|
||||
*/
|
||||
export async function moveCamera(page, view, seconds) {
|
||||
await page.evaluate(
|
||||
(v, duration) =>
|
||||
new Promise((resolve) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
const C = viewer.camera;
|
||||
C.cancelFlight();
|
||||
C.flyTo({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({
|
||||
latitude: (v.lat * Math.PI) / 180,
|
||||
longitude: (v.lon * Math.PI) / 180,
|
||||
height: v.height,
|
||||
}),
|
||||
orientation: {
|
||||
heading: ((v.heading ?? 0) * Math.PI) / 180,
|
||||
pitch: ((v.pitch ?? -45) * Math.PI) / 180,
|
||||
roll: 0,
|
||||
},
|
||||
duration,
|
||||
// Straight, eased motion like a drag or wheel zoom, not an arc.
|
||||
maximumHeight: Math.max(v.height, C.positionCartographic.height),
|
||||
complete: resolve,
|
||||
cancel: resolve,
|
||||
});
|
||||
}),
|
||||
view,
|
||||
seconds,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the page with CDP screencast frames and real per-frame timing.
|
||||
* @param {import('puppeteer').Page} page Page to record.
|
||||
* @param {string} dir Output directory (frames + clip.mp4).
|
||||
* @param {{maxWidth?: number, maxHeight?: number, quality?: number, fps?: number|null}} [options]
|
||||
*/
|
||||
export async function createScreencast(page, dir, options = {}) {
|
||||
const {
|
||||
maxWidth = 1280,
|
||||
maxHeight = 800,
|
||||
quality = 70,
|
||||
fps = null,
|
||||
} = options;
|
||||
const framesDir = path.join(dir, 'frames');
|
||||
fs.rmSync(framesDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(framesDir, { recursive: true });
|
||||
const cdp = await page.createCDPSession();
|
||||
const frames = [];
|
||||
let recording = false;
|
||||
cdp.on('Page.screencastFrame', async (frame) => {
|
||||
const file = path.join(
|
||||
framesDir,
|
||||
`f${String(frames.length).padStart(5, '0')}.jpg`,
|
||||
);
|
||||
if (recording) {
|
||||
fs.writeFileSync(file, Buffer.from(frame.data, 'base64'));
|
||||
frames.push({ file, timestamp: frame.metadata.timestamp });
|
||||
}
|
||||
await cdp
|
||||
.send('Page.screencastFrameAck', { sessionId: frame.sessionId })
|
||||
.catch(() => {});
|
||||
});
|
||||
return {
|
||||
frames,
|
||||
async start() {
|
||||
recording = true;
|
||||
await cdp.send('Page.startScreencast', {
|
||||
format: 'jpeg',
|
||||
quality,
|
||||
maxWidth,
|
||||
maxHeight,
|
||||
everyNthFrame: 1,
|
||||
});
|
||||
},
|
||||
/** Stop, encode with real timing, and return clip facts. */
|
||||
async stop() {
|
||||
recording = false;
|
||||
await cdp.send('Page.stopScreencast').catch(() => {});
|
||||
await cdp.detach().catch(() => {});
|
||||
if (frames.length < 2) return { clip: null, frames: frames.length };
|
||||
const lines = [];
|
||||
for (let i = 0; i < frames.length; i++) {
|
||||
const next = frames[i + 1]?.timestamp;
|
||||
const duration = next
|
||||
? Math.min(2, Math.max(0.001, next - frames[i].timestamp))
|
||||
: 0.05;
|
||||
lines.push(`file '${path.resolve(frames[i].file)}'`);
|
||||
lines.push(`duration ${duration.toFixed(4)}`);
|
||||
}
|
||||
// The concat demuxer ignores the last duration unless the file repeats.
|
||||
lines.push(`file '${path.resolve(frames.at(-1).file)}'`);
|
||||
const list = path.join(dir, 'frames.txt');
|
||||
fs.writeFileSync(list, `${lines.join('\n')}\n`);
|
||||
const clip = path.join(dir, 'clip.mp4');
|
||||
execFileSync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-f',
|
||||
'concat',
|
||||
'-safe',
|
||||
'0',
|
||||
'-i',
|
||||
list,
|
||||
'-fps_mode',
|
||||
fps ? 'cfr' : 'vfr',
|
||||
'-c:v',
|
||||
'libx264',
|
||||
'-pix_fmt',
|
||||
'yuv420p',
|
||||
'-vf',
|
||||
`scale=trunc(iw/2)*2:trunc(ih/2)*2${fps ? `,fps=${fps}` : ''}`,
|
||||
clip,
|
||||
]);
|
||||
const seconds = frames.at(-1).timestamp - frames[0].timestamp;
|
||||
return {
|
||||
clip,
|
||||
frames: frames.length,
|
||||
seconds: Math.round(seconds * 10) / 10,
|
||||
fps: Math.round((frames.length / Math.max(seconds, 0.001)) * 10) / 10,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
+10
-10
@@ -738,9 +738,9 @@ check({
|
||||
});
|
||||
|
||||
check({
|
||||
id: 'B2', group: 'B', desc: 'Flights proxy returns live contacts (/api/opensky)',
|
||||
id: 'B2', group: 'B', desc: 'Flights proxy returns live contacts (/api/flights)',
|
||||
run: async () => {
|
||||
const r = await jget('/api/opensky?lamin=24&lomin=-125&lamax=50&lomax=-66', { timeoutMs: 45000 });
|
||||
const r = await jget('/api/flights?lamin=24&lomin=-125&lamax=50&lomax=-66', { timeoutMs: 45000 });
|
||||
if (!r.ok) return fail(`HTTP ${r.status}: ${r.text.slice(0, 120)}`);
|
||||
const n = r.json?.states?.length || 0;
|
||||
return n > 0 ? pass(`${n} states, cache=${r.headers.get('x-opensky-cache') || 'n/a'}`) : fail('0 states returned');
|
||||
@@ -750,7 +750,7 @@ check({
|
||||
check({
|
||||
id: 'B3', group: 'B', desc: 'OpenSky credentials are actually in use (not the anonymous/fallback path)',
|
||||
run: async () => {
|
||||
const r = await jget('/api/opensky?lamin=24&lomin=-125&lamax=50&lomax=-66', { timeoutMs: 45000 });
|
||||
const r = await jget('/api/flights?lamin=24&lomin=-125&lamax=50&lomax=-66', { timeoutMs: 45000 });
|
||||
// Header names are exact: X-OpenSky-Auth-Mode-Used / X-OpenSky-Auth-Reason.
|
||||
// (An earlier guess at these names made this check pass vacuously.)
|
||||
const reason = r.headers.get('X-OpenSky-Auth-Reason') || '';
|
||||
@@ -826,9 +826,9 @@ check({
|
||||
});
|
||||
|
||||
check({
|
||||
id: 'B8', group: 'B', desc: 'AIS vessel feed is live (/api/ais-live)', needsKey: 'AIS',
|
||||
id: 'B8', group: 'B', desc: 'AIS vessel feed is live (/api/vessels)', needsKey: 'AIS',
|
||||
run: async () => {
|
||||
const r = await jget('/api/ais-live', { timeoutMs: 40000 });
|
||||
const r = await jget('/api/vessels', { timeoutMs: 40000 });
|
||||
const rows = r.json?.rows?.length || 0;
|
||||
const status = r.json?.status;
|
||||
if (!r.ok) return fail(`HTTP ${r.status} status=${status}`);
|
||||
@@ -864,7 +864,7 @@ check({
|
||||
const guard = keyGuard('AIS', env.keys.AIS);
|
||||
if (guard) return guard;
|
||||
if (env.keys.AIS === true) return skip('server HAS an AISStream key', 'N/A');
|
||||
const r = await jget('/api/ais-live');
|
||||
const r = await jget('/api/vessels');
|
||||
return r.status === 503 && r.json?.status === 'missing-key' && Array.isArray(r.json?.rows)
|
||||
? pass(`503 status=missing-key, rows=[] — "${String(r.json?.error).slice(0, 60)}"`)
|
||||
: fail(`expected 503/missing-key, got ${r.status} ${r.text.slice(0, 120)}`);
|
||||
@@ -1022,7 +1022,7 @@ check({
|
||||
check({
|
||||
id: 'B21', group: 'B', desc: 'No proxy echoes credential material back to the client (P1-5 acceptance #4)',
|
||||
run: async () => {
|
||||
const paths = ['/api/cctv/sources', '/api/tomtom/status', '/api/firms/status', '/api/celestrak/stations', '/api/ais-live'];
|
||||
const paths = ['/api/cctv/sources', '/api/tomtom/status', '/api/firms/status', '/api/celestrak/stations', '/api/vessels'];
|
||||
const leaked = [];
|
||||
const unscannable = [];
|
||||
for (const p of paths) {
|
||||
@@ -1036,7 +1036,7 @@ check({
|
||||
// An error page is not a payload. Scanning five 500s and finding no key
|
||||
// is trivially true and proves nothing — a broken app must not satisfy a
|
||||
// negative assertion. The one documented exception is the keyless
|
||||
// 503 {status:'missing-key'} from /api/ais-live, which IS its real shape.
|
||||
// 503 {status:'missing-key'} from /api/vessels, which IS its real shape.
|
||||
const documentedKeyless = r.status === 503
|
||||
&& (r.json?.status === 'missing-key' || r.json?.error === 'no_key' || /OPENAI_API_KEY is not set/.test(r.text));
|
||||
if (!r.ok && !documentedKeyless) {
|
||||
@@ -2071,13 +2071,13 @@ async function preflight() {
|
||||
env.keys.FIRMS = await statusKey('/api/firms/status');
|
||||
env.keys.TOMTOM = await statusKey('/api/tomtom/status');
|
||||
try {
|
||||
const ais = await jget('/api/ais-live');
|
||||
const ais = await jget('/api/vessels');
|
||||
if (ais.status === 503 && ais.json?.status === 'missing-key') env.keys.AIS = false;
|
||||
else if (ais.status === 200 && ais.json && Array.isArray(ais.json.rows)) env.keys.AIS = true;
|
||||
else env.keys.AIS = 'error';
|
||||
} catch { env.keys.AIS = 'error'; }
|
||||
try {
|
||||
const os = await jget('/api/opensky?lamin=29&lomin=-99&lamax=31&lomax=-97', { timeoutMs: 40000 });
|
||||
const os = await jget('/api/flights?lamin=29&lomin=-99&lamax=31&lomax=-97', { timeoutMs: 40000 });
|
||||
const reason = os.headers.get('X-OpenSky-Auth-Reason') || '';
|
||||
const used = os.headers.get('X-OpenSky-Auth-Mode-Used') || os.headers.get('X-OpenSky-Auth') || '';
|
||||
if (!os.ok) env.keys.OPENSKY = 'error';
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env node
|
||||
/** Named military areas: continent performance, close identities and a recorded zoom. */
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import puppeteer from 'puppeteer';
|
||||
import {
|
||||
checkInstallationPan,
|
||||
checkInstallationSelection,
|
||||
} from './qa-installation-polish.mjs';
|
||||
import {
|
||||
createScreencast,
|
||||
installFrameProbe,
|
||||
readFrames,
|
||||
frameStats,
|
||||
pageNow,
|
||||
moveCamera,
|
||||
} from './qa-journey-recorder.mjs';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help')) {
|
||||
console.log('Usage: node scripts/qa-military-names.mjs <url> [--no-video]');
|
||||
process.exit(0);
|
||||
}
|
||||
const url = args.find((a) => /^https?:/.test(a));
|
||||
if (!url) throw new Error('Supply the running server URL');
|
||||
const dir = process.env.QA_NAMES_DIR || 'qa-shots/military-names';
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const result = {
|
||||
views: [],
|
||||
sites: [],
|
||||
errors: [],
|
||||
forbidden: [],
|
||||
packs: [],
|
||||
video: null,
|
||||
};
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
protocolTimeout: 180000,
|
||||
args: [
|
||||
'--use-angle=metal',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
],
|
||||
});
|
||||
let recorder;
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
page.on('pageerror', (e) => result.errors.push(e.message));
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') result.errors.push(m.text());
|
||||
});
|
||||
page.on('request', (request) => {
|
||||
const u = new URL(request.url());
|
||||
if (
|
||||
/overpass/i.test(u.hostname) ||
|
||||
u.hostname === 'nominatim.openstreetmap.org'
|
||||
)
|
||||
result.forbidden.push(u.origin);
|
||||
if (u.pathname.endsWith('/osm_military_names/names.json'))
|
||||
result.packs.push(u.pathname);
|
||||
});
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
await page.goto(url + (url.includes('?') ? '&' : '?') + 'welcome=0', {
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
result.page = await page.evaluate(() => ({
|
||||
url: location.href,
|
||||
title: document.title,
|
||||
canvas: Boolean(document.querySelector('#cesiumContainer canvas')),
|
||||
errorOverlay: Boolean(document.querySelector('vite-error-overlay')),
|
||||
}));
|
||||
assert.equal(new URL(result.page.url).origin, new URL(url).origin);
|
||||
assert.ok(
|
||||
result.page.title && result.page.canvas && !result.page.errorOverlay,
|
||||
);
|
||||
await page.evaluate(async () => {
|
||||
const g = window.__godsEyeView;
|
||||
for (const [id] of g.dataManager.layers)
|
||||
await g.dataManager.setEnabled(id, false);
|
||||
g.styleManager.setDetection({ enabled: false });
|
||||
g.viewer.scene.screenSpaceCameraController.enableCollisionDetection = false;
|
||||
});
|
||||
assert.equal(result.packs.length, 0, 'names pack stays unloaded at startup');
|
||||
result.renderer = await page.evaluate(() => {
|
||||
const gl = window.__godsEyeView.viewer.scene.context._gl;
|
||||
const ext = gl.getExtension('WEBGL_debug_renderer_info');
|
||||
return ext
|
||||
? gl.getParameter(ext.UNMASKED_RENDERER_WEBGL)
|
||||
: gl.getParameter(gl.RENDERER);
|
||||
});
|
||||
assert.doesNotMatch(result.renderer, /swiftshader|software/i);
|
||||
await installFrameProbe(page);
|
||||
const stats = () =>
|
||||
page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats(),
|
||||
);
|
||||
const settle = async () => {
|
||||
await sleep(500);
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const s = window.__godsEyeView.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.getStats();
|
||||
return !s.loading && s.count > 0 && s.status === 'ready';
|
||||
},
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
await sleep(1200);
|
||||
};
|
||||
const us = { lon: -100, lat: 39, height: 6500000, pitch: -90 };
|
||||
await moveCamera(page, us, 0);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('military-installations', true),
|
||||
);
|
||||
await settle();
|
||||
result.credit = await page.evaluate(() => {
|
||||
const el = document.querySelector('#cesium-credits');
|
||||
return {
|
||||
text: el?.innerText || '',
|
||||
visible: Boolean(el && getComputedStyle(el).display !== 'none'),
|
||||
};
|
||||
});
|
||||
assert.match(result.credit.text, /© OpenStreetMap/);
|
||||
assert.doesNotMatch(
|
||||
result.credit.text,
|
||||
/© OpenMapTiles/,
|
||||
'wide bundled names use no vector tiles',
|
||||
);
|
||||
assert.ok(result.credit.visible);
|
||||
await page.screenshot({ path: `${dir}/credit.png` });
|
||||
for (const [name, view] of [
|
||||
['us', us],
|
||||
['europe', { lon: 12, lat: 50, height: 4500000, pitch: -90 }],
|
||||
]) {
|
||||
await moveCamera(page, view, 2);
|
||||
await settle();
|
||||
const before = await stats();
|
||||
const overlaps = await page.evaluate(async () => {
|
||||
const { getOverlayPaintRect } =
|
||||
await import('/src/overlays/worldOverlay.js');
|
||||
const layer = window.__godsEyeView.dataManager.layers.get(
|
||||
'military-installations',
|
||||
).module;
|
||||
const rects = [];
|
||||
layer.visitNamedMarkers((id) => {
|
||||
const rect = getOverlayPaintRect('military-installations', id);
|
||||
if (rect) rects.push(rect);
|
||||
});
|
||||
let overlaps = 0;
|
||||
for (let a = 0; a < rects.length; a++)
|
||||
for (let b = a + 1; b < rects.length; b++) {
|
||||
const x = rects[a],
|
||||
y = rects[b];
|
||||
if (
|
||||
x.x < y.x + y.w &&
|
||||
y.x < x.x + x.w &&
|
||||
x.y < y.y + y.h &&
|
||||
y.y < x.y + x.h
|
||||
)
|
||||
overlaps++;
|
||||
}
|
||||
return overlaps;
|
||||
});
|
||||
// Incumbents retain their placements during camera motion, as for Data Centers.
|
||||
assert.ok(
|
||||
before.namedInView > 0 &&
|
||||
before.namedMarkers > 0 &&
|
||||
before.pointsOnScreen > 0,
|
||||
);
|
||||
assert.ok(
|
||||
before.labelsOnScreen > 0 && before.labelsOnScreen <= before.labelCap,
|
||||
);
|
||||
await page.evaluate(async () => {
|
||||
// Use the governor's hold so an asynchronous idle transition cannot
|
||||
// overwrite a raw scene flag during the sustained-render measurement.
|
||||
const { holdContinuousRender } = await import('/src/renderGovernor.js');
|
||||
holdContinuousRender('qa-military-names-fps');
|
||||
});
|
||||
const t0 = await pageNow(page);
|
||||
await sleep(3500);
|
||||
const frames = await readFrames(page, t0);
|
||||
await page.evaluate(async () => {
|
||||
const { releaseContinuousRender } =
|
||||
await import('/src/renderGovernor.js');
|
||||
releaseContinuousRender('qa-military-names-fps');
|
||||
});
|
||||
const metrics = frameStats(frames);
|
||||
const seconds = frames.reduce((sum, frame) => sum + frame.dt, 0) / 1000;
|
||||
const fps = metrics.renderedFrames / seconds;
|
||||
const rafFps = metrics.frames / seconds;
|
||||
result.views.push({ name, ...before, overlaps, fps, rafFps, ...metrics });
|
||||
console.log(name, JSON.stringify(result.views.at(-1)));
|
||||
await page.screenshot({ path: `${dir}/${name}.png` });
|
||||
assert.ok(fps >= 55, `${name} fps ${fps.toFixed(1)} < 55`);
|
||||
}
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.setCleanView(true),
|
||||
);
|
||||
await sleep(300);
|
||||
result.cleanUi = await page.evaluate(async () => {
|
||||
const host = await import('/src/overlays/worldOverlay.js');
|
||||
const root = document.getElementById('world-overlay-root');
|
||||
return {
|
||||
active: document.body.classList.contains('ui-clean-view'),
|
||||
visible: root.checkVisibility({
|
||||
visibilityProperty: true,
|
||||
opacityProperty: true,
|
||||
}),
|
||||
labels:
|
||||
host.getWorldOverlayDiagnostics().paintedBySource[
|
||||
'military-installations'
|
||||
] || 0,
|
||||
};
|
||||
});
|
||||
assert.ok(
|
||||
result.cleanUi.active &&
|
||||
result.cleanUi.visible &&
|
||||
result.cleanUi.labels > 0,
|
||||
);
|
||||
await page.screenshot({ path: `${dir}/clean-ui.png` });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.styleManager.setCleanView(false),
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled(
|
||||
'military-installations',
|
||||
false,
|
||||
),
|
||||
);
|
||||
result.disabled = await page.evaluate(async () => {
|
||||
const host = await import('/src/overlays/worldOverlay.js');
|
||||
return host.getWorldOverlayDiagnostics().entriesBySource[
|
||||
'military-installations'
|
||||
];
|
||||
});
|
||||
assert.equal(result.disabled, 0);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('military-installations', true),
|
||||
);
|
||||
await moveCamera(page, us, 1);
|
||||
await settle();
|
||||
result.slowPan = await checkInstallationPan(page, dir);
|
||||
await moveCamera(page, us, 1);
|
||||
await settle();
|
||||
if (!args.includes('--no-video')) {
|
||||
recorder = await createScreencast(page, `${dir}/us-to-san-diego`, {
|
||||
fps: 24,
|
||||
});
|
||||
await recorder.start();
|
||||
}
|
||||
await page.evaluate(async () => {
|
||||
const C = await import('/node_modules/cesium/Build/Cesium/index.js');
|
||||
const { getOverlayPaintRect } =
|
||||
await import('/src/overlays/worldOverlay.js');
|
||||
const g = window.__godsEyeView;
|
||||
const layer = g.dataManager.layers.get('military-installations').module;
|
||||
const overlay = document.createElement('div');
|
||||
overlay.id = 'installation-label-probe';
|
||||
overlay.style.cssText =
|
||||
'position:fixed;top:60px;left:360px;z-index:99999;background:#07121cdd;color:#fff;padding:8px 12px;font:13px monospace;pointer-events:none';
|
||||
document.body.append(overlay);
|
||||
const screen = new C.Cartesian2();
|
||||
window.__installationFrames = [];
|
||||
window.__stopInstallationFrames =
|
||||
g.viewer.scene.postRender.addEventListener(() => {
|
||||
const stats = layer.getStats();
|
||||
const markers = [],
|
||||
labels = [],
|
||||
rects = [];
|
||||
layer.visitNamedMarkers((id, point) => {
|
||||
const rect = getOverlayPaintRect('military-installations', id);
|
||||
if (rect) rects.push(rect);
|
||||
const p = C.SceneTransforms.worldToWindowCoordinates(
|
||||
g.viewer.scene,
|
||||
point.position,
|
||||
screen,
|
||||
);
|
||||
if (
|
||||
point.show &&
|
||||
p &&
|
||||
p.x >= 0 &&
|
||||
p.y >= 0 &&
|
||||
p.x < 1280 &&
|
||||
p.y < 800
|
||||
) {
|
||||
markers.push(id);
|
||||
if (rect) labels.push(id);
|
||||
}
|
||||
});
|
||||
let overlaps = 0;
|
||||
for (let a = 0; a < rects.length; a++)
|
||||
for (let b = a + 1; b < rects.length; b++) {
|
||||
const x = rects[a],
|
||||
y = rects[b];
|
||||
if (
|
||||
x.x < y.x + y.w &&
|
||||
y.x < x.x + x.w &&
|
||||
x.y < y.y + y.h &&
|
||||
y.y < x.y + x.h
|
||||
)
|
||||
overlaps++;
|
||||
}
|
||||
const altitude = g.viewer.camera.positionCartographic.height;
|
||||
window.__installationFrames.push({
|
||||
t: performance.now(),
|
||||
altitude,
|
||||
markers,
|
||||
labels,
|
||||
paintedLabels: stats.labelsOnScreen,
|
||||
overlaps,
|
||||
loading: stats.loading,
|
||||
wide: stats.wide,
|
||||
});
|
||||
overlay.textContent = `${(altitude / 1000).toFixed(1)} km | markers ${markers.length} | labels ${labels.length} / 24`;
|
||||
});
|
||||
});
|
||||
const clipStart = await pageNow(page);
|
||||
await sleep(1200);
|
||||
for (const view of [
|
||||
{ lon: -116, lat: 35, height: 1500000, pitch: -85 },
|
||||
{ lon: -117.2, lat: 33, height: 200000, pitch: -80 },
|
||||
{ lon: -117.1, lat: 32.86, height: 18000, pitch: -80 },
|
||||
]) {
|
||||
await moveCamera(page, view, 3);
|
||||
await sleep(800);
|
||||
}
|
||||
await settle();
|
||||
await sleep(1500);
|
||||
result.clipFrames = await readFrames(page, clipStart);
|
||||
result.labelFrames = await page.evaluate(() => {
|
||||
window.__stopInstallationFrames();
|
||||
document.getElementById('installation-label-probe').remove();
|
||||
return window.__installationFrames;
|
||||
});
|
||||
await fs.writeFile(
|
||||
`${dir}/label-frames.json`,
|
||||
JSON.stringify(result.labelFrames, null, 2),
|
||||
);
|
||||
assert.ok(result.labelFrames.length > 100);
|
||||
assert.ok(
|
||||
result.labelFrames.every(
|
||||
(frame) =>
|
||||
frame.markers.length > 0 &&
|
||||
frame.labels.length > 0 &&
|
||||
frame.paintedLabels > 0 &&
|
||||
frame.paintedLabels <= 24,
|
||||
),
|
||||
'zoom never loses all visible named markers or labels',
|
||||
);
|
||||
if (recorder) {
|
||||
result.video = await recorder.stop();
|
||||
recorder = null;
|
||||
}
|
||||
result.selection = await checkInstallationSelection(page, dir, {
|
||||
video: !args.includes('--no-video'),
|
||||
});
|
||||
const sites = [
|
||||
['mabry', 'Camp Mabry', -97.765, 30.3125, 3500],
|
||||
['pendleton', 'Marine Corps Base Camp Pendleton', -117.4, 33.33, 25000],
|
||||
['miramar', 'Marine Corps Air Station Miramar', -117.14, 32.87, 18000],
|
||||
['ramstein', 'Ramstein Air Base', 7.6, 49.44, 7000],
|
||||
['lakenheath', 'RAF Lakenheath', 0.56, 52.41, 7000],
|
||||
['hood', 'Fort Hood', -97.78, 31.135, 30000],
|
||||
];
|
||||
for (const [key, expected, lon, lat, height] of sites) {
|
||||
await moveCamera(page, { lon, lat, height, pitch: -80 }, 1.5);
|
||||
await settle();
|
||||
const snapshot = await page.evaluate(
|
||||
(lat, lon) => {
|
||||
const g = window.__godsEyeView;
|
||||
const layer = g.dataManager.layers.get('military-installations').module;
|
||||
const position = g.viewer.scene.globe.ellipsoid.cartographicToCartesian(
|
||||
{
|
||||
longitude: (lon * Math.PI) / 180,
|
||||
latitude: (lat * Math.PI) / 180,
|
||||
height: 0,
|
||||
},
|
||||
);
|
||||
return {
|
||||
stats: layer.getStats(),
|
||||
records: layer.getNearby(position, 100000, 1000).map((r) => ({
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
memberNames: r.memberNames || [],
|
||||
hasPolygon: Boolean(r.footprints?.length),
|
||||
})),
|
||||
};
|
||||
},
|
||||
lat,
|
||||
lon,
|
||||
);
|
||||
result.sites.push({ key, expected, ...snapshot });
|
||||
assert.equal(
|
||||
new Set(snapshot.records.map((r) => r.id)).size,
|
||||
snapshot.records.length,
|
||||
);
|
||||
assert.ok(
|
||||
snapshot.stats.labelsOnScreen > 0 &&
|
||||
snapshot.stats.labelsOnScreen <= snapshot.stats.labelCap,
|
||||
);
|
||||
assert.ok(
|
||||
snapshot.records.some((r) => r.name === expected && r.hasPolygon),
|
||||
`${key}: missing named polygon ${expected}; got ${snapshot.records.map((r) => r.name).join(', ')}`,
|
||||
);
|
||||
await page.screenshot({ path: `${dir}/${key}.png` });
|
||||
console.log(key, expected);
|
||||
}
|
||||
await page.evaluate(() => {
|
||||
if (
|
||||
document
|
||||
.getElementById('global-context-panel')
|
||||
.classList.contains('collapsed')
|
||||
)
|
||||
document
|
||||
.querySelector('[data-collapse-target="global-context-panel"]')
|
||||
.click();
|
||||
});
|
||||
await page.waitForSelector('#global-context-flights-btn', { visible: true });
|
||||
await page.evaluate(() => {
|
||||
const button = document.getElementById('global-context-flights-btn');
|
||||
if (button.getAttribute('aria-selected') !== 'true') button.click();
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() => {
|
||||
const state = window.__godsEyeView.styleManager.getContextModeState();
|
||||
return state.mode === 'flights' && !state.changing;
|
||||
},
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
result.card = await page.evaluate(() => {
|
||||
const g = window.__godsEyeView;
|
||||
return g.dataManager.layers
|
||||
.get('military-installations')
|
||||
.module.focusById('osm:military:r13529728');
|
||||
});
|
||||
assert.equal(result.card, true);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document
|
||||
.getElementById('military-awareness-panel')
|
||||
?.textContent.includes('Named areas') &&
|
||||
document
|
||||
.getElementById('military-awareness-panel')
|
||||
.checkVisibility({ visibilityProperty: true, opacityProperty: true }),
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
result.cardText = await page.$eval(
|
||||
'#military-awareness-panel',
|
||||
(el) => el.textContent,
|
||||
);
|
||||
assert.match(result.cardText, /Fort Hood/);
|
||||
assert.match(result.cardText, /TA-25/);
|
||||
await sleep(1800);
|
||||
await page.$eval(
|
||||
'#military-awareness-panel .military-awareness-names',
|
||||
(el) => {
|
||||
el.open = true;
|
||||
},
|
||||
);
|
||||
await page.screenshot({ path: `${dir}/member-names-card.png` });
|
||||
assert.equal(result.packs.length, 1, 'single lazy asset fetch');
|
||||
assert.deepEqual(result.forbidden, []);
|
||||
assert.deepEqual(result.errors, []);
|
||||
} finally {
|
||||
if (recorder)
|
||||
result.video = await recorder.stop().catch((e) => ({ error: e.message }));
|
||||
await fs.writeFile(`${dir}/result.json`, JSON.stringify(result, null, 2));
|
||||
await browser.close();
|
||||
}
|
||||
@@ -180,7 +180,7 @@ async function installDeterministicDevEndpoints(page) {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.pathname === '/api/ais-live') {
|
||||
if (url.pathname === '/api/vessels') {
|
||||
await request.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,481 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Browser QA for the floating CCTV panel: lift, resize from every handle,
|
||||
* minimum size, reload, and dock again. `--fail-on-retry` (or
|
||||
* QA_FAIL_ON_RETRY=1) fails on any header-press retry.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { RESIZE_DIRECTIONS, resizeBox } from '../src/ui/panelResize.js';
|
||||
import {
|
||||
describePress,
|
||||
dockPanelByDoubleClick,
|
||||
dragPointer,
|
||||
liftPanelByHeader,
|
||||
nextFrames,
|
||||
} from './qa-panelDrag.mjs';
|
||||
import {
|
||||
hookRenderErrors,
|
||||
readRenderErrors,
|
||||
saveFailureArtifacts,
|
||||
watchPage,
|
||||
} from './qa-browserEvidence.mjs';
|
||||
|
||||
export const PANEL_ID = 'cctv-panel';
|
||||
export const STORAGE_KEY = `godsEyeView.v8.panelPos.${PANEL_ID}`;
|
||||
export const MIN_SIZE = { width: 300, height: 160 };
|
||||
export const ARTIFACT_DIR = 'qa-artifacts/panel-resize';
|
||||
const VIEWPORT = { width: 1400, height: 900 };
|
||||
const EDGE_BY_SIDE = { n: 'top', s: 'bottom', e: 'right', w: 'left' };
|
||||
|
||||
/** Box edges a resize in `dir` must leave exactly where they were. */
|
||||
export function pinnedEdges(dir) {
|
||||
return Object.entries(EDGE_BY_SIDE)
|
||||
.filter(([side]) => !dir.includes(side))
|
||||
.map(([, edge]) => edge);
|
||||
}
|
||||
|
||||
/** Pointer travel that grows a panel by `amount` on every edge named in `dir`. */
|
||||
export function growthDelta(dir, amount) {
|
||||
return {
|
||||
dx: dir.includes('e') ? amount : dir.includes('w') ? -amount : 0,
|
||||
dy: dir.includes('s') ? amount : dir.includes('n') ? -amount : 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Viewport point inside the handle for `dir`, given the panel's client rect.
|
||||
* Edge strips straddle the border by 3px; corners and the grip sit inside.
|
||||
*/
|
||||
export function handlePoint(rect, dir) {
|
||||
const inset = dir.length === 2 ? 4 : 1;
|
||||
const x = dir.includes('w')
|
||||
? rect.left + inset
|
||||
: dir.includes('e')
|
||||
? rect.right - inset
|
||||
: rect.left + rect.width / 2;
|
||||
const y = dir.includes('n')
|
||||
? rect.top + inset
|
||||
: dir.includes('s')
|
||||
? rect.bottom - inset
|
||||
: rect.top + rect.height / 2;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
/** Pinned edges that moved during a resize in `dir`. */
|
||||
export function driftedEdges(before, after, dir, tolerance = 1.5) {
|
||||
const edges = (box) => ({
|
||||
left: box.left,
|
||||
top: box.top,
|
||||
right: box.left + box.width,
|
||||
bottom: box.top + box.height,
|
||||
});
|
||||
const a = edges(before);
|
||||
const b = edges(after);
|
||||
return pinnedEdges(dir).filter(
|
||||
(edge) => Math.abs(a[edge] - b[edge]) > tolerance,
|
||||
);
|
||||
}
|
||||
|
||||
function near(actual, expected, tolerance, label) {
|
||||
assert.ok(
|
||||
Math.abs(actual - expected) <= tolerance,
|
||||
`${label}: expected ${expected} ± ${tolerance}, got ${actual}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { default: puppeteer } = await import('puppeteer');
|
||||
const args = process.argv.slice(2);
|
||||
const urlIndex = args.indexOf('--url');
|
||||
const url = urlIndex >= 0 ? args[urlIndex + 1] : 'http://localhost:4173';
|
||||
if (args.includes('--fail-on-retry')) process.env.QA_FAIL_ON_RETRY = '1';
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath:
|
||||
process.env.PUPPETEER_EXECUTABLE_PATH ||
|
||||
(await puppeteer.executablePath()),
|
||||
args: ['--no-sandbox', '--use-gl=angle', '--use-angle=swiftshader'],
|
||||
});
|
||||
const monitors = [];
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport(VIEWPORT);
|
||||
const errors = [];
|
||||
monitors.push(watchPage(page, { name: 'panel', errors }));
|
||||
await hookRenderErrors(page);
|
||||
|
||||
const boot = async () => {
|
||||
await page.goto(`${url}/`, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(() => window.__godsEyeView?.viewer, {
|
||||
timeout: 90_000,
|
||||
});
|
||||
await page.waitForSelector(`#${PANEL_ID} .panel-header`);
|
||||
await page.evaluate(() => {
|
||||
document.querySelector('.first-run-explore')?.click();
|
||||
});
|
||||
await page.keyboard.press('Escape');
|
||||
await page.evaluate(() => {
|
||||
document.getElementById('first-run-launcher')?.remove();
|
||||
for (const node of document.querySelectorAll('[class*=first-run]'))
|
||||
node.remove();
|
||||
});
|
||||
await page.mouse.click(VIEWPORT.width / 2, VIEWPORT.height / 2);
|
||||
await nextFrames(page);
|
||||
};
|
||||
const readPanel = () =>
|
||||
page.evaluate((id) => {
|
||||
const panel = document.getElementById(id);
|
||||
const rect = panel.getBoundingClientRect();
|
||||
return {
|
||||
left: rect.left,
|
||||
top: rect.top,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
// handlePoint() reads the far edges for s/e handles.
|
||||
right: rect.right,
|
||||
bottom: rect.bottom,
|
||||
floating: panel.classList.contains('panel-floating'),
|
||||
collapsed: panel.classList.contains('collapsed'),
|
||||
inRail: panel.parentElement?.id === 'right-context-rail',
|
||||
allocated: panel.style.getPropertyValue(
|
||||
'--right-panel-allocated-height',
|
||||
),
|
||||
inlineWidth: panel.style.width,
|
||||
inlineHeight: panel.style.height,
|
||||
stored: localStorage.getItem(`godsEyeView.v8.panelPos.${id}`),
|
||||
};
|
||||
}, PANEL_ID);
|
||||
/**
|
||||
* Best effort: wait for the box to hold still for half a second. The lift
|
||||
* is checked against the box at pointerdown, so a timeout only costs time.
|
||||
*/
|
||||
const settle = async () => {
|
||||
try {
|
||||
await page.waitForFunction(
|
||||
(id) => {
|
||||
const r = document.getElementById(id).getBoundingClientRect();
|
||||
const box = [r.left, r.top, r.width, r.height]
|
||||
.map(Math.round)
|
||||
.join(',');
|
||||
const now = performance.now();
|
||||
if (window.__qaLastBox !== box) {
|
||||
window.__qaLastBox = box;
|
||||
window.__qaSince = now;
|
||||
}
|
||||
return now - window.__qaSince >= 500;
|
||||
},
|
||||
{ polling: 100, timeout: 20_000 },
|
||||
PANEL_ID,
|
||||
);
|
||||
} catch (error) {
|
||||
if (error?.name !== 'TimeoutError') throw error;
|
||||
console.log('note: the CCTV panel was still moving; lifting anyway');
|
||||
}
|
||||
};
|
||||
const headerPoint = async () => {
|
||||
const rect = await page.evaluate((id) => {
|
||||
const title = document.querySelector(
|
||||
`#${id} .panel-header .panel-title`,
|
||||
);
|
||||
const r = title.getBoundingClientRect();
|
||||
return { x: r.left + r.width / 2, y: r.top + r.height / 2 };
|
||||
}, PANEL_ID);
|
||||
return rect;
|
||||
};
|
||||
const drag = (from, dx, dy) => dragPointer(page, from, dx, dy);
|
||||
|
||||
await page.goto(`${url}/`, { waitUntil: 'domcontentloaded' });
|
||||
await page.evaluate((key) => {
|
||||
localStorage.removeItem(key);
|
||||
localStorage.removeItem('godsEyeView.v8.panelFloatHintShown');
|
||||
}, STORAGE_KEY);
|
||||
await boot();
|
||||
|
||||
await page.evaluate((id) => {
|
||||
const panel = document.getElementById(id);
|
||||
if (panel.classList.contains('collapsed'))
|
||||
document
|
||||
.querySelector(`.panel-collapse-btn[data-collapse-target="${id}"]`)
|
||||
.click();
|
||||
}, PANEL_ID);
|
||||
await page.waitForFunction(
|
||||
(id) => !document.getElementById(id).classList.contains('collapsed'),
|
||||
{},
|
||||
PANEL_ID,
|
||||
);
|
||||
// The rail animates panel heights, and on a slow software renderer the
|
||||
// panels above CCTV can still be moving after a fixed sleep.
|
||||
await settle();
|
||||
const docked = await readPanel();
|
||||
assert.equal(docked.floating, false);
|
||||
assert.equal(docked.inRail, true);
|
||||
|
||||
// Names whatever above CCTV in the rail shifted it, for a retry note.
|
||||
const railAbove = () =>
|
||||
page.evaluate((id) => {
|
||||
const out = [];
|
||||
for (const node of document.getElementById('right-context-rail')
|
||||
?.children || []) {
|
||||
if (node.id === id) break;
|
||||
out.push(
|
||||
`${node.id || node.className}:${Math.round(node.offsetHeight)}`,
|
||||
);
|
||||
}
|
||||
return out.join(' ');
|
||||
}, PANEL_ID);
|
||||
const aboveAtDock = await railAbove();
|
||||
// On CI a panel above CCTV grows after load and can shift the header out
|
||||
// from under the press; the helper retries only such a provable miss.
|
||||
const attempt = await liftPanelByHeader(page, PANEL_ID, {
|
||||
dx: -200,
|
||||
dy: -100,
|
||||
note: async () =>
|
||||
`rail above CCTV ${aboveAtDock} -> ${await railAbove()}`,
|
||||
});
|
||||
const { pressed } = attempt;
|
||||
const lifted = await readPanel();
|
||||
assert.equal(
|
||||
lifted.floating,
|
||||
true,
|
||||
`a header drag lifts the panel out (${describePress(attempt)}; panel docked at ${docked.top}, at press ${pressed?.top})`,
|
||||
);
|
||||
assert.equal(
|
||||
lifted.collapsed,
|
||||
false,
|
||||
'the drag-ending click must not collapse',
|
||||
);
|
||||
const moved = `docked at ${docked.left},${docked.top}; pressed at ${pressed.left},${pressed.top}`;
|
||||
near(
|
||||
lifted.left,
|
||||
Math.max(6, pressed.left - 200),
|
||||
2,
|
||||
`lifted left (${moved})`,
|
||||
);
|
||||
near(
|
||||
lifted.top,
|
||||
Math.max(6, pressed.top - 100),
|
||||
2,
|
||||
`lifted top (${moved})`,
|
||||
);
|
||||
near(lifted.width, pressed.width, 2, 'lifted width');
|
||||
// The rail lets go of the lifted panel on its next layout pass.
|
||||
await page
|
||||
.waitForFunction(
|
||||
(id) =>
|
||||
document
|
||||
.getElementById(id)
|
||||
.style.getPropertyValue('--right-panel-allocated-height') === '',
|
||||
{ timeout: 5_000 },
|
||||
PANEL_ID,
|
||||
)
|
||||
.catch(() => {});
|
||||
const afterLayout = await readPanel();
|
||||
assert.equal(afterLayout.allocated, '', 'the rail no longer allocates it');
|
||||
const railExcludes = await page.evaluate((id) => {
|
||||
const rail = document.getElementById('right-context-rail');
|
||||
// Same rule as the rail: hidden panels take no room.
|
||||
const counted = [...rail.children].filter((panel) =>
|
||||
panel.matches(
|
||||
'[data-panel-id]:not(.panel-floating):not(.collapsed):not([hidden])',
|
||||
),
|
||||
);
|
||||
return {
|
||||
expandedCount: rail.dataset.expandedCount,
|
||||
counted: counted.length,
|
||||
includesPanel: counted.some((panel) => panel.id === id),
|
||||
};
|
||||
}, PANEL_ID);
|
||||
assert.equal(railExcludes.includesPanel, false);
|
||||
if (railExcludes.expandedCount !== undefined)
|
||||
assert.equal(Number(railExcludes.expandedCount), railExcludes.counted);
|
||||
const storedAfterLift = JSON.parse(afterLayout.stored);
|
||||
assert.equal(storedAfterLift.floating, true);
|
||||
assert.equal(
|
||||
'height' in storedAfterLift,
|
||||
false,
|
||||
'a drag alone must not freeze the measured height',
|
||||
);
|
||||
console.log('PASS: header drag lifts CCTV out of the rail');
|
||||
|
||||
// Park near the top: the voice dock would cover the south corners once
|
||||
// the loop below has grown the window.
|
||||
{
|
||||
const header = await headerPoint();
|
||||
await drag(header, 0, 120 - header.y);
|
||||
}
|
||||
|
||||
const limits = {
|
||||
minWidth: MIN_SIZE.width,
|
||||
minHeight: MIN_SIZE.height,
|
||||
viewportWidth: VIEWPORT.width,
|
||||
viewportHeight: VIEWPORT.height,
|
||||
};
|
||||
for (const dir of RESIZE_DIRECTIONS) {
|
||||
const before = await readPanel();
|
||||
const { dx, dy } = growthDelta(dir, 40);
|
||||
const point = handlePoint(before, dir);
|
||||
const hit = await page.evaluate(({ x, y }) => {
|
||||
const node = document.elementFromPoint(x, y);
|
||||
if (!node) return 'nothing';
|
||||
const dirAttr = node.dataset?.dir
|
||||
? `[data-dir=${node.dataset.dir}]`
|
||||
: '';
|
||||
return `${node.tagName.toLowerCase()}${node.id ? `#${node.id}` : ''}.${[...node.classList].join('.')}${dirAttr}`;
|
||||
}, point);
|
||||
await drag(point, dx, dy);
|
||||
const after = await readPanel();
|
||||
const expected = resizeBox(before, dir, dx, dy, limits);
|
||||
for (const key of ['left', 'top', 'width', 'height'])
|
||||
near(after[key], expected[key], 2, `${dir} ${key} (pointer on ${hit})`);
|
||||
assert.deepEqual(driftedEdges(before, after, dir, 2), [], `${dir} pins`);
|
||||
}
|
||||
console.log('PASS: eight resize handles keep the opposite edge pinned');
|
||||
|
||||
let before = await readPanel();
|
||||
await drag(handlePoint(before, 'se'), -2000, -2000);
|
||||
let after = await readPanel();
|
||||
near(after.width, MIN_SIZE.width, 2, 'min width (se)');
|
||||
near(after.height, MIN_SIZE.height, 2, 'min height (se)');
|
||||
assert.deepEqual(driftedEdges(before, after, 'se', 2), []);
|
||||
before = after;
|
||||
await drag(handlePoint(before, 'nw'), 2000, 2000);
|
||||
after = await readPanel();
|
||||
near(after.width, MIN_SIZE.width, 2, 'min width (nw)');
|
||||
near(after.height, MIN_SIZE.height, 2, 'min height (nw)');
|
||||
assert.deepEqual(driftedEdges(before, after, 'nw', 2), []);
|
||||
console.log('PASS: minimum size holds from both corners');
|
||||
|
||||
const saved = await readPanel();
|
||||
const record = JSON.parse(saved.stored);
|
||||
assert.equal(record.floating, true);
|
||||
await boot();
|
||||
const restored = await readPanel();
|
||||
assert.equal(restored.floating, true, 'floating survives a reload');
|
||||
for (const key of ['left', 'top', 'width', 'height'])
|
||||
near(restored[key], record[key], 2, `restored ${key}`);
|
||||
console.log('PASS: reload restores position and size');
|
||||
|
||||
// A collapsed window keeps the height chosen for it, through a drag of
|
||||
// its header strip and a reload.
|
||||
const setCollapsed = async (collapsed) => {
|
||||
await page.evaluate(
|
||||
(id, want) => {
|
||||
const panel = document.getElementById(id);
|
||||
if (panel.classList.contains('collapsed') !== want)
|
||||
document
|
||||
.querySelector(
|
||||
`.panel-collapse-btn[data-collapse-target="${id}"]`,
|
||||
)
|
||||
.click();
|
||||
},
|
||||
PANEL_ID,
|
||||
collapsed,
|
||||
);
|
||||
await page.waitForFunction(
|
||||
(id, want) =>
|
||||
document.getElementById(id).classList.contains('collapsed') === want,
|
||||
{},
|
||||
PANEL_ID,
|
||||
collapsed,
|
||||
);
|
||||
await nextFrames(page);
|
||||
};
|
||||
// Grow it from the corner once the restored window holds still, with the
|
||||
// pointer really on the handle (a press beside it selects page text).
|
||||
let chosen = await readPanel();
|
||||
const tried = [];
|
||||
for (
|
||||
let attempt = 0;
|
||||
attempt < 3 && chosen.height <= MIN_SIZE.height + 150;
|
||||
attempt++
|
||||
) {
|
||||
await settle();
|
||||
const box = await readPanel();
|
||||
const corner = handlePoint(box, 'se');
|
||||
const hit = await page.evaluate(({ x, y }) => {
|
||||
const node = document.elementFromPoint(x, y);
|
||||
return {
|
||||
dir: node?.dataset?.dir || null,
|
||||
target: node
|
||||
? `${node.tagName.toLowerCase()}${node.id ? `#${node.id}` : ''}.${[...node.classList].join('.')}`
|
||||
: 'nothing',
|
||||
};
|
||||
}, corner);
|
||||
tried.push(
|
||||
`${Math.round(corner.x)},${Math.round(corner.y)} on ${hit.target} (box ${Math.round(box.left)},${Math.round(box.top)} ${Math.round(box.width)}x${Math.round(box.height)})`,
|
||||
);
|
||||
if (hit.dir === 'se') await drag(corner, 0, 200);
|
||||
chosen = await readPanel();
|
||||
}
|
||||
assert.ok(
|
||||
chosen.height > MIN_SIZE.height + 150,
|
||||
`the window grew to ${chosen.height}px; tried ${tried.join('; ')}`,
|
||||
);
|
||||
await setCollapsed(true);
|
||||
const strip = await readPanel();
|
||||
assert.ok(strip.height < chosen.height - 100, 'collapsed to its header');
|
||||
await drag(await headerPoint(), 40, 30);
|
||||
const movedStrip = await readPanel();
|
||||
assert.equal(movedStrip.floating, true, 'the strip stays a window');
|
||||
near(
|
||||
JSON.parse(movedStrip.stored).height,
|
||||
chosen.height,
|
||||
2,
|
||||
'stored height while collapsed',
|
||||
);
|
||||
await boot();
|
||||
await setCollapsed(false);
|
||||
near(
|
||||
(await readPanel()).height,
|
||||
chosen.height,
|
||||
2,
|
||||
'height after expanding',
|
||||
);
|
||||
console.log(
|
||||
'PASS: a collapsed window keeps its height through a drag and a reload',
|
||||
);
|
||||
|
||||
const dock = await dockPanelByDoubleClick(page, PANEL_ID);
|
||||
const snapped = await readPanel();
|
||||
assert.equal(
|
||||
snapped.floating,
|
||||
false,
|
||||
`double-click snaps back (${describePress(dock)})`,
|
||||
);
|
||||
assert.equal(snapped.inRail, true);
|
||||
assert.equal(snapped.inlineWidth, '');
|
||||
assert.equal(snapped.inlineHeight, '');
|
||||
assert.equal(snapped.stored, null, 'the storage key is gone');
|
||||
console.log('PASS: header double-click returns the panel to the rail');
|
||||
|
||||
const renderErrors = await readRenderErrors(page);
|
||||
assert.deepEqual(
|
||||
[...errors, ...renderErrors.map((text) => `scene.renderError: ${text}`)],
|
||||
[],
|
||||
'no page errors and no Cesium render-loop errors',
|
||||
);
|
||||
console.log('PASS: no page errors and no render-loop errors');
|
||||
} catch (error) {
|
||||
const dir = await saveFailureArtifacts({
|
||||
dir: ARTIFACT_DIR,
|
||||
browser,
|
||||
monitors,
|
||||
error,
|
||||
}).catch(() => null);
|
||||
if (dir) console.error(`failure evidence saved in ${dir}/`);
|
||||
throw error;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
import.meta.url === pathToFileURL(process.argv[1]).href
|
||||
) {
|
||||
main().catch((error) => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
/**
|
||||
* Header press-and-verify for the QA gates that lift a rail panel into a
|
||||
* floating window and dock it again. The rail animates panel heights, so a
|
||||
* point read once can be under another panel by the time the pointer goes
|
||||
* down. Only a provable miss (a recorded press outside the header) is
|
||||
* retried, with a GitHub `::warning::`. With QA_FAIL_ON_RETRY=1 a miss the
|
||||
* panel's own movement cannot explain fails the run instead.
|
||||
*/
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** Retries after the first attempt, and the pause before each. */
|
||||
export const PRESS_RETRIES = 2;
|
||||
const RETRY_PAUSE_MS = 1000;
|
||||
|
||||
/** Whether a retry fails the run (QA_FAIL_ON_RETRY=1 or --fail-on-retry). */
|
||||
export function failOnRetryDefault(
|
||||
env = process.env,
|
||||
argv = process.argv.slice(2),
|
||||
) {
|
||||
return env.QA_FAIL_ON_RETRY === '1' || argv.includes('--fail-on-retry');
|
||||
}
|
||||
|
||||
/**
|
||||
* A failed attempt whose press landed outside the header. No recorded press
|
||||
* is a failure, not a miss: a retry would only hide it.
|
||||
* @param {{done: boolean, pressed: {inHeader: boolean}|null}} attempt
|
||||
*/
|
||||
export function pressMissed(attempt) {
|
||||
return !attempt.done && attempt.pressed?.inHeader === false;
|
||||
}
|
||||
|
||||
/** Whether the panel's box changed by more than 2 px between two reads. */
|
||||
export function panelMoved(before, after) {
|
||||
if (!before || !after) return false;
|
||||
return ['left', 'top', 'width', 'height'].some(
|
||||
(key) => Math.abs((after[key] ?? NaN) - (before[key] ?? NaN)) > 2,
|
||||
);
|
||||
}
|
||||
|
||||
/** The panel's box, rounded, as `left,top widthxheight`. */
|
||||
export function formatBox(box) {
|
||||
if (!box) return 'unknown';
|
||||
const r = (value) => Math.round(value ?? NaN);
|
||||
return `${r(box.left)},${r(box.top)} ${r(box.width)}x${r(box.height)}`;
|
||||
}
|
||||
|
||||
function panelBox(page, panelId) {
|
||||
return page.evaluate((id) => {
|
||||
const r = document.getElementById(id).getBoundingClientRect();
|
||||
return { left: r.left, top: r.top, width: r.width, height: r.height };
|
||||
}, panelId);
|
||||
}
|
||||
|
||||
export function headerPoint(page, panelId) {
|
||||
return page.evaluate((id) => {
|
||||
const title = document.querySelector(`#${id} .panel-header .panel-title`);
|
||||
const r = title.getBoundingClientRect();
|
||||
return { x: r.left + r.width / 2, y: r.top + r.height / 2 };
|
||||
}, panelId);
|
||||
}
|
||||
|
||||
/** Name the element at a point, and whether it is the panel's header. */
|
||||
export function hitAt(page, point, panelId) {
|
||||
return page.evaluate(
|
||||
({ x, y, id }) => {
|
||||
const node = document.elementFromPoint(x, y);
|
||||
if (!node) return { target: 'nothing', inHeader: false };
|
||||
return {
|
||||
target: `${node.tagName.toLowerCase()}${node.id ? `#${node.id}` : ''}.${[...node.classList].join('.')}`,
|
||||
inHeader: Boolean(node.closest(`#${id} .panel-header`)),
|
||||
};
|
||||
},
|
||||
{ ...point, id: panelId },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-resolve the header point until it hits the header or the wait runs out
|
||||
* (the press then records the miss).
|
||||
*/
|
||||
export async function headerTarget(
|
||||
page,
|
||||
panelId,
|
||||
{ tries = 20, intervalMs = 500 } = {},
|
||||
) {
|
||||
let point = await headerPoint(page, panelId);
|
||||
let hit = await hitAt(page, point, panelId);
|
||||
for (let i = 0; i < tries && !hit.inHeader; i++) {
|
||||
await delay(intervalMs);
|
||||
point = await headerPoint(page, panelId);
|
||||
hit = await hitAt(page, point, panelId);
|
||||
}
|
||||
const box = await panelBox(page, panelId);
|
||||
return { point, hit, timedOut: !hit.inHeader, box };
|
||||
}
|
||||
|
||||
/**
|
||||
* Best effort: wait until the panel's box has held still for `stillMs`, so
|
||||
* the header does not move between the hit check and the press.
|
||||
*/
|
||||
export async function waitForStill(
|
||||
page,
|
||||
panelId,
|
||||
{ stillMs = 300, timeout = 5000 } = {},
|
||||
) {
|
||||
try {
|
||||
await page.waitForFunction(
|
||||
(id, ms) => {
|
||||
const r = document.getElementById(id).getBoundingClientRect();
|
||||
const box = [r.left, r.top, r.width, r.height].map(Math.round).join();
|
||||
const now = performance.now();
|
||||
if (window.__qaStillBox !== box) {
|
||||
window.__qaStillBox = box;
|
||||
window.__qaStillSince = now;
|
||||
}
|
||||
return now - window.__qaStillSince >= ms;
|
||||
},
|
||||
{ polling: 100, timeout },
|
||||
panelId,
|
||||
stillMs,
|
||||
);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error?.name !== 'TimeoutError') throw error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Record the next pointerdown's target and the panel's box at that moment. */
|
||||
export function recordNextPress(page, panelId) {
|
||||
return page.evaluate((id) => {
|
||||
window.__qaPressBox = null;
|
||||
window.addEventListener(
|
||||
'pointerdown',
|
||||
(event) => {
|
||||
const r = document.getElementById(id).getBoundingClientRect();
|
||||
const target = event.target;
|
||||
window.__qaPressBox = {
|
||||
left: r.left,
|
||||
top: r.top,
|
||||
width: r.width,
|
||||
height: r.height,
|
||||
target: `${target.tagName?.toLowerCase()}${target.id ? `#${target.id}` : ''}.${[...(target.classList || [])].join('.')}`,
|
||||
inHeader: Boolean(target.closest?.(`#${id} .panel-header`)),
|
||||
};
|
||||
},
|
||||
{ capture: true, once: true },
|
||||
);
|
||||
}, panelId);
|
||||
}
|
||||
|
||||
function readPress(page) {
|
||||
return page.evaluate(() => window.__qaPressBox ?? null);
|
||||
}
|
||||
|
||||
export async function waitForFloating(
|
||||
page,
|
||||
panelId,
|
||||
floating,
|
||||
{ timeout = 3000 } = {},
|
||||
) {
|
||||
try {
|
||||
await page.waitForFunction(
|
||||
(id, want) =>
|
||||
document.getElementById(id).classList.contains('panel-floating') ===
|
||||
want,
|
||||
{ polling: 50, timeout },
|
||||
panelId,
|
||||
floating,
|
||||
);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error?.name !== 'TimeoutError') throw error;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Two paints: by then every layout change an input caused is on screen. */
|
||||
export function nextFrames(page) {
|
||||
return page.evaluate(
|
||||
() =>
|
||||
new Promise((resolve) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve())),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export async function dragPointer(page, from, dx, dy, { steps = 6 } = {}) {
|
||||
await page.mouse.move(from.x, from.y);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move(from.x + dx / 2, from.y + dy / 2, { steps });
|
||||
await page.mouse.move(from.x + dx, from.y + dy, { steps });
|
||||
await page.mouse.up();
|
||||
await nextFrames(page);
|
||||
}
|
||||
|
||||
export function describeRetry(
|
||||
panelId,
|
||||
attempt,
|
||||
{ round, retries, extra = '' },
|
||||
) {
|
||||
const { hit, pressed, timedOut, still, box } = attempt;
|
||||
return [
|
||||
`retry ${round + 1}/${retries}: the press missed the ${panelId} header (on ${pressed?.target ?? hit.target})`,
|
||||
`pre-press hit check ${timedOut ? `timed out (on ${hit.target})` : 'passed'}`,
|
||||
`stillness ${still === false ? 'timed out' : 'held'}`,
|
||||
`panel box at hit check ${formatBox(box)} -> at press ${formatBox(pressed)}`,
|
||||
]
|
||||
.concat(extra ? [extra] : [])
|
||||
.join('; ');
|
||||
}
|
||||
|
||||
/** Retry `press` only while it provably misses the header. */
|
||||
async function pressUntilDecided(
|
||||
page,
|
||||
panelId,
|
||||
press,
|
||||
{
|
||||
retries,
|
||||
log,
|
||||
note,
|
||||
pauseMs = RETRY_PAUSE_MS,
|
||||
failOnRetry = failOnRetryDefault(),
|
||||
},
|
||||
) {
|
||||
for (let round = 0; ; round++) {
|
||||
const still = await waitForStill(page, panelId);
|
||||
await recordNextPress(page, panelId);
|
||||
const { point, hit, timedOut, box } = await headerTarget(page, panelId);
|
||||
const done = await press(point);
|
||||
const attempt = {
|
||||
point,
|
||||
hit,
|
||||
timedOut,
|
||||
still,
|
||||
box,
|
||||
done,
|
||||
pressed: await readPress(page),
|
||||
retries: round,
|
||||
};
|
||||
if (round >= retries || !pressMissed(attempt)) return attempt;
|
||||
const why = describeRetry(panelId, attempt, {
|
||||
round,
|
||||
retries,
|
||||
extra: note ? await note() : '',
|
||||
});
|
||||
if (failOnRetry && !panelMoved(box, attempt.pressed))
|
||||
throw new Error(
|
||||
`QA_FAIL_ON_RETRY=1 turns this retry into a failure: ${why}`,
|
||||
);
|
||||
log(`note: ${why}; retrying`);
|
||||
log(`::warning title=Panel press retried::${why}`);
|
||||
await delay(pauseMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lift a docked panel into a floating window with a header drag.
|
||||
* @returns {Promise<{point: {x: number, y: number}, hit: {target: string, inHeader: boolean}, done: boolean, pressed: object|null}>}
|
||||
*/
|
||||
export function liftPanelByHeader(
|
||||
page,
|
||||
panelId,
|
||||
{
|
||||
dx,
|
||||
dy,
|
||||
retries = PRESS_RETRIES,
|
||||
log = console.log,
|
||||
note,
|
||||
pauseMs,
|
||||
failOnRetry,
|
||||
} = {},
|
||||
) {
|
||||
return pressUntilDecided(
|
||||
page,
|
||||
panelId,
|
||||
async (point) => {
|
||||
await dragPointer(page, point, dx, dy);
|
||||
return waitForFloating(page, panelId, true);
|
||||
},
|
||||
{ retries, log, note, pauseMs, failOnRetry },
|
||||
);
|
||||
}
|
||||
|
||||
/** Dock a floating panel with a header double-click; same result as lifting. */
|
||||
export function dockPanelByDoubleClick(
|
||||
page,
|
||||
panelId,
|
||||
{
|
||||
retries = PRESS_RETRIES,
|
||||
log = console.log,
|
||||
note,
|
||||
pauseMs,
|
||||
failOnRetry,
|
||||
} = {},
|
||||
) {
|
||||
return pressUntilDecided(
|
||||
page,
|
||||
panelId,
|
||||
async (point) => {
|
||||
await page.mouse.click(point.x, point.y, { clickCount: 1 });
|
||||
await page.mouse.click(point.x, point.y, { clickCount: 2 });
|
||||
return waitForFloating(page, panelId, false);
|
||||
},
|
||||
{ retries, log, note, pauseMs, failOnRetry },
|
||||
);
|
||||
}
|
||||
|
||||
/** Where an attempt pressed, for an assertion message. */
|
||||
export function describePress(attempt) {
|
||||
const { point, hit, pressed } = attempt;
|
||||
const where = `pressed ${Math.round(point.x)},${Math.round(point.y)}`;
|
||||
if (!pressed)
|
||||
return `${where}: no pointerdown reached the page (hit check on ${hit.target}, in header: ${hit.inHeader})`;
|
||||
const n = attempt.retries || 0;
|
||||
const retried = n ? `, after ${n} ${n === 1 ? 'retry' : 'retries'}` : '';
|
||||
return `${where} on ${pressed.target}, in header: ${pressed.inHeader}${retried}`;
|
||||
}
|
||||
@@ -219,7 +219,7 @@ async function main() {
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (url.origin === APP_ORIGIN && url.pathname === '/api/ais-live') {
|
||||
if (url.origin === APP_ORIGIN && url.pathname === '/api/vessels') {
|
||||
request.respond({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
|
||||
@@ -229,11 +229,11 @@ async function main() {
|
||||
const nowSec = Date.now() / 1000 + (S.timeOffsetSec || 0);
|
||||
const tRel = nowSec - S.epochMs / 1000;
|
||||
|
||||
if (url.includes('/api/opensky-track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/adsblol/trace')) {
|
||||
if (url.includes('/api/flights/track')) return Promise.resolve(jsonResponse({ path: [] }));
|
||||
if (url.includes('/api/military/track')) {
|
||||
return Promise.resolve(jsonResponse({ timestamp: Math.floor(nowSec), trace: [] }));
|
||||
}
|
||||
if (url.includes('/api/opensky')) {
|
||||
if (url.includes('/api/flights')) {
|
||||
window.__SPR_HITS.opensky++;
|
||||
const states = S.flights.map((f) => {
|
||||
const s = S.stateAt(f, tRel);
|
||||
@@ -249,7 +249,7 @@ async function main() {
|
||||
});
|
||||
return Promise.resolve(jsonResponse({ time: Math.floor(nowSec), states }));
|
||||
}
|
||||
if (url.includes('/api/adsblol/mil')) {
|
||||
if (url.includes('/api/military')) {
|
||||
window.__SPR_HITS.mil++;
|
||||
const ac = S.military.map((m) => {
|
||||
const s = S.stateAt(m, tRel);
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Keyless terrain throttling probe.
|
||||
*
|
||||
* On the keyless map source the browser fetches Re:Earth quantized-mesh tiles
|
||||
* (`terrain.reearth.land/cesium-mesh/ellipsoid/{z}/{x}/{y}.terrain`) directly.
|
||||
* A zoom-out burst asks for a few dozen shallow tiles at once and the upstream
|
||||
* answers HTTP 429 for most of them. This harness drives that burst, counts the
|
||||
* throttled responses, keeps the first 429's headers as evidence, and then
|
||||
* checks whether every throttled tile eventually loaded — the contract the
|
||||
* client-side retry in `src/maps/terrain.js` provides.
|
||||
*
|
||||
* Run against a keyless dev server (no Google or ion credentials):
|
||||
* env -u GOOGLE_MAPS_API_KEY -u CESIUM_ION_TOKEN npm run dev -- --port 4180
|
||||
* QA_BASE_URL=http://localhost:4180 node scripts/qa-terrain-429.mjs
|
||||
*
|
||||
* Flags: `--mode zoomOut|wheel` (default zoomOut), `--notches N` (default 12),
|
||||
* `--settle MS` (default 25000), `--rounds N` (default 1), `--inject N`
|
||||
* (answer the first N tile requests of each burst with a synthetic 429),
|
||||
* `--headful`.
|
||||
* Exits 1 when a throttled tile never recovered or a terrain tile is left in
|
||||
* Cesium's FAILED state.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const appUrl = process.env.QA_BASE_URL || 'http://localhost:4173';
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, fallback) => {
|
||||
const index = argv.indexOf(`--${name}`);
|
||||
return index >= 0 && argv[index + 1] !== undefined
|
||||
? argv[index + 1]
|
||||
: fallback;
|
||||
};
|
||||
const mode = flag('mode', 'zoomOut');
|
||||
const notches = Number(flag('notches', 12));
|
||||
const settleMs = Number(flag('settle', 25000));
|
||||
const rounds = Number(flag('rounds', 1));
|
||||
// The upstream throttle is load-dependent and does not fire on demand. With
|
||||
// `--inject N`, the first N tile requests of each burst are answered 429 by an
|
||||
// XMLHttpRequest shim inside the page so the recovery contract can be proven
|
||||
// deterministically; real 429s are still counted and kept as evidence. (CDP
|
||||
// request interception is not used: it stalls Cesium's request scheduler.)
|
||||
const inject = Number(flag('inject', 0));
|
||||
const headful = argv.includes('--headful');
|
||||
const TERRAIN_HOST = 'terrain.reearth.land';
|
||||
const TILE_PATTERN = /\/cesium-mesh\/ellipsoid\/\d+\/\d+\/\d+\.terrain/;
|
||||
|
||||
const executablePath =
|
||||
process.env.PUPPETEER_EXECUTABLE_PATH ||
|
||||
(await puppeteer.executablePath().catch(() => null));
|
||||
if (!executablePath || !fs.existsSync(executablePath)) {
|
||||
throw new Error('Puppeteer Chrome for Testing is unavailable');
|
||||
}
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
headless: headful ? false : 'new',
|
||||
executablePath,
|
||||
args: [
|
||||
...(process.platform === 'darwin'
|
||||
? ['--use-angle=metal', '--enable-gpu']
|
||||
: ['--use-gl=angle', '--use-angle=swiftshader']),
|
||||
'--no-sandbox',
|
||||
],
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
|
||||
/** @type {Map<string, {statuses: number[], failures: number}>} per tile URL. */
|
||||
const tiles = new Map();
|
||||
const evidence = [];
|
||||
let layerJsonSeen = false;
|
||||
let recording = false;
|
||||
let requestsInWindow = 0;
|
||||
let throttledInWindow = 0;
|
||||
const tileEntry = (url) => {
|
||||
if (!tiles.has(url)) tiles.set(url, { statuses: [], failures: 0 });
|
||||
return tiles.get(url);
|
||||
};
|
||||
|
||||
if (inject > 0) {
|
||||
// Cesium fetches terrain through XMLHttpRequest and reads status, response
|
||||
// and getAllResponseHeaders() in its onload handler; shadowing those on the
|
||||
// instance is enough to make it see a throttled reply.
|
||||
await page.evaluateOnNewDocument((pattern) => {
|
||||
const tilePattern = new RegExp(pattern);
|
||||
const shim = { remaining: 0, injected: [] };
|
||||
window.__qaTerrainThrottle = shim;
|
||||
const open = XMLHttpRequest.prototype.open;
|
||||
const send = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function (method, url, ...rest) {
|
||||
this.__qaUrl = String(url);
|
||||
return open.call(this, method, url, ...rest);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function (...args) {
|
||||
if (!(shim.remaining > 0 && tilePattern.test(this.__qaUrl || ''))) {
|
||||
return send.apply(this, args);
|
||||
}
|
||||
shim.remaining -= 1;
|
||||
shim.injected.push(this.__qaUrl);
|
||||
const xhr = this;
|
||||
setTimeout(() => {
|
||||
for (const [key, value] of [
|
||||
['status', 429],
|
||||
['readyState', 4],
|
||||
['response', null],
|
||||
]) {
|
||||
Object.defineProperty(xhr, key, { value, configurable: true });
|
||||
}
|
||||
xhr.getAllResponseHeaders = () => 'retry-after: 1\r\n';
|
||||
xhr.onload?.(new Event('load'));
|
||||
}, 30);
|
||||
return undefined;
|
||||
};
|
||||
}, TILE_PATTERN.source);
|
||||
}
|
||||
|
||||
page.on('response', async (response) => {
|
||||
const url = response.url();
|
||||
if (!url.includes(TERRAIN_HOST)) return;
|
||||
if (url.endsWith('layer.json')) layerJsonSeen = true;
|
||||
if (!TILE_PATTERN.test(url)) return;
|
||||
const status = response.status();
|
||||
tileEntry(url).statuses.push(status);
|
||||
if (recording) requestsInWindow += 1;
|
||||
if (status !== 429) return;
|
||||
if (recording) throttledInWindow += 1;
|
||||
if (evidence.length < 3) {
|
||||
let body = '';
|
||||
try {
|
||||
body = (await response.text()).slice(0, 160);
|
||||
} catch {
|
||||
/* body unavailable */
|
||||
}
|
||||
evidence.push({ url, status, headers: response.headers(), body });
|
||||
}
|
||||
});
|
||||
page.on('requestfailed', (request) => {
|
||||
const url = request.url();
|
||||
if (!TILE_PATTERN.test(url)) return;
|
||||
tileEntry(url).failures += 1;
|
||||
if (recording) requestsInWindow += 1;
|
||||
});
|
||||
|
||||
const consoleErrors = [];
|
||||
page.on('pageerror', (error) => consoleErrors.push(error.message));
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error') consoleErrors.push(message.text());
|
||||
});
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const waitFor = async (predicate, timeoutMs, label) => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (await predicate()) return;
|
||||
await sleep(250);
|
||||
}
|
||||
throw new Error(`timed out waiting for ${label}`);
|
||||
};
|
||||
|
||||
const setCamera = (lon, lat, height) =>
|
||||
page.evaluate(
|
||||
({ lon, lat, height }) => {
|
||||
const viewer = window.__godsEyeView.viewer;
|
||||
// The app's own startup flight would otherwise keep steering the camera
|
||||
// through the burst and turn the zoom-out into a zoom-in.
|
||||
viewer.camera.cancelFlight();
|
||||
const cartographic =
|
||||
viewer.camera.positionCartographic.constructor.fromDegrees(
|
||||
lon,
|
||||
lat,
|
||||
height,
|
||||
);
|
||||
viewer.camera.setView({
|
||||
destination:
|
||||
viewer.scene.globe.ellipsoid.cartographicToCartesian(cartographic),
|
||||
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
|
||||
});
|
||||
window.__godsEyeView.requestRender?.();
|
||||
},
|
||||
{ lon, lat, height },
|
||||
);
|
||||
|
||||
const cameraHeight = () =>
|
||||
page.evaluate(
|
||||
() => window.__godsEyeView.viewer.camera.positionCartographic.height,
|
||||
);
|
||||
|
||||
const globeState = () =>
|
||||
page.evaluate(() => {
|
||||
const viewer = window.__godsEyeView.viewer;
|
||||
const surface = viewer.scene.globe._surface;
|
||||
let failed = 0;
|
||||
let loaded = 0;
|
||||
const visit = (tile) => {
|
||||
if (!tile) return;
|
||||
const state = tile.data?.terrainState;
|
||||
if (state === 0) failed += 1;
|
||||
else if (state === 6) loaded += 1;
|
||||
for (const child of tile._children || []) visit(child);
|
||||
};
|
||||
for (const tile of surface._levelZeroTiles || []) visit(tile);
|
||||
return {
|
||||
failed,
|
||||
loaded,
|
||||
tilesLoaded: viewer.scene.globe.tilesLoaded,
|
||||
provider: viewer.terrainProvider?.constructor?.name,
|
||||
};
|
||||
});
|
||||
|
||||
const burst = async () => {
|
||||
if (mode === 'zoomOut') {
|
||||
for (let i = 0; i < notches; i += 1) {
|
||||
await page.evaluate(() => {
|
||||
const viewer = window.__godsEyeView.viewer;
|
||||
viewer.camera.zoomOut(viewer.camera.positionCartographic.height * 0.6);
|
||||
window.__godsEyeView.requestRender?.();
|
||||
});
|
||||
await sleep(40);
|
||||
}
|
||||
return;
|
||||
}
|
||||
await page.mouse.move(640, 400);
|
||||
for (let i = 0; i < notches; i += 1) {
|
||||
await page.mouse.wheel({ deltaY: 200 });
|
||||
await page.evaluate(() => window.__godsEyeView.requestRender?.());
|
||||
await sleep(40);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
await page.goto(appUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForFunction(() => Boolean(window.__godsEyeView?.viewer), {
|
||||
timeout: 60000,
|
||||
});
|
||||
await waitFor(() => layerJsonSeen, 60000, 'the Re:Earth layer.json fetch');
|
||||
await waitFor(
|
||||
async () => (await globeState()).provider === 'CesiumTerrainProvider',
|
||||
30000,
|
||||
'the keyless terrain provider',
|
||||
);
|
||||
|
||||
const summaries = [];
|
||||
for (let round = 1; round <= rounds; round += 1) {
|
||||
// Re-apply the view until it sticks: a still-running startup flight can
|
||||
// overwrite the first setView on the very next frame.
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
await setCamera(-97.7431, 30.2672, 40000);
|
||||
await sleep(1500);
|
||||
const height = await cameraHeight();
|
||||
if (Math.abs(height - 40000) < 1000) break;
|
||||
}
|
||||
await waitFor(
|
||||
async () => (await globeState()).tilesLoaded,
|
||||
30000,
|
||||
'the 40 km view to settle',
|
||||
).catch(() => {});
|
||||
await sleep(2000);
|
||||
const settledHeight = await cameraHeight();
|
||||
if (Math.abs(settledHeight - 40000) > 1000) {
|
||||
throw new Error(
|
||||
`camera did not hold the 40 km start view (at ${Math.round(settledHeight)} m)`,
|
||||
);
|
||||
}
|
||||
const before = await globeState();
|
||||
tiles.clear();
|
||||
requestsInWindow = 0;
|
||||
throttledInWindow = 0;
|
||||
if (inject > 0) {
|
||||
await page.evaluate((count) => {
|
||||
window.__qaTerrainThrottle.remaining = count;
|
||||
window.__qaTerrainThrottle.injected = [];
|
||||
}, inject);
|
||||
}
|
||||
recording = true;
|
||||
const startHeight = await cameraHeight();
|
||||
await burst();
|
||||
const endHeight = await cameraHeight();
|
||||
await sleep(settleMs);
|
||||
recording = false;
|
||||
|
||||
// Injected throttles never reach the network, so fold them in from the
|
||||
// page: a later real 200 for the same tile is the retry succeeding.
|
||||
const injected =
|
||||
inject > 0
|
||||
? await page.evaluate(() => window.__qaTerrainThrottle.injected)
|
||||
: [];
|
||||
for (const url of injected) tileEntry(url).statuses.unshift(429);
|
||||
const throttled = [...tiles.entries()].filter(([, entry]) =>
|
||||
entry.statuses.includes(429),
|
||||
);
|
||||
const recovered = throttled.filter(([, entry]) =>
|
||||
entry.statuses.includes(200),
|
||||
);
|
||||
const unrecovered = throttled.filter(
|
||||
([, entry]) => !entry.statuses.includes(200),
|
||||
);
|
||||
const after = await globeState();
|
||||
summaries.push({
|
||||
round,
|
||||
mode,
|
||||
notches,
|
||||
startHeightKm: Math.round(startHeight / 1000),
|
||||
endHeightKm: Math.round(endHeight / 1000),
|
||||
tileResponses: requestsInWindow,
|
||||
throttledResponses: throttledInWindow,
|
||||
injectedThrottles: injected.length,
|
||||
throttledTiles: throttled.length,
|
||||
recoveredTiles: recovered.length,
|
||||
unrecoveredTiles: unrecovered.length,
|
||||
failedTilesBefore: before.failed,
|
||||
failedTilesAfter: after.failed,
|
||||
loadedTilesAfter: after.loaded,
|
||||
});
|
||||
console.log(JSON.stringify(summaries.at(-1)));
|
||||
if (unrecovered.length) {
|
||||
console.log(
|
||||
' unrecovered: ' +
|
||||
unrecovered
|
||||
.slice(0, 8)
|
||||
.map(
|
||||
([url, entry]) =>
|
||||
`${new URL(url).pathname} [${entry.statuses.join(',')}]`,
|
||||
)
|
||||
.join(' · '),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (evidence.length) {
|
||||
console.log('first 429 evidence:');
|
||||
for (const item of evidence) {
|
||||
const headers = Object.fromEntries(
|
||||
Object.entries(item.headers).filter(([name]) =>
|
||||
/^(cf-|retry-after|server|content-type|date|x-ratelimit|ratelimit|access-control)/i.test(
|
||||
name,
|
||||
),
|
||||
),
|
||||
);
|
||||
console.log(
|
||||
` ${new URL(item.url).pathname} ${JSON.stringify(headers)} body=${JSON.stringify(item.body)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const failures = summaries.filter(
|
||||
(summary) => summary.unrecoveredTiles > 0 || summary.failedTilesAfter > 0,
|
||||
);
|
||||
const terrainErrors = consoleErrors.filter((text) =>
|
||||
/terrain tile/i.test(text),
|
||||
);
|
||||
if (terrainErrors.length) {
|
||||
console.log(`terrain console errors: ${terrainErrors.length}`);
|
||||
console.log(` ${terrainErrors[0].slice(0, 200)}`);
|
||||
}
|
||||
console.log(
|
||||
failures.length
|
||||
? `FAIL — ${failures.length}/${summaries.length} round(s) left throttled terrain tiles unloaded`
|
||||
: `PASS — every throttled terrain tile recovered in ${summaries.length} round(s)`,
|
||||
);
|
||||
process.exitCode = failures.length ? 1 : 0;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
@@ -0,0 +1,533 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Traffic motion acceptance: real-GPU Puppeteer, stable identities, stationary
|
||||
* camera holds and timestamped CDP JPEGs encoded with ffmpeg. No tile mocks.
|
||||
* Run: node scripts/qa-traffic-motion.mjs http://localhost:4186
|
||||
* --baseline records the unmodified pre-fix implementation (object identities
|
||||
* via WeakMap, never collection indices); it reports failures without exiting 1.
|
||||
* --mode=tomtom|osm|hybrid limits a diagnostic run. --tag=name keeps captures.
|
||||
* --journey follows a neighborhood-to-city route with layer toggles, then
|
||||
* checks Capitol/greenbelt/parking/corridor views against a 1.5 s limit.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import puppeteer from 'puppeteer';
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-traffic-motion.mjs <url> [--baseline] [--mode=tomtom|osm|hybrid] [--tag=name] [--journey]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const url = args.find((a) => /^https?:/.test(a));
|
||||
if (!url) throw new Error('A running dev server URL is required');
|
||||
const baseline = args.includes('--baseline');
|
||||
const journey = args.includes('--journey');
|
||||
const option = (key) =>
|
||||
args.find((a) => a.startsWith(`--${key}=`))?.split('=')[1];
|
||||
const modes = option('mode') ? [option('mode')] : ['tomtom', 'osm', 'hybrid'];
|
||||
if (modes.some((m) => !['tomtom', 'osm', 'hybrid'].includes(m)))
|
||||
throw new Error('Invalid road mode');
|
||||
const root = path.resolve(
|
||||
'qa-shots/motion',
|
||||
option('tag') || (baseline ? 'before' : '.'),
|
||||
);
|
||||
fs.mkdirSync(root, { recursive: true });
|
||||
const revision = execFileSync('git', ['rev-parse', 'HEAD'], {
|
||||
encoding: 'utf8',
|
||||
}).trim();
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
protocolTimeout: 300000,
|
||||
args: [
|
||||
'--use-angle=metal',
|
||||
'--window-size=1280,800',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
],
|
||||
});
|
||||
const results = [];
|
||||
const delay = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
try {
|
||||
for (const mode of modes) {
|
||||
const dir = path.join(root, mode);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const page = await browser.newPage();
|
||||
const errors = [],
|
||||
forbidden = [],
|
||||
logs = [],
|
||||
tileRequests = [];
|
||||
page.on('pageerror', (e) => errors.push(e.message));
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') errors.push(m.text());
|
||||
if (m.text().startsWith('[Data:Traffic]')) logs.push(m.text());
|
||||
});
|
||||
page.on('request', (r) => {
|
||||
const u = new URL(r.url());
|
||||
if (
|
||||
u.hostname === 'tiles.openfreemap.org' &&
|
||||
/\/\d+\/\d+\/\d+(?:\.pbf)?$/.test(u.pathname)
|
||||
)
|
||||
tileRequests.push(u.pathname);
|
||||
if (
|
||||
/overpass/i.test(u.hostname) ||
|
||||
u.hostname === 'nominatim.openstreetmap.org'
|
||||
)
|
||||
forbidden.push(u.hostname);
|
||||
});
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
const u = new URL(url);
|
||||
u.searchParams.set('trafficRoads', mode);
|
||||
u.searchParams.set('trafficDebug', '1');
|
||||
await page.goto(u.href, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await page.evaluate(() => {
|
||||
const g = window.__godsEyeView;
|
||||
g.styleManager.setDetection({ enabled: false });
|
||||
for (const [id] of g.dataManager.layers)
|
||||
g.dataManager.setEnabled(id, false);
|
||||
g.viewer.scene.screenSpaceCameraController.enableCollisionDetection = false;
|
||||
});
|
||||
const identity = await page.evaluate(() => ({
|
||||
title: document.title,
|
||||
text: document.body.innerText.length,
|
||||
overlay: Boolean(document.querySelector('vite-error-overlay')),
|
||||
}));
|
||||
await page.evaluate((baseline) => {
|
||||
const { viewer, dataManager } = window.__godsEyeView;
|
||||
const layer = dataManager.layers.get('traffic').module;
|
||||
if (!baseline && typeof layer.visitMotionDots !== 'function')
|
||||
throw new Error('Traffic must expose stable dot identities');
|
||||
const scene = viewer.scene;
|
||||
const weakIds = new WeakMap();
|
||||
let nextId = 0,
|
||||
rebuilds = 0;
|
||||
const collections = [];
|
||||
function discoverCollections() {
|
||||
for (let i = 0; i < scene.primitives.length; i++) {
|
||||
const p = scene.primitives.get(i);
|
||||
if (!Array.isArray(p._pointPrimitives) || collections.includes(p))
|
||||
continue;
|
||||
collections.push(p);
|
||||
const removeAll = p.removeAll;
|
||||
p.removeAll = function () {
|
||||
if (this.length) rebuilds++;
|
||||
return removeAll.call(this);
|
||||
};
|
||||
}
|
||||
}
|
||||
let previous = new Map(),
|
||||
previousN = 0,
|
||||
lastRebuilds = 0,
|
||||
lastTime = 0;
|
||||
const rows = [];
|
||||
window.__motion = {
|
||||
rows,
|
||||
active: false,
|
||||
moveEnds: [],
|
||||
label: 'camera moving',
|
||||
holdStart: null,
|
||||
};
|
||||
const clock = document.createElement('div');
|
||||
clock.style.cssText =
|
||||
'position:fixed;bottom:4px;left:38%;padding:3px 8px;background:#000c;color:#fff;font:12px monospace;z-index:999999;pointer-events:none';
|
||||
document.body.append(clock);
|
||||
viewer.camera.moveEnd.addEventListener(() =>
|
||||
window.__motion.moveEnds.push(performance.now()),
|
||||
);
|
||||
scene.postRender.addEventListener(() => {
|
||||
if (!window.__motion.active) return;
|
||||
discoverCollections();
|
||||
const now = performance.now(),
|
||||
points = new Map();
|
||||
clock.textContent =
|
||||
`QA: ${window.__motion.label}` +
|
||||
(window.__motion.holdStart === null
|
||||
? ''
|
||||
: ` | camera stopped +${((now - window.__motion.holdStart) / 1000).toFixed(1)}s`);
|
||||
const visit = (id, roadId, point) => {
|
||||
if (!point.show) return;
|
||||
const s = scene.cartesianToCanvasCoordinates(point.position);
|
||||
if (s && s.x >= 0 && s.y >= 0 && s.x <= 1280 && s.y <= 800)
|
||||
points.set(id, { x: s.x, y: s.y, roadId });
|
||||
};
|
||||
let n = 0,
|
||||
offRoad = 0;
|
||||
if (!baseline && layer.visitMotionDots) {
|
||||
layer.visitMotionDots((id, roadId, point, road) => {
|
||||
if (point.show) {
|
||||
const p = road?.roadProperties;
|
||||
const allowed = road?.directFlow
|
||||
? [
|
||||
'Motorway',
|
||||
'International road',
|
||||
'Major road',
|
||||
'Secondary road',
|
||||
'Connecting road',
|
||||
'Major local road',
|
||||
'Local road',
|
||||
'Minor local road',
|
||||
].includes(p?.roadType)
|
||||
: [
|
||||
'motorway',
|
||||
'trunk',
|
||||
'primary',
|
||||
'secondary',
|
||||
'tertiary',
|
||||
'minor',
|
||||
].includes(p?.class) &&
|
||||
!p?.service &&
|
||||
!p?.subclass &&
|
||||
!['no', 'private'].includes(p?.access) &&
|
||||
p?.brunnel !== 'tunnel';
|
||||
if (!allowed) offRoad++;
|
||||
}
|
||||
if (point.show) n++;
|
||||
visit(id, roadId, point);
|
||||
});
|
||||
} else {
|
||||
for (const coll of collections)
|
||||
if (coll.show)
|
||||
for (let i = 0; i < coll.length; i++) {
|
||||
const p = coll.get(i);
|
||||
if (!weakIds.has(p)) weakIds.set(p, ++nextId);
|
||||
if (p.show) n++;
|
||||
visit(weakIds.get(p), null, p);
|
||||
}
|
||||
}
|
||||
let tracked = 0,
|
||||
jumps = 0,
|
||||
reassigned = 0,
|
||||
births = 0,
|
||||
deaths = 0;
|
||||
for (const [id, p] of points) {
|
||||
const prev = previous.get(id);
|
||||
if (!prev) {
|
||||
births++;
|
||||
continue;
|
||||
}
|
||||
tracked++;
|
||||
if (Math.hypot(p.x - prev.x, p.y - prev.y) > 30) jumps++;
|
||||
if (p.roadId !== prev.roadId) reassigned++;
|
||||
}
|
||||
for (const id of previous.keys()) if (!points.has(id)) deaths++;
|
||||
rows.push({
|
||||
t: now,
|
||||
loading: layer.getStats().loading,
|
||||
syncing:
|
||||
document
|
||||
.getElementById('traffic-sync-chip')
|
||||
?.classList.contains('visible') &&
|
||||
document.getElementById('traffic-sync-progress')?.textContent ===
|
||||
'...',
|
||||
surfacePending: layer.getStats().surfacePending,
|
||||
offRoad,
|
||||
dt: now - lastTime,
|
||||
n,
|
||||
visible: points.size,
|
||||
tracked,
|
||||
jumps,
|
||||
jumpFrac: tracked ? jumps / tracked : 0,
|
||||
reassigned,
|
||||
births,
|
||||
deaths,
|
||||
sizeChanged: n !== previousN,
|
||||
rebuilds: rebuilds - lastRebuilds,
|
||||
});
|
||||
previous = points;
|
||||
previousN = n;
|
||||
lastRebuilds = rebuilds;
|
||||
lastTime = now;
|
||||
});
|
||||
}, baseline);
|
||||
const fly = (lat, lon, height, pitch, duration, heading = 20) =>
|
||||
page.evaluate(
|
||||
async (v) => {
|
||||
const { viewer } = window.__godsEyeView;
|
||||
window.__motion.label = 'camera moving';
|
||||
window.__motion.holdStart = null;
|
||||
viewer.camera.cancelFlight();
|
||||
await new Promise((resolve) =>
|
||||
viewer.camera.flyTo({
|
||||
destination: viewer.scene.globe.ellipsoid.cartographicToCartesian(
|
||||
{
|
||||
latitude: (v.lat * Math.PI) / 180,
|
||||
longitude: (v.lon * Math.PI) / 180,
|
||||
height: v.height,
|
||||
},
|
||||
),
|
||||
orientation: {
|
||||
heading: (v.heading * Math.PI) / 180,
|
||||
pitch: (v.pitch * Math.PI) / 180,
|
||||
roll: 0,
|
||||
},
|
||||
duration: v.duration,
|
||||
complete: resolve,
|
||||
cancel: resolve,
|
||||
}),
|
||||
);
|
||||
},
|
||||
{ lat, lon, height, pitch, duration, heading },
|
||||
);
|
||||
const cdp = await page.createCDPSession();
|
||||
let fi = 0;
|
||||
const frames = [];
|
||||
cdp.on('Page.screencastFrame', (f) => {
|
||||
const name = `f${String(fi++).padStart(5, '0')}.jpg`;
|
||||
fs.writeFileSync(path.join(dir, name), Buffer.from(f.data, 'base64'));
|
||||
frames.push({ name, timestamp: f.metadata.timestamp });
|
||||
cdp
|
||||
.send('Page.screencastFrameAck', { sessionId: f.sessionId })
|
||||
.catch(() => {});
|
||||
});
|
||||
await fly(
|
||||
...(journey
|
||||
? [30.25, -97.7497, 700, -40, 0, 10]
|
||||
: [30.2685, -97.7425, 450, -35, 0]),
|
||||
);
|
||||
await cdp.send('Page.startScreencast', {
|
||||
format: 'jpeg',
|
||||
quality: 65,
|
||||
maxWidth: 1280,
|
||||
maxHeight: 800,
|
||||
everyNthFrame: journey ? 1 : 3,
|
||||
});
|
||||
await page.evaluate(() => {
|
||||
window.__motion.active = true;
|
||||
});
|
||||
const holds = [];
|
||||
const hold = async (name, warm = false, holdMs = 10000, empty = false) => {
|
||||
const start = await page.evaluate((label) => {
|
||||
const start = performance.now();
|
||||
window.__motion.label = label;
|
||||
window.__motion.holdStart = start;
|
||||
return start;
|
||||
}, name);
|
||||
await delay(holdMs);
|
||||
const end = await page.evaluate(() => performance.now());
|
||||
holds.push({
|
||||
name,
|
||||
warm,
|
||||
start,
|
||||
end,
|
||||
empty,
|
||||
settleMs: journey ? 1500 : warm ? 1000 : 3000,
|
||||
});
|
||||
await page.screenshot({ path: path.join(dir, `${name}.png`) });
|
||||
console.log(`${mode}: captured ${name}`);
|
||||
};
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
// Match motion-reference: begin the camera-motion scenario from a
|
||||
// settled street view. Initial mesh acquisition is recorded in the clip
|
||||
// and first-dot diagnostics, but is not a stationary motion hold.
|
||||
if (journey) {
|
||||
await delay(4000);
|
||||
await hold('south-congress', true, 6000);
|
||||
const step = async (name, view, ms = 6000, empty = false) => {
|
||||
await fly(...view);
|
||||
await hold(name, false, ms, empty);
|
||||
};
|
||||
await step('pan-north-400m', [30.2536, -97.7497, 700, -40, 2, 10], 4000);
|
||||
await step('pan-east-500m', [30.2536, -97.7445, 700, -40, 2, 10], 4000);
|
||||
await step('zoom-out-2.5km', [30.2536, -97.7445, 2500, -45, 2.5, 10]);
|
||||
await step(
|
||||
'downtown-2.5km',
|
||||
[30.2672, -97.7431, 2500, -45, 2.5, 10],
|
||||
5000,
|
||||
);
|
||||
await step('downtown-500m', [30.2672, -97.7431, 500, -35, 2.5, 20]);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('alpr-cameras', true),
|
||||
);
|
||||
await hold('enable-alpr', true, 6000);
|
||||
await step('pan-west-600m', [30.2672, -97.7494, 500, -35, 2, 20], 5000);
|
||||
await step('city-12km', [30.28, -97.75, 12000, -60, 3, 0], 8000, true);
|
||||
await fly(30.3125, -97.765, 1000, -45, 3, 0);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled(
|
||||
'military-installations',
|
||||
true,
|
||||
),
|
||||
);
|
||||
await hold('camp-mabry-1km', false, 8000);
|
||||
await step('camp-mabry-pan', [30.3125, -97.76, 1000, -45, 2, 30], 5000);
|
||||
await step('greenbelt', [30.312, -97.775, 1000, -55, 3, 0], 6000);
|
||||
await step('capitol', [30.2747, -97.7404, 500, -70, 3, 0], 6000);
|
||||
await step('commercial-parking', [30.307, -97.735, 600, -60, 3, 0], 6000);
|
||||
await step('corridor', [30.347, -97.799, 1000, -45, 3, 0], 6000);
|
||||
} else {
|
||||
await delay(12000);
|
||||
await page.waitForFunction(
|
||||
() => window.__motion.rows.some((r) => r.n > 0),
|
||||
{ timeout: 1000 },
|
||||
);
|
||||
await hold('street-warm', true);
|
||||
await fly(30.2685, -97.7425, 3000, -45, 1.5);
|
||||
await hold('zoom-out');
|
||||
await fly(30.2685, -97.72, 3000, -45, 1.5);
|
||||
await hold('pan-east');
|
||||
await fly(30.2685, -97.725, 600, -35, 1.5);
|
||||
await hold('zoom-in');
|
||||
await fly(-23.5505, -46.6333, 1250, -35, 1.5);
|
||||
await hold('sao-paulo');
|
||||
await fly(30.2685, -97.7425, 450, -35, 1.5);
|
||||
await hold('austin-revisit', true);
|
||||
}
|
||||
await page.evaluate(() => {
|
||||
window.__motion.active = false;
|
||||
});
|
||||
await cdp.send('Page.stopScreencast');
|
||||
const rows = await page.evaluate(() => window.__motion.rows);
|
||||
const surfacePhases = await page.evaluate(() =>
|
||||
performance
|
||||
.getEntriesByType('measure')
|
||||
.filter(
|
||||
(e) => e.name.startsWith('roads:') || e.name.startsWith('traffic:'),
|
||||
)
|
||||
.map((e) => ({
|
||||
name: e.name,
|
||||
start: e.startTime,
|
||||
duration: e.duration,
|
||||
detail: e.detail,
|
||||
})),
|
||||
);
|
||||
const stats = await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.layers.get('traffic').module.getStats(),
|
||||
);
|
||||
const phases = holds.map((h) => {
|
||||
const all = rows.filter((r) => r.t >= h.start && r.t < h.end);
|
||||
const settled = all.filter((r) => r.t >= h.start + h.settleMs);
|
||||
const n = settled.map((r) => r.n);
|
||||
const lastBusy = all.findLastIndex(
|
||||
(r) => r.loading || r.syncing || r.sizeChanged || r.jumpFrac > 0.02,
|
||||
);
|
||||
// First observed settled frame after the last population/loading event.
|
||||
const steadyMs =
|
||||
lastBusy < 0
|
||||
? 0
|
||||
: Math.round((all[lastBusy + 1]?.t ?? h.end) - h.start);
|
||||
const phase = {
|
||||
mode,
|
||||
phase: h.name,
|
||||
steadyMs,
|
||||
lateLoadingFrames: settled.filter((r) => r.loading).length,
|
||||
offRoadDots: Math.max(0, ...all.map((r) => r.offRoad)),
|
||||
settleMs: h.settleMs,
|
||||
frames: settled.length,
|
||||
rebuilds: all.reduce((s, r) => s + r.rebuilds, 0),
|
||||
lateRebuilds: settled.reduce((s, r) => s + r.rebuilds, 0),
|
||||
jumpFrames: settled.filter((r) => r.jumpFrac > 0.02).length,
|
||||
reassigned: settled.reduce((s, r) => s + r.reassigned, 0),
|
||||
sizeChanges: settled.filter((r) => r.sizeChanged).length,
|
||||
minDots: Math.min(...n),
|
||||
maxDots: Math.max(...n),
|
||||
maxFrameMs: Math.round(Math.max(...settled.map((r) => r.dt))),
|
||||
};
|
||||
phase.passed =
|
||||
phase.frames >= 20 &&
|
||||
phase.rebuilds <= 1 &&
|
||||
phase.lateRebuilds === 0 &&
|
||||
phase.jumpFrames === 0 &&
|
||||
phase.reassigned === 0 &&
|
||||
phase.sizeChanges === 0 &&
|
||||
(h.empty ? phase.maxDots === 0 : phase.minDots > 0) &&
|
||||
(!journey ||
|
||||
(phase.steadyMs <= 1500 &&
|
||||
phase.offRoadDots === 0 &&
|
||||
all.at(-1)?.surfacePending === 0));
|
||||
return phase;
|
||||
});
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'frames.ffconcat'),
|
||||
'ffconcat version 1.0\n' +
|
||||
frames
|
||||
.map(
|
||||
(f, i) =>
|
||||
`file '${f.name}'\nduration ${Math.max(0.001, Math.min(2, (frames[i + 1]?.timestamp || f.timestamp + 0.1) - f.timestamp))}\n`,
|
||||
)
|
||||
.join(''),
|
||||
);
|
||||
execFileSync('ffmpeg', [
|
||||
'-y',
|
||||
'-loglevel',
|
||||
'error',
|
||||
'-safe',
|
||||
'0',
|
||||
'-f',
|
||||
'concat',
|
||||
'-i',
|
||||
path.join(dir, 'frames.ffconcat'),
|
||||
'-fps_mode',
|
||||
'vfr',
|
||||
'-c:v',
|
||||
'libx264',
|
||||
'-pix_fmt',
|
||||
'yuv420p',
|
||||
'-vf',
|
||||
journey ? 'scale=1280:-2' : 'scale=960:-2',
|
||||
path.join(dir, 'clip.mp4'),
|
||||
]);
|
||||
const result = {
|
||||
mode,
|
||||
revision,
|
||||
baseline,
|
||||
identity,
|
||||
url: u.href,
|
||||
phases,
|
||||
errors,
|
||||
forbidden,
|
||||
tileRequests,
|
||||
duplicateTiles: tileRequests.filter(
|
||||
(p, i) => tileRequests.indexOf(p) !== i,
|
||||
),
|
||||
stats,
|
||||
surfacePhases,
|
||||
logs,
|
||||
holds,
|
||||
rows,
|
||||
moveEnds: await page.evaluate(() => window.__motion.moveEnds),
|
||||
clip: path.join(dir, 'clip.mp4'),
|
||||
frames: frames.length,
|
||||
};
|
||||
result.passed =
|
||||
phases.every((p) => p.passed) &&
|
||||
!errors.length &&
|
||||
!forbidden.length &&
|
||||
result.duplicateTiles.length === 0 &&
|
||||
!identity.overlay &&
|
||||
identity.text > 100;
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'metrics.json'),
|
||||
JSON.stringify(result, null, 2),
|
||||
);
|
||||
results.push(result);
|
||||
console.table(phases);
|
||||
await page.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
fs.writeFileSync(
|
||||
path.join(root, 'results.json'),
|
||||
JSON.stringify(
|
||||
results.map(({ rows, ...r }) => r),
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (!baseline && results.some((r) => !r.passed)) process.exitCode = 1;
|
||||
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env node
|
||||
/** Reticle coverage on oblique arrivals: source requests, toggle oracle and zoom revisits. */
|
||||
import fs from 'node:fs';
|
||||
import puppeteer from 'puppeteer';
|
||||
import {
|
||||
moveCamera,
|
||||
createScreencast,
|
||||
installFrameProbe,
|
||||
} from './qa-journey-recorder.mjs';
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes('--help')) {
|
||||
console.log(
|
||||
'Usage: node scripts/qa-traffic-oblique.mjs <url> [--baseline] [--mode=hybrid|osm|tomtom]',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
const baseline = args.includes('--baseline');
|
||||
const modeArg = args.find((a) => a.startsWith('--mode='))?.split('=')[1];
|
||||
const modes = modeArg ? [modeArg] : ['tomtom', 'osm', 'hybrid'];
|
||||
const out = `qa-shots/oblique-${baseline ? 'before' : 'after'}`;
|
||||
fs.mkdirSync(out, { recursive: true });
|
||||
const result = { views: [], errors: [], forbidden: [], failures: [] };
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
protocolTimeout: 180000,
|
||||
args: [
|
||||
'--use-angle=metal',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-renderer-backgrounding',
|
||||
],
|
||||
});
|
||||
try {
|
||||
for (const mode of modes) {
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
const counts = { ofm: 0, tomtom: 0 };
|
||||
page.on('request', (r) => {
|
||||
const u = new URL(r.url());
|
||||
if (
|
||||
u.hostname === 'tiles.openfreemap.org' &&
|
||||
/\/\d+\/\d+\/\d+(\.pbf)?$/.test(u.pathname)
|
||||
)
|
||||
counts.ofm++;
|
||||
if (u.pathname.startsWith('/api/tomtom/flow/')) counts.tomtom++;
|
||||
if (
|
||||
/overpass/i.test(u.hostname) ||
|
||||
u.hostname === 'nominatim.openstreetmap.org'
|
||||
)
|
||||
result.forbidden.push(u.hostname);
|
||||
});
|
||||
page.on('pageerror', (e) => result.errors.push(e.message));
|
||||
page.on('console', (m) => {
|
||||
if (m.type() === 'error') result.errors.push(m.text());
|
||||
});
|
||||
await page.evaluateOnNewDocument(() =>
|
||||
sessionStorage.setItem('gev:first-run-mission-session:v1', 'dismissed'),
|
||||
);
|
||||
const url = new URL(args.find((a) => /^https?:/.test(a)));
|
||||
url.searchParams.set('trafficRoads', mode);
|
||||
await page.goto(url.href, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.dataManager &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 120000 },
|
||||
);
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.styleManager.initialRestorePromise,
|
||||
);
|
||||
await page.keyboard.press('Escape');
|
||||
await page.evaluate(async () => {
|
||||
const g = window.__godsEyeView;
|
||||
for (const [id] of g.dataManager.layers)
|
||||
await g.dataManager.setEnabled(id, false);
|
||||
g.styleManager.setDetection({ enabled: false });
|
||||
g.viewer.scene.screenSpaceCameraController.enableCollisionDetection = false;
|
||||
});
|
||||
const snap = () =>
|
||||
page.evaluate(() => {
|
||||
const { viewer, dataManager } = window.__godsEyeView;
|
||||
let upper = 0,
|
||||
lower = 0,
|
||||
near = 0;
|
||||
const camera = viewer.camera.positionWC;
|
||||
dataManager.layers
|
||||
.get('traffic')
|
||||
.module.visitMotionDots((id, key, p) => {
|
||||
if (!p.show) return;
|
||||
const s = viewer.scene.cartesianToCanvasCoordinates(p.position);
|
||||
if (s && s.x >= 240 && s.x <= 1040 && s.y >= 0 && s.y <= 800) {
|
||||
if (s.y < 400) upper++;
|
||||
else lower++;
|
||||
}
|
||||
});
|
||||
return {
|
||||
upper,
|
||||
lower,
|
||||
...dataManager.layers.get('traffic').module.getStats(),
|
||||
};
|
||||
});
|
||||
const settle = async () => {
|
||||
await sleep(700);
|
||||
await page
|
||||
.waitForFunction(
|
||||
() => {
|
||||
const s = window.__godsEyeView.dataManager.layers
|
||||
.get('traffic')
|
||||
.module.getStats();
|
||||
return !s.loading;
|
||||
},
|
||||
{ timeout: 15000 },
|
||||
)
|
||||
.catch(() => {});
|
||||
await sleep(1200);
|
||||
return snap();
|
||||
};
|
||||
for (const [name, lat, lon, height] of [
|
||||
['dubai', 24.964, 55.037, 276],
|
||||
['austin', 30.2672, -97.7431, 300],
|
||||
['london', 51.507, -0.128, 300],
|
||||
]) {
|
||||
const before = { ...counts };
|
||||
await moveCamera(page, { lat, lon, height: 3000, pitch: -45 }, 0);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
await moveCamera(page, { lat, lon, height, pitch: -15 }, 1.4);
|
||||
const settled = await settle();
|
||||
const load = {
|
||||
ofm: counts.ofm - before.ofm,
|
||||
tomtom: counts.tomtom - before.tomtom,
|
||||
};
|
||||
await page.screenshot({ path: `${out}/${name}-${mode}.png` });
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', false),
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
const toggled = await settle();
|
||||
const oracle =
|
||||
Math.abs(settled.upper - toggled.upper) / Math.max(1, toggled.upper);
|
||||
const preZoom = { ...counts };
|
||||
await moveCamera(page, { lat, lon, height: 3000, pitch: -15 }, 0.7);
|
||||
await settle();
|
||||
const atWide = { ...counts };
|
||||
await moveCamera(page, { lat, lon, height, pitch: -15 }, 0.7);
|
||||
await settle();
|
||||
const revisit = {
|
||||
ofm: counts.ofm - atWide.ofm,
|
||||
tomtom: counts.tomtom - atWide.tomtom,
|
||||
};
|
||||
const row = {
|
||||
mode,
|
||||
name,
|
||||
load,
|
||||
settled,
|
||||
toggled,
|
||||
oracle,
|
||||
revisit,
|
||||
zoomOut: {
|
||||
ofm: atWide.ofm - preZoom.ofm,
|
||||
tomtom: atWide.tomtom - preZoom.tomtom,
|
||||
},
|
||||
};
|
||||
result.views.push(row);
|
||||
console.log(JSON.stringify(row));
|
||||
if (oracle > 0.15)
|
||||
result.failures.push(`${name}/${mode}: upper oracle ${oracle}`);
|
||||
if (revisit.ofm || revisit.tomtom)
|
||||
result.failures.push(`${name}/${mode}: revisit requests`);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', false),
|
||||
);
|
||||
}
|
||||
const locationView = { lat: 24.973, lon: 55.037 };
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat: 24.964, lon: 55.037, height: 3000, pitch: -45 },
|
||||
0,
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
const dir = `${out}/dubai-${mode}-journey`;
|
||||
const recording =
|
||||
mode === 'hybrid' ? await createScreencast(page, dir, { fps: 24 }) : null;
|
||||
await installFrameProbe(page);
|
||||
await page.evaluate(() => {
|
||||
const { viewer, dataManager } = window.__godsEyeView;
|
||||
window.__obliqueFrames = [];
|
||||
window.__obliqueMoving = true;
|
||||
viewer.scene.postRender.addEventListener(() => {
|
||||
const s = dataManager.layers.get('traffic').module.getStats();
|
||||
let upper = 0,
|
||||
lower = 0,
|
||||
near = 0;
|
||||
const camera = viewer.camera.positionWC;
|
||||
dataManager.layers
|
||||
.get('traffic')
|
||||
.module.visitMotionDots((id, key, p) => {
|
||||
if (!p.show) return;
|
||||
const q = viewer.scene.cartesianToCanvasCoordinates(p.position);
|
||||
if (q && q.x >= 240 && q.x <= 1040 && q.y >= 0 && q.y <= 800) {
|
||||
if (q.y < 400) upper++;
|
||||
else lower++;
|
||||
const dx = p.position.x - camera.x,
|
||||
dy = p.position.y - camera.y,
|
||||
dz = p.position.z - camera.z;
|
||||
if (dx * dx + dy * dy + dz * dz <= 1500 * 1500) near++;
|
||||
}
|
||||
});
|
||||
window.__obliqueFrames.push({
|
||||
t: performance.now(),
|
||||
moving: window.__obliqueMoving,
|
||||
count: s.count,
|
||||
loading: s.loading,
|
||||
upper,
|
||||
lower,
|
||||
near,
|
||||
publishes: s.motion.publishes,
|
||||
rebuilds: s.motion.rebuilds,
|
||||
});
|
||||
});
|
||||
});
|
||||
await recording?.start();
|
||||
await page.evaluate(async (v) => {
|
||||
const { flyToLandmark } = await import('/src/locations.js');
|
||||
await new Promise((resolve) =>
|
||||
flyToLandmark(window.__godsEyeView.viewer, v.lat, v.lon, {
|
||||
range: 1000,
|
||||
pitch: -15,
|
||||
heading: 0,
|
||||
duration: 1.6,
|
||||
buildingHeight: 0,
|
||||
onComplete: resolve,
|
||||
onCancel: resolve,
|
||||
}),
|
||||
);
|
||||
window.__obliqueMoving = false;
|
||||
}, locationView);
|
||||
const arrived = await settle();
|
||||
await page.evaluate(() => {
|
||||
window.__obliqueMoving = true;
|
||||
});
|
||||
await moveCamera(
|
||||
page,
|
||||
{ lat: 24.964, lon: 55.042, height: 300, pitch: -15 },
|
||||
1.2,
|
||||
);
|
||||
await page.evaluate(() => {
|
||||
window.__obliqueMoving = false;
|
||||
});
|
||||
const panned = await settle();
|
||||
await sleep(1800);
|
||||
const frames = await page.evaluate(() => window.__obliqueFrames);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', false),
|
||||
);
|
||||
await page.evaluate(() =>
|
||||
window.__godsEyeView.dataManager.setEnabled('traffic', true),
|
||||
);
|
||||
const fresh = await settle();
|
||||
const oracle =
|
||||
Math.abs(panned.upper - fresh.upper) / Math.max(1, fresh.upper);
|
||||
const stops = frames.flatMap((f, i) =>
|
||||
i && !f.moving && frames[i - 1].moving ? [i] : [],
|
||||
);
|
||||
const nearSettles = stops.map((start, index) => {
|
||||
let end = start + 1;
|
||||
while (end < frames.length && !frames[end].moving) end++;
|
||||
const phase = frames.slice(start, end),
|
||||
target = phase.at(-1).near;
|
||||
const reached = phase.find((f) => f.near >= target * 0.85);
|
||||
const ms = reached ? Math.round(reached.t - phase[0].t) : Infinity;
|
||||
if (target && ms > 1100)
|
||||
result.failures.push(
|
||||
`dubai-location-pan/${mode}/${index}: near settle ${ms} ms`,
|
||||
);
|
||||
return {
|
||||
phase: index === 0 ? 'arrival' : 'pan',
|
||||
ms,
|
||||
target,
|
||||
first: phase[0].near,
|
||||
};
|
||||
});
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(`${dir}/metrics.json`, JSON.stringify(frames));
|
||||
const clip = await recording?.stop();
|
||||
result.views.push({
|
||||
mode,
|
||||
name: 'dubai-location-pan',
|
||||
nearSettles,
|
||||
arrived,
|
||||
settled: panned,
|
||||
toggled: fresh,
|
||||
oracle,
|
||||
clip,
|
||||
});
|
||||
if (oracle > 0.15)
|
||||
result.failures.push(
|
||||
`dubai-location-pan/${mode}: upper oracle ${oracle}`,
|
||||
);
|
||||
await page.close();
|
||||
}
|
||||
} finally {
|
||||
fs.writeFileSync(`${out}/result.json`, JSON.stringify(result, null, 2));
|
||||
await browser.close();
|
||||
}
|
||||
if (
|
||||
!baseline &&
|
||||
(result.failures.length || result.errors.length || result.forbidden.length)
|
||||
)
|
||||
process.exitCode = 1;
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Check auth-mode keyboard focus with the real settings template and controls.
|
||||
* Run: node --test --test-reporter=spec scripts/qa-voice-auth-focus.mjs
|
||||
* Sign-in responses are fixtures. No credentials or live login are used.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { createServer } from 'node:http';
|
||||
import { after, before, test } from 'node:test';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const root = new URL('../', import.meta.url);
|
||||
const template = await readFile(
|
||||
new URL('src/ui/templates/provider-settings.html', root),
|
||||
'utf8',
|
||||
);
|
||||
const assets = new Set([
|
||||
'/src/keySetup.js',
|
||||
'/src/voice/cloudVoiceAuth.js',
|
||||
'/src/ui/surfaceKeyboard.js',
|
||||
'/src/ui/styles/provider-settings.css',
|
||||
]);
|
||||
const observations = [];
|
||||
let server;
|
||||
let browser;
|
||||
let origin;
|
||||
|
||||
before(async () => {
|
||||
server = createServer(async (req, res) => {
|
||||
const pathname = new URL(req.url, 'http://localhost').pathname;
|
||||
if (pathname === '/') {
|
||||
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
||||
res.end(`<!doctype html><html><head><meta charset="utf-8">
|
||||
<title>Voice auth keyboard focus test</title>
|
||||
<link rel="stylesheet" href="/src/ui/styles/provider-settings.css">
|
||||
</head><body>${template}</body></html>`);
|
||||
return;
|
||||
}
|
||||
if (!assets.has(pathname)) {
|
||||
res.statusCode = 404;
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
res.setHeader(
|
||||
'Content-Type',
|
||||
pathname.endsWith('.css') ? 'text/css' : 'text/javascript',
|
||||
);
|
||||
res.end(await readFile(new URL(pathname.slice(1), root)));
|
||||
} catch {
|
||||
res.statusCode = 500;
|
||||
res.end('Could not load the test asset');
|
||||
}
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject);
|
||||
server.listen(0, '127.0.0.1', resolve);
|
||||
});
|
||||
origin = `http://127.0.0.1:${server.address().port}`;
|
||||
const chrome = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
||||
browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath:
|
||||
process.env.PUPPETEER_EXECUTABLE_PATH ||
|
||||
(existsSync(chrome) ? chrome : await puppeteer.executablePath()),
|
||||
});
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await browser?.close();
|
||||
if (server?.listening) await new Promise((resolve) => server.close(resolve));
|
||||
const output = new URL('.gev-logs/voice-auth-focus/', root);
|
||||
await mkdir(output, { recursive: true });
|
||||
await writeFile(
|
||||
new URL('results.json', output),
|
||||
`${JSON.stringify(observations, null, 2)}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
const scenarios = [
|
||||
{
|
||||
name: 'OAuth to API key',
|
||||
initial: 'oauth',
|
||||
expected: 'api-key',
|
||||
flow: 'unused',
|
||||
},
|
||||
{
|
||||
name: 'API key to existing OAuth',
|
||||
initial: 'api-key',
|
||||
expected: 'oauth',
|
||||
flow: 'existing',
|
||||
},
|
||||
{
|
||||
name: 'API key to OAuth ready at login launch',
|
||||
initial: 'api-key',
|
||||
expected: 'oauth',
|
||||
flow: 'login-ready',
|
||||
},
|
||||
{
|
||||
name: 'API key to OAuth ready after polling',
|
||||
initial: 'api-key',
|
||||
expected: 'oauth',
|
||||
flow: 'poll-ready',
|
||||
},
|
||||
];
|
||||
|
||||
async function mountSettings(page, scenario) {
|
||||
await page.goto(`${origin}/?setup=1`, {
|
||||
waitUntil: 'domcontentloaded',
|
||||
});
|
||||
await page.evaluate(async ({ initial, flow }) => {
|
||||
localStorage.clear();
|
||||
localStorage.setItem('godsEyeView.voice.cloudAuth', initial);
|
||||
window.__focusTestCalls = [];
|
||||
let statusReads = 0;
|
||||
const fetchImpl = async (url) => {
|
||||
window.__focusTestCalls.push(url);
|
||||
let payload;
|
||||
let status = 200;
|
||||
if (url === '/api/setup/status') {
|
||||
payload = {
|
||||
setCount: 0,
|
||||
total: 1,
|
||||
keys: [
|
||||
{
|
||||
id: 'openai',
|
||||
title: 'OPENAI',
|
||||
tier: 'metered',
|
||||
set: false,
|
||||
unlocks: 'Voice control',
|
||||
envVars: ['OPENAI_API_KEY'],
|
||||
getUrl: 'https://example.invalid/',
|
||||
},
|
||||
],
|
||||
};
|
||||
} else if (url === '/api/realtime/oauth-status') {
|
||||
statusReads += 1;
|
||||
if (flow === 'deferred') {
|
||||
return await new Promise((resolve) => {
|
||||
window.__resolveOAuthStatus = (available) =>
|
||||
resolve(
|
||||
new Response(
|
||||
JSON.stringify({ available, error: 'Fixture failure' }),
|
||||
{
|
||||
status: available ? 200 : 503,
|
||||
},
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
payload = {
|
||||
available:
|
||||
flow === 'existing' || (flow === 'poll-ready' && statusReads > 1),
|
||||
};
|
||||
} else if (url === '/api/realtime/oauth-login') {
|
||||
const available = flow === 'login-ready';
|
||||
status = available ? 200 : 202;
|
||||
payload = { available, pending: !available, started: !available };
|
||||
} else {
|
||||
throw new Error(`Unexpected request: ${url}`);
|
||||
}
|
||||
return new Response(JSON.stringify(payload), { status });
|
||||
};
|
||||
const { initKeySetup } = await import('/src/keySetup.js');
|
||||
window.__focusTestSettings = await initKeySetup({ fetchImpl });
|
||||
}, scenario);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document.getElementById('key-setup').classList.contains('visible') &&
|
||||
document.activeElement.matches('input[data-env-var]'),
|
||||
);
|
||||
}
|
||||
|
||||
for (const scenario of scenarios) {
|
||||
for (const key of ['Enter', 'Space', 'Mouse click']) {
|
||||
test(`${scenario.name} keeps keyboard focus after ${key}`, async (t) => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await mountSettings(page, scenario);
|
||||
|
||||
if (key !== 'Mouse click') {
|
||||
// Tab from the initial key field to the real auth button.
|
||||
await page.keyboard.press('Tab');
|
||||
assert.equal(
|
||||
await page.evaluate(() =>
|
||||
document.activeElement.matches('[data-cloud-voice-auth-toggle]'),
|
||||
),
|
||||
true,
|
||||
'Keyboard navigation must reach the auth toggle before activation',
|
||||
);
|
||||
}
|
||||
await page.evaluate(() => {
|
||||
const toggle = document.querySelector(
|
||||
'[data-cloud-voice-auth-toggle]',
|
||||
);
|
||||
window.__originalFocusToggle = toggle;
|
||||
window.__focusAtActivation = null;
|
||||
toggle.addEventListener(
|
||||
'click',
|
||||
() => {
|
||||
window.__focusAtActivation = document.activeElement === toggle;
|
||||
},
|
||||
{ capture: true, once: true },
|
||||
);
|
||||
});
|
||||
if (key === 'Mouse click') {
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
getComputedStyle(document.getElementById('key-setup')).opacity ===
|
||||
'1',
|
||||
);
|
||||
await page.click('[data-cloud-voice-auth-toggle]');
|
||||
} else {
|
||||
await page.keyboard.press(key);
|
||||
}
|
||||
await page.waitForFunction(
|
||||
(expected) =>
|
||||
localStorage.getItem('godsEyeView.voice.cloudAuth') === expected &&
|
||||
!document.querySelector('[data-cloud-voice-auth-toggle]')
|
||||
.disabled &&
|
||||
document
|
||||
.querySelector('[data-cloud-voice-auth-toggle]')
|
||||
.getAttribute('aria-disabled') !== 'true',
|
||||
{},
|
||||
scenario.expected,
|
||||
);
|
||||
const selection = await page.evaluate(() => ({
|
||||
focusOnToggleAtActivation: window.__focusAtActivation,
|
||||
focusOnToggle:
|
||||
document.activeElement ===
|
||||
document.querySelector('[data-cloud-voice-auth-toggle]'),
|
||||
activeTag: document.activeElement.tagName,
|
||||
originalToggleConnected: window.__originalFocusToggle.isConnected,
|
||||
requests: window.__focusTestCalls,
|
||||
}));
|
||||
await page.keyboard.press('Tab');
|
||||
const nextTab = await page.evaluate(() => ({
|
||||
focusOnSave: document.activeElement.matches('[data-key-setup-apply]'),
|
||||
activeLabel:
|
||||
document.activeElement.getAttribute('aria-label') ||
|
||||
document.activeElement.textContent.trim(),
|
||||
}));
|
||||
const observed = { scenario: scenario.name, key, selection, nextTab };
|
||||
observations.push(observed);
|
||||
t.diagnostic(JSON.stringify(observed));
|
||||
assert.equal(
|
||||
selection.focusOnToggle,
|
||||
true,
|
||||
'Auth selection must keep focus on its usable toggle',
|
||||
);
|
||||
assert.equal(
|
||||
nextTab.focusOnSave,
|
||||
true,
|
||||
'Tab must continue to SAVE KEYS rather than restart at Close',
|
||||
);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const available of [true, false]) {
|
||||
test(`pending OAuth ${available ? 'success' : 'failure'} blocks repeats without moving focus or losing a key draft`, async () => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await mountSettings(page, { initial: 'api-key', flow: 'deferred' });
|
||||
await page.type('input[data-env-var]', 'test-draft-only');
|
||||
await page.keyboard.press('Tab');
|
||||
await page.keyboard.press('Enter');
|
||||
await page.waitForFunction(
|
||||
() => typeof window.__resolveOAuthStatus === 'function',
|
||||
);
|
||||
assert.equal(
|
||||
await page.evaluate(() =>
|
||||
document.activeElement.matches('[data-cloud-voice-auth-toggle]'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
await page.keyboard.press('Enter');
|
||||
await page.keyboard.press('Space');
|
||||
assert.equal(
|
||||
await page.evaluate(
|
||||
() =>
|
||||
window.__focusTestCalls.filter(
|
||||
(url) => url === '/api/realtime/oauth-status',
|
||||
).length,
|
||||
),
|
||||
1,
|
||||
);
|
||||
await page.keyboard.press('Tab');
|
||||
assert.equal(
|
||||
await page.evaluate(() =>
|
||||
document.activeElement.matches('[data-key-setup-apply]'),
|
||||
),
|
||||
true,
|
||||
);
|
||||
await page.evaluate(
|
||||
(available) => window.__resolveOAuthStatus(available),
|
||||
available,
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
document
|
||||
.querySelector('[data-cloud-voice-auth-toggle]')
|
||||
.getAttribute('aria-disabled') === 'false',
|
||||
);
|
||||
const observed = await page.evaluate(() => ({
|
||||
mode: localStorage.getItem('godsEyeView.voice.cloudAuth'),
|
||||
focusOnSave: document.activeElement.matches('[data-key-setup-apply]'),
|
||||
draft: document.querySelector('input[data-env-var]').value,
|
||||
requests: window.__focusTestCalls,
|
||||
}));
|
||||
observations.push({ scenario: 'pending OAuth', available, observed });
|
||||
assert.equal(observed.mode, available ? 'oauth' : 'api-key');
|
||||
assert.equal(
|
||||
observed.focusOnSave,
|
||||
true,
|
||||
'Completing a request must not take focus from another control',
|
||||
);
|
||||
assert.equal(observed.draft, 'test-draft-only');
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Live Realtime auth acceptance with a prerecorded Chromium microphone.
|
||||
* Run against the local dev server after configuring the selected credential:
|
||||
* node scripts/qa-voice-auth.mjs http://127.0.0.1:4189 oauth microphone.wav
|
||||
* Use a short spoken reply request with three seconds of leading silence.
|
||||
* Saves only usage, transport metadata, and screenshots; never credentials.
|
||||
*/
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import puppeteer from 'puppeteer';
|
||||
|
||||
const root = fileURLToPath(new URL('../', import.meta.url));
|
||||
const url = new URL(process.argv[2] || 'http://127.0.0.1:4189');
|
||||
const auth = process.argv[3] || 'oauth';
|
||||
assert.ok(['oauth', 'api-key'].includes(auth), 'Choose oauth or api-key');
|
||||
assert.ok(
|
||||
['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname),
|
||||
'Use a local server',
|
||||
);
|
||||
assert.ok(process.argv[4], 'Provide a prerecorded microphone WAV');
|
||||
const wav = path.resolve(process.argv[4]);
|
||||
assert.ok(fs.existsSync(wav), 'Microphone WAV must exist');
|
||||
url.searchParams.set('welcome', '0');
|
||||
url.searchParams.set('setup', '1');
|
||||
const output = path.join(root, '.gev-logs', 'voice-auth');
|
||||
fs.mkdirSync(output, { recursive: true });
|
||||
const executablePath =
|
||||
process.env.PUPPETEER_EXECUTABLE_PATH ||
|
||||
(fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome')
|
||||
? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
|
||||
: puppeteer.executablePath());
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true,
|
||||
executablePath,
|
||||
args: [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--use-fake-device-for-media-stream',
|
||||
`--use-file-for-fake-audio-capture=${wav}%noloop`,
|
||||
'--autoplay-policy=no-user-gesture-required',
|
||||
'--disable-background-timer-throttling',
|
||||
'--window-size=1440,900',
|
||||
],
|
||||
});
|
||||
const result = {
|
||||
ok: false,
|
||||
auth,
|
||||
input: 'prerecorded microphone',
|
||||
mints: [],
|
||||
negotiations: [],
|
||||
};
|
||||
let page;
|
||||
const readSession = () =>
|
||||
page.evaluate(async () => {
|
||||
const voice = window.__godsEyeView.voiceCommands;
|
||||
const stats = voice.pc ? [...(await voice.pc.getStats()).values()] : [];
|
||||
const input = stats.find(
|
||||
(entry) => entry.type === 'outbound-rtp' && entry.kind === 'audio',
|
||||
);
|
||||
const source = stats.find(
|
||||
(entry) => entry.type === 'media-source' && entry.kind === 'audio',
|
||||
);
|
||||
const audio = stats.find(
|
||||
(entry) => entry.type === 'inbound-rtp' && entry.kind === 'audio',
|
||||
);
|
||||
const meter = document.getElementById('gev-voice-cost-value');
|
||||
return {
|
||||
...window.__voiceAuthEvidence,
|
||||
connection: voice.pc?.connectionState,
|
||||
microphoneEnabled: voice.stream
|
||||
?.getAudioTracks()
|
||||
.every((track) => track.enabled),
|
||||
inputBytes: input?.bytesSent || 0,
|
||||
inputAudioEnergy: source?.totalAudioEnergy ?? null,
|
||||
outputBytes: audio?.bytesReceived || 0,
|
||||
outputAudioEnergy: audio?.totalAudioEnergy || 0,
|
||||
playbackTime: voice.audioEl?.currentTime || 0,
|
||||
meter: {
|
||||
text: meter?.textContent,
|
||||
usage: meter?.title,
|
||||
visible: Boolean(meter?.getClientRects().length),
|
||||
},
|
||||
};
|
||||
});
|
||||
try {
|
||||
await browser
|
||||
.defaultBrowserContext()
|
||||
.overridePermissions(url.origin, ['microphone']);
|
||||
page = await browser.newPage();
|
||||
await page.setViewport({ width: 1440, height: 900 });
|
||||
page.on('response', (response) => {
|
||||
const route = new URL(response.url()).pathname;
|
||||
if (route === '/api/realtime/token')
|
||||
result.mints.push({
|
||||
status: response.status(),
|
||||
auth: response.headers()['x-gev-voice-auth'],
|
||||
model: response.headers()['x-gev-voice-model'],
|
||||
});
|
||||
if (
|
||||
route === '/v1/realtime/calls' &&
|
||||
response.request().method() === 'POST'
|
||||
)
|
||||
result.negotiations.push({ status: response.status() });
|
||||
});
|
||||
// Observe the application's real transport. Do not replace fetch or replies.
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
window.__voiceAuthEvidence = {
|
||||
speechStopped: 0,
|
||||
transcripts: [],
|
||||
replies: [],
|
||||
responses: [],
|
||||
errors: [],
|
||||
eventCounts: {},
|
||||
};
|
||||
const create = RTCPeerConnection.prototype.createDataChannel;
|
||||
RTCPeerConnection.prototype.createDataChannel = function (...args) {
|
||||
const channel = create.apply(this, args);
|
||||
channel.addEventListener('message', ({ data }) => {
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(data);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const evidence = window.__voiceAuthEvidence;
|
||||
evidence.eventCounts[event.type] =
|
||||
(evidence.eventCounts[event.type] || 0) + 1;
|
||||
if (event.type === 'input_audio_buffer.speech_stopped')
|
||||
evidence.speechStopped += 1;
|
||||
if (
|
||||
event.type === 'conversation.item.input_audio_transcription.completed'
|
||||
)
|
||||
evidence.transcripts.push(event.transcript);
|
||||
if (event.type === 'response.output_audio_transcript.done')
|
||||
evidence.replies.push(event.transcript);
|
||||
if (event.type === 'response.done')
|
||||
evidence.responses.push({
|
||||
status: event.response?.status,
|
||||
inputTokens: event.response?.usage?.input_tokens,
|
||||
outputTokens: event.response?.usage?.output_tokens,
|
||||
});
|
||||
if (event.type === 'error')
|
||||
evidence.errors.push({ code: event.error?.code });
|
||||
});
|
||||
return channel;
|
||||
};
|
||||
});
|
||||
await page.goto(url.href, { waitUntil: 'domcontentloaded', timeout: 30_000 });
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView?.voiceCommands &&
|
||||
document.getElementById('loading-screen')?.classList.contains('hidden'),
|
||||
{ timeout: 45_000 },
|
||||
);
|
||||
await page.waitForSelector('[data-cloud-voice-auth-toggle]', {
|
||||
visible: true,
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
getComputedStyle(document.getElementById('key-setup')).opacity === '1',
|
||||
);
|
||||
const selected = await page.evaluate(
|
||||
() => localStorage.getItem('godsEyeView.voice.cloudAuth') || 'api-key',
|
||||
);
|
||||
if (selected !== auth) {
|
||||
await page.click('[data-cloud-voice-auth-toggle]');
|
||||
await page.waitForFunction(
|
||||
(mode) => localStorage.getItem('godsEyeView.voice.cloudAuth') === mode,
|
||||
{ timeout: 125_000 },
|
||||
auth,
|
||||
);
|
||||
}
|
||||
await page.click('[data-key-setup-close]');
|
||||
await page.click('#gev-voice-button');
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.voiceCommands.dc?.readyState === 'open' ||
|
||||
window.__godsEyeView.voiceCommands.status === 'error',
|
||||
{ timeout: 35_000 },
|
||||
);
|
||||
assert.equal(
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.voiceCommands.dc?.readyState,
|
||||
),
|
||||
'open',
|
||||
'Realtime channel must open',
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__voiceAuthEvidence.speechStopped > 0 &&
|
||||
window.__voiceAuthEvidence.responses.some(
|
||||
(response) =>
|
||||
response.status === 'completed' && response.outputTokens > 0,
|
||||
),
|
||||
{ timeout: 45_000 },
|
||||
);
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__voiceAuthEvidence.eventCounts['output_audio_buffer.stopped'] > 0,
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
result.session = await readSession();
|
||||
assert.ok(
|
||||
result.session.inputBytes > 0 && result.session.outputBytes > 0,
|
||||
'Audio must flow in both directions',
|
||||
);
|
||||
assert.ok(
|
||||
result.session.outputAudioEnergy > 0 && result.session.playbackTime > 0,
|
||||
'Reply audio must play',
|
||||
);
|
||||
assert.equal(
|
||||
result.session.meter.text === 'COST UNKNOWN',
|
||||
auth === 'oauth',
|
||||
'Use the correct accounting display',
|
||||
);
|
||||
await page.screenshot({ path: path.join(output, `${auth}-usage.png`) });
|
||||
await page.click('#key-setup-chip');
|
||||
await page.waitForSelector('[data-cloud-voice-auth-toggle]', {
|
||||
visible: true,
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
getComputedStyle(document.getElementById('key-setup')).opacity === '1',
|
||||
);
|
||||
await page.screenshot({ path: path.join(output, `${auth}-settings.png`) });
|
||||
await page.click('[data-key-setup-close]');
|
||||
result.stopped = await page.evaluate(() => {
|
||||
const voice = window.__godsEyeView.voiceCommands;
|
||||
const pc = voice.pc;
|
||||
const tracks = voice.stream?.getTracks() || [];
|
||||
voice.session.stop();
|
||||
return (
|
||||
pc.connectionState === 'closed' &&
|
||||
tracks.every((track) => track.readyState === 'ended')
|
||||
);
|
||||
});
|
||||
assert.ok(result.stopped, 'Stop must close the peer and microphone tracks');
|
||||
await page.click('#gev-voice-button');
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__godsEyeView.voiceCommands.dc?.readyState === 'open' ||
|
||||
window.__godsEyeView.voiceCommands.status === 'error',
|
||||
{ timeout: 35_000 },
|
||||
);
|
||||
assert.equal(
|
||||
await page.evaluate(
|
||||
() => window.__godsEyeView.voiceCommands.dc?.readyState,
|
||||
),
|
||||
'open',
|
||||
'Reconnect must work',
|
||||
);
|
||||
assert.equal(
|
||||
result.mints.length,
|
||||
2,
|
||||
'Reconnect must request a fresh client secret',
|
||||
);
|
||||
assert.ok(
|
||||
result.mints.every(
|
||||
(mint) =>
|
||||
mint.status === 200 &&
|
||||
mint.auth === (auth === 'oauth' ? 'codex-oauth' : 'env'),
|
||||
),
|
||||
);
|
||||
assert.ok(result.negotiations.every((entry) => entry.status === 201));
|
||||
result.ok = true;
|
||||
} catch (error) {
|
||||
result.error = error.message;
|
||||
if (page) {
|
||||
result.session = await readSession().catch(() => null);
|
||||
result.detail = await page
|
||||
.$eval('#gev-voice-detail', (element) => element.textContent)
|
||||
.catch(() => null);
|
||||
await page
|
||||
.screenshot({ path: path.join(output, `${auth}-failure.png`) })
|
||||
.catch(() => {});
|
||||
}
|
||||
} finally {
|
||||
await page
|
||||
?.evaluate(() => window.__godsEyeView?.voiceCommands?.session?.stop())
|
||||
.catch(() => {});
|
||||
await browser.close();
|
||||
fs.writeFileSync(
|
||||
path.join(output, `${auth}-result.json`),
|
||||
`${JSON.stringify(result, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (!result.ok) process.exitCode = 1;
|
||||
@@ -0,0 +1,411 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* qa-voice-bench.mjs — provider-neutral TEXT-mode voice-routing benchmark.
|
||||
*
|
||||
* Every provider gets the same prompt: production's realtimeInstructions()
|
||||
* and GEV_REALTIME_TOOLS (the Realtime adapter uses the exact session body the
|
||||
* token handler mints). Two suites, reported separately:
|
||||
* core the qa-voice-routing phrase table, graded with its semantics
|
||||
* (plus benchmark-only typed-argument checks)
|
||||
* coverage the newer-layers coverage probe, graded correct /
|
||||
* honest-refusal / hallucinated-capability / wrong-tool
|
||||
* dialogue two-turn dialogues in one context; turn 2 depends on turn 1
|
||||
* capability area handles / imagery / OSM / point-and-ask phrases (and two
|
||||
* dialogues) for the tools added after 4b56d0e9
|
||||
* Coverage rows are graded twice: v1 (original rubric, 4b56d0e9 tool schema)
|
||||
* and v2 (rubric for the current tool surface).
|
||||
*
|
||||
* Methodology: a fresh context per phrase (new Realtime/Live session, or a
|
||||
* stateless chat request); stub tool results; continuations until the model
|
||||
* stops calling tools (max 4). No app page is involved.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/qa-voice-bench.mjs --provider openai-realtime --model gpt-realtime-2 --reps 2
|
||||
* node scripts/qa-voice-bench.mjs --provider gemini-live --model gemini-3.8-live
|
||||
* node scripts/qa-voice-bench.mjs --provider ollama --model gemma4:e4b
|
||||
* node scripts/qa-voice-bench.mjs --provider openai-chat --base http://localhost:1234/v1 --model <id>
|
||||
* node scripts/qa-voice-bench.mjs --summarize --out qa-shots/voice-bench/<run>
|
||||
* Common flags: --out <dir> (reuse one dir to collect several models),
|
||||
* --suite core,coverage --only <substr> --budget <usd, cloud total>
|
||||
* --variant <label> --temperature <t> --num-ctx <n> --think --note <text>
|
||||
* --reps <n> --start-rep <k> (add repetitions k..k+n-1 to an existing run)
|
||||
* --replace (supersede earlier runs of this model for the suites run)
|
||||
* --compare <run dir> (add a before → after table against another run)
|
||||
*
|
||||
* Output (gitignored): <out>/results.jsonl, summary.json, summary.md.
|
||||
* Keys come from the environment or the macOS Keychain at runtime only.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import {
|
||||
CORE_PHRASES, COVERAGE_PROBE, DIALOGUES, CAPABILITY_EXTRAS,
|
||||
dialogueResult, coverageRubric,
|
||||
} from './voice-bench/phrases.mjs';
|
||||
import { gradeCore, gradeCoverage, gradeDialogue, expectLabel } from './voice-bench/grade.mjs';
|
||||
import { stubResult } from './voice-bench/stub.mjs';
|
||||
import { summarize, renderMarkdown, renderComparison } from './voice-bench/report.mjs';
|
||||
import { GEV_REALTIME_TOOLS, realtimeInstructions } from './voice-bench/productionConfig.mjs';
|
||||
import { chatNormalizationDiff } from './voice-bench/toolFormats.mjs';
|
||||
import { createOpenAiRealtimeAdapter } from './voice-bench/adapters/openaiRealtime.mjs';
|
||||
import { createGeminiLiveAdapter } from './voice-bench/adapters/geminiLive.mjs';
|
||||
import { createOpenAiChatAdapter, createOllamaAdapter } from './voice-bench/adapters/localChat.mjs';
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const argv = process.argv.slice(2);
|
||||
const opt = (flag, fallback) => {
|
||||
const i = argv.indexOf(flag);
|
||||
return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : fallback;
|
||||
};
|
||||
const has = (flag) => argv.includes(flag);
|
||||
|
||||
const PROVIDER = opt('--provider', null);
|
||||
const MODEL = opt('--model', null);
|
||||
const CLOUD = PROVIDER === 'openai-realtime' || PROVIDER === 'gemini-live';
|
||||
const REPS = Number(opt('--reps', CLOUD ? '2' : '1'));
|
||||
const START_REP = Number(opt('--start-rep', '1'));
|
||||
const SUITES = opt('--suite', 'core,coverage,dialogue,capability').split(',');
|
||||
const ONLY = opt('--only', null);
|
||||
const BUDGET = Number(opt('--budget', '30'));
|
||||
const VARIANT = opt('--variant', null);
|
||||
const OUT = path.resolve(ROOT, opt('--out', path.join('qa-shots', 'voice-bench', new Date().toISOString().replace(/[:.]/g, '-'))));
|
||||
const LEDGER = path.join(ROOT, 'qa-shots', 'voice-bench', 'cloud-spend.json');
|
||||
|
||||
fs.mkdirSync(OUT, { recursive: true });
|
||||
const RESULTS = path.join(OUT, 'results.jsonl');
|
||||
const NOTES = path.join(OUT, 'notes.json');
|
||||
|
||||
function readRows() {
|
||||
if (!fs.existsSync(RESULTS)) return [];
|
||||
// Several runs may append to one results file (a cloud run beside a local
|
||||
// one); skip a line another process is still writing.
|
||||
return fs.readFileSync(RESULTS, 'utf8').split('\n').filter(Boolean).flatMap((l) => {
|
||||
try { return [JSON.parse(l)]; } catch { return []; }
|
||||
});
|
||||
}
|
||||
function appendRow(row) {
|
||||
fs.appendFileSync(RESULTS, `${JSON.stringify(row)}\n`);
|
||||
}
|
||||
function readNotes() {
|
||||
try { return JSON.parse(fs.readFileSync(NOTES, 'utf8')); } catch { return []; }
|
||||
}
|
||||
/**
|
||||
* Re-grade stored turns with the CURRENT phrase tables and grader, so a
|
||||
* grader fix applies to every model already in the results file. The raw
|
||||
* calls and text in results.jsonl are the source of truth; `gradeAtRun`
|
||||
* keeps the verdict recorded at run time.
|
||||
*/
|
||||
function regrade(rows) {
|
||||
const core = new Map(CORE_PHRASES.map((p) => [p.phrase, p]));
|
||||
const cov = new Map(COVERAGE_PROBE.map((c) => [c.id, c]));
|
||||
const dia = new Map(DIALOGUES.map((d) => [d.id, d]));
|
||||
const cap = new Map(CAPABILITY_ITEMS.map((c) => [c.id, c]));
|
||||
// A run started with --replace supersedes earlier runs of the same
|
||||
// provider/model/variant for the suites it covers.
|
||||
const key = (r) => `${r.provider}:${r.model}#${r.variant || ''}`;
|
||||
const runs = rows.filter((r) => r.type === 'run');
|
||||
const superseded = new Set();
|
||||
for (const r of runs.filter((x) => x.replace)) {
|
||||
for (const older of runs) {
|
||||
if (older.at < r.at && key(older) === key(r)) superseded.add(`${older.runId}|${(r.suites || []).join(',')}`);
|
||||
}
|
||||
}
|
||||
const isSuperseded = (row) => [...superseded].some((s) => {
|
||||
const [runId, suites] = s.split('|');
|
||||
return runId === row.runId && suites.split(',').includes(row.suite);
|
||||
});
|
||||
return rows.filter((row) => row.type !== 'turn' || !isSuperseded(row)).map((row) => {
|
||||
if (row.type !== 'turn') return row;
|
||||
if (row.suite === 'dialogue' && dia.has(row.phraseId)) {
|
||||
return { ...row, gradeAtRun: row.grade, grade: gradeDialogue(dia.get(row.phraseId), [row.turn1?.calls || [], row.calls || []]) };
|
||||
}
|
||||
if (row.suite === 'core' && core.has(row.phraseId)) {
|
||||
return { ...row, gradeAtRun: row.grade, grade: gradeCore(core.get(row.phraseId), row.calls || []) };
|
||||
}
|
||||
if (row.suite === 'coverage' && cov.has(row.phraseId)) {
|
||||
const item = cov.get(row.phraseId);
|
||||
return {
|
||||
...row,
|
||||
gradeAtRun: row.grade,
|
||||
grade: gradeCoverage(item, row.calls || [], row.text, BASELINE_TOOLS),
|
||||
gradeV2: gradeCoverage(coverageRubric(item, 'v2'), row.calls || [], row.text, GEV_REALTIME_TOOLS),
|
||||
};
|
||||
}
|
||||
if (row.suite === 'capability' && cap.has(row.phraseId)) {
|
||||
const c = cap.get(row.phraseId);
|
||||
const grade = c.dialogue ? gradeDialogue(c.dialogue, [row.turn1?.calls || [], row.calls || []]) : gradeCore(c.phrase, row.calls || []);
|
||||
return { ...row, gradeAtRun: row.grade, grade };
|
||||
}
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
function readRowsFrom(dir) {
|
||||
const file = path.join(dir, 'results.jsonl');
|
||||
if (!fs.existsSync(file)) return [];
|
||||
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((l) => {
|
||||
try { return [JSON.parse(l)]; } catch { return []; }
|
||||
});
|
||||
}
|
||||
|
||||
function writeSummary() {
|
||||
const summary = summarize(regrade(readRows()));
|
||||
// --compare <dir> (remembered in compare-with.txt): a before → after table
|
||||
// against another run folder, both re-graded with the current grader.
|
||||
const compareFile = path.join(OUT, 'compare-with.txt');
|
||||
if (has('--compare')) fs.writeFileSync(compareFile, `${path.resolve(ROOT, opt('--compare', ''))}\n`);
|
||||
let comparison = '';
|
||||
if (fs.existsSync(compareFile)) {
|
||||
const beforeDir = fs.readFileSync(compareFile, 'utf8').trim();
|
||||
const before = summarize(regrade(readRowsFrom(beforeDir)));
|
||||
const beforeHead = readRowsFrom(beforeDir).find((r) => r.type === 'run' && r.prompt?.gitHead)?.prompt?.gitHead || '4b56d0e9';
|
||||
const afterHead = readRows().find((r) => r.type === 'run' && r.prompt?.gitHead)?.prompt?.gitHead || '?';
|
||||
comparison = renderComparison(before, summary, {
|
||||
beforeLabel: `${path.relative(ROOT, beforeDir)} (prompt/tools @ ${beforeHead})`,
|
||||
afterLabel: `${path.relative(ROOT, OUT)} (prompt/tools @ ${afterHead})`,
|
||||
});
|
||||
summary.comparison = { beforeDir: path.relative(ROOT, beforeDir), before: before.models.map((m) => ({ key: m.key, coreFull: m.core.fullCallAccuracyPct, coverageV1Pct: m.coverage.acceptablePct, hcV1: m.coverage.counts['hallucinated-capability'], dialogueT2: m.dialogue.turn2FullPct, toolCallP50: m.latencyMs.toolCallP50, promptTokens: m.warmup?.promptTokens ?? m.tokens.inputPerRequestMean })) };
|
||||
fs.writeFileSync(path.join(OUT, 'comparison.md'), `${comparison}\n`);
|
||||
}
|
||||
fs.writeFileSync(path.join(OUT, 'summary.json'), JSON.stringify(summary, null, 2));
|
||||
const md = renderMarkdown(summary, { notes: readNotes(), payload: payloadChanges() });
|
||||
fs.writeFileSync(path.join(OUT, 'summary.md'), comparison ? md.replace('## Routing accuracy', `${comparison}\n## Routing accuracy`) : md);
|
||||
return summary;
|
||||
}
|
||||
/** Exactly what each provider receives differently from production's payload. */
|
||||
function payloadChanges() {
|
||||
const chat = chatNormalizationDiff(GEV_REALTIME_TOOLS)
|
||||
.map((c) => `\`${c.path}\`: ${JSON.stringify(c.from)} → ${JSON.stringify(c.to)}`);
|
||||
return [
|
||||
`openai-realtime: none — the session body is captured from production's token handler (${GEV_REALTIME_TOOLS.length} tools, instructions, tool_choice auto, reasoning low, truncation). Only difference from a mic session: response.create asks for output_modalities ["text"] (as qa-voice-routing does).`,
|
||||
'gemini-live: same instructions text as systemInstruction; the same tool schemas passed verbatim as functionDeclarations[].parametersJsonSchema (no keyword changes); tool_choice has no Live equivalent (AUTO is the default); responseModalities AUDIO + outputAudioTranscription (TEXT is rejected by the model).',
|
||||
`ollama / openai-chat: same instructions as the system message; tools wrapped as {type:"function",function:{name,description,parameters}} with ${chat.length} schema change(s): ${chat.join('; ') || 'none'}. Descriptions, enums and required lists are unchanged; nothing is stringified.`,
|
||||
];
|
||||
}
|
||||
|
||||
function ledgerSpent() {
|
||||
try { return JSON.parse(fs.readFileSync(LEDGER, 'utf8')).spentUsd || 0; } catch { return 0; }
|
||||
}
|
||||
function ledgerAdd(usd) {
|
||||
const spent = ledgerSpent() + usd;
|
||||
fs.mkdirSync(path.dirname(LEDGER), { recursive: true });
|
||||
fs.writeFileSync(LEDGER, JSON.stringify({ spentUsd: spent, updatedAt: new Date().toISOString() }));
|
||||
return spent;
|
||||
}
|
||||
const sh = (cmd, args) => {
|
||||
try { return execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 10_000 }).trim(); } catch { return null; }
|
||||
};
|
||||
|
||||
/** Snapshot of what else is using the machine (GPU-heavy servers, load). */
|
||||
function machineSnapshot() {
|
||||
const load = os.loadavg().map((x) => x.toFixed(2)).join(' ');
|
||||
const ollamaPs = sh('ollama', ['ps']);
|
||||
const procs = sh('ps', ['-Ao', 'pcpu=,rss=,comm=']) || '';
|
||||
const heavy = procs.split('\n').map((l) => l.trim().split(/\s+/)).filter((p) => Number(p[0]) > 20)
|
||||
.map((p) => `${path.basename(p.slice(2).join(' '))} ${p[0]}%cpu`).slice(0, 6);
|
||||
const gpuServers = procs.split('\n').filter((l) => /ollama runner|llama-server|LM Studio|lms|mlx|whisper/i.test(l))
|
||||
.map((l) => path.basename(l.trim().split(/\s+/).slice(2).join(' '))).slice(0, 6);
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
loadavg: load,
|
||||
ollamaPs,
|
||||
busyProcesses: heavy,
|
||||
modelServers: gpuServers,
|
||||
summary: `load ${load}; busy: ${heavy.join(', ') || 'none'}`,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Ollama lifecycle ────────────────────────────────────────────────────────
|
||||
|
||||
const OLLAMA = opt('--base', 'http://localhost:11434').replace(/\/v1\/?$/, '');
|
||||
|
||||
async function ollamaUnloadAll() {
|
||||
const ps = await fetch(`${OLLAMA}/api/ps`).then((r) => r.json()).catch(() => ({ models: [] }));
|
||||
for (const m of ps.models || []) {
|
||||
await fetch(`${OLLAMA}/api/generate`, { method: 'POST', body: JSON.stringify({ model: m.name, keep_alive: 0 }) }).then((r) => r.text()).catch(() => null);
|
||||
}
|
||||
return (ps.models || []).map((m) => m.name);
|
||||
}
|
||||
|
||||
async function ollamaLoad(model, numCtx) {
|
||||
const t = Date.now();
|
||||
const res = await fetch(`${OLLAMA}/api/generate`, {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ model, keep_alive: '30m', options: { num_ctx: numCtx } }),
|
||||
signal: AbortSignal.timeout(600_000),
|
||||
});
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok || body.error) throw new Error(`load failed: ${body.error || res.status}`);
|
||||
return { wallMs: Date.now() - t, loadMs: body.load_duration ? body.load_duration / 1e6 : null };
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function buildAdapter() {
|
||||
const temperature = Number(opt('--temperature', '0.8'));
|
||||
switch (PROVIDER) {
|
||||
case 'openai-realtime':
|
||||
return createOpenAiRealtimeAdapter({ model: MODEL });
|
||||
case 'gemini-live':
|
||||
return createGeminiLiveAdapter({ model: MODEL, schemaMode: opt('--gemini-schema', 'jsonSchema') });
|
||||
case 'ollama':
|
||||
return createOllamaAdapter({ model: MODEL, baseUrl: OLLAMA, temperature, numCtx: Number(opt('--num-ctx', '32768')), think: has('--think') });
|
||||
case 'openai-chat':
|
||||
return createOpenAiChatAdapter({ model: MODEL, baseUrl: opt('--base', 'http://localhost:1234/v1'), temperature });
|
||||
default:
|
||||
throw new Error(`unknown --provider ${PROVIDER}`);
|
||||
}
|
||||
}
|
||||
|
||||
const COLD_PHRASE = 'Take me to Tokyo';
|
||||
|
||||
/** Capability suite: single phrases (optional pointer context) and dialogues. */
|
||||
const CAPABILITY_ITEMS = CAPABILITY_EXTRAS.map((c) => (c.turns
|
||||
? { suite: 'capability', id: c.id, text: c.turns.join(' → '), dialogue: c }
|
||||
: { suite: 'capability', id: c.context ? `${c.phrase} [pointer]` : c.phrase, text: c.phrase, phrase: c, context: c.context || null }));
|
||||
|
||||
/** The 4b56d0e9 tool schema, for grading coverage by the original yardstick. */
|
||||
const BASELINE_TOOLS = JSON.parse(fs.readFileSync(path.join(ROOT, 'scripts', 'voice-bench', 'baseline-tools-4b56d0e9.json'), 'utf8'));
|
||||
|
||||
/** What prompt this run actually sent (so before/after runs are identifiable). */
|
||||
function promptVersion() {
|
||||
let head = null;
|
||||
try { head = execFileSync('git', ['-C', ROOT, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { /* not a checkout */ }
|
||||
return {
|
||||
gitHead: head,
|
||||
tools: GEV_REALTIME_TOOLS.length,
|
||||
toolJsonChars: JSON.stringify(GEV_REALTIME_TOOLS).length,
|
||||
instructionsChars: realtimeInstructions().length,
|
||||
};
|
||||
}
|
||||
|
||||
function workList() {
|
||||
const items = [];
|
||||
if (SUITES.includes('core')) {
|
||||
for (const p of CORE_PHRASES) items.push({ suite: 'core', id: p.phrase, text: p.phrase, phrase: p });
|
||||
}
|
||||
if (SUITES.includes('coverage')) {
|
||||
for (const c of COVERAGE_PROBE) items.push({ suite: 'coverage', id: c.id, text: c.utterance, item: c });
|
||||
}
|
||||
if (SUITES.includes('dialogue')) {
|
||||
for (const d of DIALOGUES) items.push({ suite: 'dialogue', id: d.id, text: d.turns.join(' → '), dialogue: d });
|
||||
}
|
||||
if (SUITES.includes('capability')) items.push(...CAPABILITY_ITEMS);
|
||||
return items.filter((w) => !ONLY || w.text.toLowerCase().includes(ONLY.toLowerCase()));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (has('--note')) {
|
||||
const notes = readNotes();
|
||||
notes.push(opt('--note', ''));
|
||||
fs.writeFileSync(NOTES, JSON.stringify(notes, null, 2));
|
||||
}
|
||||
if (has('--summarize') || !PROVIDER) {
|
||||
writeSummary();
|
||||
console.log(`summary written to ${path.join(OUT, 'summary.md')}`);
|
||||
return;
|
||||
}
|
||||
if (!MODEL) throw new Error('--model is required');
|
||||
const adapter = buildAdapter();
|
||||
await adapter.open();
|
||||
|
||||
let warmup = null;
|
||||
if (PROVIDER === 'ollama') {
|
||||
const unloaded = await ollamaUnloadAll();
|
||||
const numCtx = Number(opt('--num-ctx', '32768'));
|
||||
const load = await ollamaLoad(MODEL, numCtx);
|
||||
// COLD turn: the first request after load pays the full-prompt prefill.
|
||||
// It is recorded separately (warmup.cold*) and primes the prefix cache
|
||||
// the way a live session's first turn would; every benchmark turn after
|
||||
// it is steady state.
|
||||
const w = await adapter.runTurn({ text: COLD_PHRASE, onCall: () => ({ ok: true }) });
|
||||
warmup = { unloaded, loadWallMs: load.wallMs, loadMs: load.loadMs, coldPhrase: COLD_PHRASE, prefillTokS: w.local.prefillTokS, promptTokens: w.local.promptEvalCount, coldFirstTokenMs: w.tFirstTokenMs, coldToolCallMs: w.tFirstCallMs, coldCalls: w.calls.map((c) => c.name), warmupTurnMs: w.tDoneMs, errors: w.errors };
|
||||
console.log(`warm-up: load ${Math.round(load.wallMs)}ms, prefill ${Math.round(w.local.prefillTokS || 0)} tok/s over ${w.local.promptEvalCount} tokens`);
|
||||
if (w.errors.length) console.log(` warm-up errors: ${w.errors.join('; ')}`);
|
||||
} else if (PROVIDER === 'openai-chat') {
|
||||
const w = await adapter.runTurn({ text: COLD_PHRASE, onCall: () => ({ ok: true }) });
|
||||
warmup = { loadMs: Number(opt('--load-ms', 'NaN')) || null, coldPhrase: COLD_PHRASE, prefillTokS: w.local.prefillTokS, promptTokens: w.local.promptEvalCount, coldFirstTokenMs: w.tFirstTokenMs, coldToolCallMs: w.tFirstCallMs, coldCalls: w.calls.map((c) => c.name), warmupTurnMs: w.tDoneMs, errors: w.errors };
|
||||
console.log(`warm-up: ${w.tDoneMs}ms prefill≈${Math.round(w.local.prefillTokS || 0)} tok/s`);
|
||||
if (w.errors.length) throw new Error(`warm-up failed, not benchmarking: ${w.errors[0]}`);
|
||||
}
|
||||
|
||||
const runId = `${Date.now().toString(36)}`;
|
||||
const machine = CLOUD ? null : machineSnapshot();
|
||||
appendRow({ type: 'run', runId, at: new Date().toISOString(), provider: PROVIDER, model: MODEL, variant: VARIANT, reps: REPS, suites: SUITES, settings: adapter.settings, machine, warmup, replace: has('--replace'), prompt: promptVersion(), methodology: 'fresh context per phrase/dialogue; stub tool results (canned results for dialogue turn 1); ≤4 continuations' });
|
||||
|
||||
const items = workList();
|
||||
let stopReason = null;
|
||||
let turnCosts = [];
|
||||
outer: for (let rep = START_REP; rep < START_REP + REPS; rep += 1) {
|
||||
console.log(`\n== ${PROVIDER} ${MODEL}${VARIANT ? ` (${VARIANT})` : ''} — rep ${rep}/${REPS}, ${items.length} phrases`);
|
||||
for (const w of items) {
|
||||
if (CLOUD) {
|
||||
const avg = turnCosts.length ? turnCosts.reduce((a, b) => a + b, 0) / turnCosts.length : 0.1;
|
||||
if (ledgerSpent() + avg > BUDGET) { stopReason = `cloud budget $${BUDGET} reached (ledger $${ledgerSpent().toFixed(2)})`; break outer; }
|
||||
}
|
||||
let r;
|
||||
let turn1 = null;
|
||||
let grade;
|
||||
if (w.dialogue) {
|
||||
const rs = await adapter.runDialogue({
|
||||
turns: w.dialogue.turns,
|
||||
onCall: (c, turn) => dialogueResult(w.dialogue, c.name, turn) ?? stubResult('core', c, GEV_REALTIME_TOOLS),
|
||||
});
|
||||
const [a, b] = rs;
|
||||
turn1 = { text: a.text.slice(0, 400), calls: a.calls, tFirstCallMs: a.tFirstCallMs, tDoneMs: a.tDoneMs, errors: a.errors };
|
||||
r = {
|
||||
...b,
|
||||
errors: [...a.errors, ...b.errors],
|
||||
timeout: a.timeout || b.timeout,
|
||||
costUsd: a.costUsd + b.costUsd,
|
||||
usage: Object.fromEntries(Object.keys(b.usage).map((k) => [k, (a.usage[k] || 0) + (b.usage[k] || 0)])),
|
||||
setupMs: a.setupMs,
|
||||
};
|
||||
grade = gradeDialogue(w.dialogue, [a.calls, b.calls]);
|
||||
} else {
|
||||
r = await adapter.runTurn({ text: w.text, context: w.context || undefined, onCall: (c) => stubResult(w.suite, c, GEV_REALTIME_TOOLS) });
|
||||
grade = w.suite === 'coverage' ? gradeCoverage(w.item, r.calls, r.text, GEV_REALTIME_TOOLS) : gradeCore(w.phrase, r.calls);
|
||||
}
|
||||
const row = {
|
||||
type: 'turn', runId, provider: PROVIDER, model: MODEL, variant: VARIANT, cloud: CLOUD, rep, suite: w.suite,
|
||||
phraseId: w.id, phrase: w.text,
|
||||
expect: w.dialogue ? `t1 ${expectLabel(w.dialogue.first)} → t2 ${expectLabel(w.dialogue.second)}` : w.suite === 'coverage' ? w.item.expect : expectLabel(w.phrase),
|
||||
context: w.context || undefined,
|
||||
expectNone: Boolean(w.phrase?.expectNone), supported: w.item ? w.item.supported : undefined,
|
||||
calls: r.calls, text: r.text.slice(0, 800), grade, turn1: turn1 || undefined,
|
||||
tFirstTokenMs: r.tFirstTokenMs, tFirstCallMs: r.tFirstCallMs, tDoneMs: r.tDoneMs, setupMs: r.setupMs,
|
||||
responses: r.responses, usage: r.usage, costUsd: r.costUsd, errors: r.errors, timeout: r.timeout,
|
||||
local: r.local || undefined,
|
||||
at: new Date().toISOString(),
|
||||
};
|
||||
appendRow(row);
|
||||
if (CLOUD) { ledgerAdd(r.costUsd); turnCosts.push(r.costUsd); }
|
||||
const verdict = w.suite === 'coverage' ? grade.verdict : (grade.ok ? 'PASS' : 'FAIL');
|
||||
console.log(` [${verdict}] ${w.text} → ${turn1 ? `${turn1.calls.map((c) => c.name).join(',') || '(none)'} ⇒ ` : ''}${r.calls.map((c) => c.name).join(',') || '(none)'} tool@${r.tFirstCallMs ?? '—'}ms done@${r.tDoneMs}ms $${r.costUsd.toFixed(4)}${r.errors.length ? ` ERR ${r.errors[0]}` : ''}${r.timeout ? ' TIMEOUT' : ''}`);
|
||||
if (r.errors.some((e) => /HTTP 4\d\d|close 100[78]|invalid|not found|model_not_found|does not exist/i.test(e)) && r.calls.length === 0 && !r.text) {
|
||||
const same = readRows().filter((x) => x.type === 'turn' && x.runId === runId).slice(-3);
|
||||
if (same.length === 3 && same.every((x) => x.errors?.length && !x.calls.length)) { stopReason = `3 consecutive hard errors: ${r.errors[0]}`; break outer; }
|
||||
}
|
||||
}
|
||||
writeSummary();
|
||||
}
|
||||
await adapter.close();
|
||||
if (PROVIDER === 'ollama' && !has('--keep-loaded')) {
|
||||
await fetch(`${OLLAMA}/api/generate`, { method: 'POST', body: JSON.stringify({ model: MODEL, keep_alive: 0 }) }).catch(() => null);
|
||||
}
|
||||
if (stopReason) {
|
||||
console.log(`STOPPED: ${stopReason}`);
|
||||
appendRow({ type: 'stop', runId, provider: PROVIDER, model: MODEL, reason: stopReason, at: new Date().toISOString() });
|
||||
}
|
||||
const summary = writeSummary();
|
||||
const mine = summary.models.find((m) => m.provider === PROVIDER && m.model === MODEL && (m.variant || null) === (VARIANT || null));
|
||||
if (mine) console.log(`\n${mine.key}: core ${mine.core.accuracyPct}% | coverage acceptable ${mine.coverage.acceptablePct}% | p50 tool ${mine.latencyMs.toolCallP50}ms | $${mine.costUsd}`);
|
||||
console.log(`results: ${RESULTS}\nsummary: ${path.join(OUT, 'summary.md')}\ncloud ledger: $${ledgerSpent().toFixed(3)}`);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
+1667
-361
File diff suppressed because it is too large.
Load diff
@@ -34,6 +34,7 @@ export const CREDENTIALS = Object.freeze([
|
||||
)),
|
||||
},
|
||||
{ name: 'LL2_API_TOKEN', label: 'Launch Library 2', keychain: [] },
|
||||
{ name: 'MAPILLARY_CLIENT_TOKEN', label: 'Mapillary street level', keychain: [['mapillary', 'client-token'], ['mapillary', 'token']] },
|
||||
]);
|
||||
|
||||
export function isConfiguredValue(value) {
|
||||
@@ -175,7 +176,9 @@ export function buildCapabilitySummary(
|
||||
? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available'
|
||||
: 'Esri World Imagery (keyless satellite basemap) with keyless terrain',
|
||||
flights,
|
||||
voice: configured('OPENAI_API_KEY') ? 'available' : 'off until an OpenAI key is added',
|
||||
voice: configured('OPENAI_API_KEY')
|
||||
? 'available'
|
||||
: 'off until an OpenAI key is added, or a Codex ChatGPT sign-in is selected in Provider Settings (experimental)',
|
||||
vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added',
|
||||
fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added',
|
||||
traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation',
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
* `fetch` shim in the page that intercepts the poll endpoints and returns
|
||||
* SYNTHETIC aircraft positioned near the camera, in the EXACT upstream payload
|
||||
* shapes the layers parse:
|
||||
* - flights → GET /api/opensky → { states: [ <state-vector[]> ] }
|
||||
* - military → GET /api/adsblol/mil → { ac: [ <adsblol-aircraft{}> ] }
|
||||
* - vessels → GET /api/ais-live → connected, zero-row snapshot
|
||||
* - flights → GET /api/flights → { states: [ <state-vector[]> ] }
|
||||
* - military → GET /api/military → { ac: [ <adsblol-aircraft{}> ] }
|
||||
* - vessels → GET /api/vessels → connected, zero-row snapshot
|
||||
* The shim stays installed so the layers' setInterval pollers keep the
|
||||
* synthetic planes alive (under MISSING_POLL_LIMIT=3 so they're never pruned).
|
||||
*
|
||||
@@ -343,7 +343,7 @@ async function main() {
|
||||
const url = new URL(requestUrl, window.location.href);
|
||||
const isAppRequest = url.origin === appOrigin;
|
||||
// OpenSky (commercial flights): { states: [ state-vector[] ] }
|
||||
if (isAppRequest && url.pathname === '/api/opensky') {
|
||||
if (isAppRequest && url.pathname === '/api/flights') {
|
||||
window.__SYNTH_HITS.opensky++;
|
||||
// A withheld contact models the ordinary case where the recipient's
|
||||
// first authoritative refresh does not yet carry the shared aircraft.
|
||||
@@ -371,11 +371,11 @@ async function main() {
|
||||
// Trail backfill (fires on trackById). Stub with valid-but-empty
|
||||
// payloads so the deterministic run never 404s against the dev proxy.
|
||||
// flights: { path: [ [time, lat, lon, baroAlt, true_track, on_ground] ] }
|
||||
if (isAppRequest && url.pathname === '/api/opensky-track') {
|
||||
if (isAppRequest && url.pathname === '/api/flights/track') {
|
||||
return Promise.resolve(jsonResponse({ path: [] }));
|
||||
}
|
||||
// military: { timestamp: <epochSec>, trace: [ [secAfter, lat, lon, ...] ] }
|
||||
if (isAppRequest && url.pathname === '/api/adsblol/trace') {
|
||||
if (isAppRequest && url.pathname === '/api/military/track') {
|
||||
return Promise.resolve(jsonResponse({ timestamp: Math.floor(Date.now() / 1000), trace: [] }));
|
||||
}
|
||||
// adsbdb enrichment (fires for tracked/model-eligible planes): empty
|
||||
@@ -388,7 +388,7 @@ async function main() {
|
||||
// Contacts enables the optional AIS source alongside aircraft. Keep
|
||||
// this tracking harness independent of local AIS credentials while
|
||||
// preserving the truthful "awaiting first message" lifecycle.
|
||||
if (isAppRequest && url.pathname === '/api/ais-live') {
|
||||
if (isAppRequest && url.pathname === '/api/vessels') {
|
||||
return Promise.resolve(jsonResponse({
|
||||
status: 'connected',
|
||||
rows: [],
|
||||
@@ -450,7 +450,7 @@ async function main() {
|
||||
return Promise.resolve(jsonResponse({ places: [] }));
|
||||
}
|
||||
// adsb.lol military: { ac: [ aircraft{} ] } (must come AFTER /trace check)
|
||||
if (isAppRequest && url.pathname === '/api/adsblol/mil') {
|
||||
if (isAppRequest && url.pathname === '/api/military') {
|
||||
window.__SYNTH_HITS.mil++;
|
||||
const ac = window.__SYNTH.military.map((m) => ({
|
||||
hex: m.hex,
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* Gemini Live adapter — BidiGenerateContent over WebSocket.
|
||||
*
|
||||
* Same prompt as production (system instruction = realtimeInstructions(),
|
||||
* tools = GEV_REALTIME_TOOLS as functionDeclarations with the production JSON
|
||||
* schema passed verbatim via parametersJsonSchema). The native-audio Live
|
||||
* model rejects a TEXT response modality (close 1007), so the session
|
||||
* requests AUDIO with output transcription on; tool calls arrive as
|
||||
* `toolCall` messages and the transcript stands in for the final words.
|
||||
*
|
||||
* Fresh session per phrase / dialogue. Each toolCall gets stub
|
||||
* `functionResponses`; the model continues on its own. A turn ends at a
|
||||
* `turnComplete` whose generation had no toolCall, or after 5 tool rounds
|
||||
* (mirrors the Realtime adapter's continuation cap), or 12 s of silence after
|
||||
* a tool round.
|
||||
*/
|
||||
import WebSocket from 'ws';
|
||||
import { providerKey } from '../keys.mjs';
|
||||
import { realtimeInstructions, GEV_REALTIME_TOOLS } from '../productionConfig.mjs';
|
||||
import { toGeminiDeclarations } from '../toolFormats.mjs';
|
||||
import { geminiUsageCostUsd } from '../pricing.mjs';
|
||||
import { newTurnResult } from './openaiRealtime.mjs';
|
||||
|
||||
const ENDPOINT = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
|
||||
const MAX_TOOL_ROUNDS = 5;
|
||||
const IDLE_AFTER_TOOL_MS = 12_000;
|
||||
|
||||
export function createGeminiLiveAdapter({ model, schemaMode = 'jsonSchema', turnTimeoutMs = 60_000 }) {
|
||||
const instructions = realtimeInstructions();
|
||||
const functionDeclarations = toGeminiDeclarations(GEV_REALTIME_TOOLS, { mode: schemaMode });
|
||||
const adapter = {
|
||||
provider: 'gemini-live',
|
||||
model,
|
||||
settings: {
|
||||
responseModalities: ['AUDIO'],
|
||||
outputAudioTranscription: true,
|
||||
schemaMode,
|
||||
temperature: 'server default (not set; production has no Gemini config)',
|
||||
thinking: 'server default',
|
||||
input: 'realtimeInput.text',
|
||||
context: 'fresh session per phrase / dialogue',
|
||||
},
|
||||
async open() {},
|
||||
async close() {},
|
||||
async runTurn({ text, onCall, context }) {
|
||||
return (await adapter.runDialogue({ turns: [text], onCall, contexts: [context] }))[0];
|
||||
},
|
||||
runDialogue({ turns, onCall, contexts = [] }) {
|
||||
return new Promise((resolveAll) => {
|
||||
const tSetup = Date.now();
|
||||
const ws = new WebSocket(`${ENDPOINT}?key=${providerKey('gemini')}`);
|
||||
const results = [];
|
||||
let setupMs = null;
|
||||
let cur = null;
|
||||
let closed = false;
|
||||
const send = (o) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(o)); };
|
||||
const shutdown = (error) => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
if (cur) {
|
||||
clearTimeout(cur.timer);
|
||||
clearTimeout(cur.idle);
|
||||
if (error) cur.r.errors.push(error);
|
||||
if (cur.r.tDoneMs === null) cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
cur = null;
|
||||
}
|
||||
while (results.length < turns.length) {
|
||||
const r = newTurnResult();
|
||||
r.errors.push(error || 'dialogue aborted');
|
||||
results.push(r);
|
||||
}
|
||||
try { ws.terminate(); } catch { /* noop */ }
|
||||
resolveAll(results);
|
||||
};
|
||||
const startTurn = (turn) => {
|
||||
const r = newTurnResult();
|
||||
r.setupMs = turn === 0 ? setupMs : 0;
|
||||
cur = { r, t0: Date.now(), turn, toolRounds: 0, toolCallThisGeneration: false, idle: null };
|
||||
cur.timer = setTimeout(() => { cur.r.timeout = true; endTurn(); }, turnTimeoutMs);
|
||||
// Live has no system-role mid-session item; a context item (pointer_context)
|
||||
// rides as a JSON line ahead of the user's words in the same text turn.
|
||||
const ctx = contexts[turn] ? `${JSON.stringify(contexts[turn])}\n` : '';
|
||||
send({ realtimeInput: { text: `${ctx}${turns[turn]}` } });
|
||||
};
|
||||
const endTurn = () => {
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.timer);
|
||||
clearTimeout(cur.idle);
|
||||
if (cur.r.tDoneMs === null) cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
const next = cur.turn + 1;
|
||||
const timedOut = cur.r.timeout;
|
||||
cur = null;
|
||||
if (next < turns.length && !timedOut) startTurn(next);
|
||||
else shutdown();
|
||||
};
|
||||
const mark = (key) => { if (cur && cur.r[key] === null) cur.r[key] = Date.now() - cur.t0; };
|
||||
|
||||
ws.on('open', () => {
|
||||
send({
|
||||
setup: {
|
||||
model: `models/${model}`,
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
systemInstruction: { parts: [{ text: instructions }] },
|
||||
tools: [{ functionDeclarations }],
|
||||
outputAudioTranscription: {},
|
||||
},
|
||||
});
|
||||
});
|
||||
ws.on('unexpected-response', (_req, res) => shutdown(`HTTP ${res.statusCode}`));
|
||||
ws.on('error', (e) => shutdown(`ws: ${e.message}`));
|
||||
ws.on('close', (code, reason) => {
|
||||
shutdown(results.length < turns.length ? `close ${code}: ${reason.toString().slice(0, 200)}` : undefined);
|
||||
});
|
||||
ws.on('message', (raw) => {
|
||||
let m;
|
||||
try { m = JSON.parse(raw.toString()); } catch { return; }
|
||||
if (m.setupComplete) {
|
||||
if (setupMs === null) {
|
||||
setupMs = Date.now() - tSetup;
|
||||
startTurn(0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.idle);
|
||||
const r = cur.r;
|
||||
const lastAt = Date.now();
|
||||
const sc = m.serverContent;
|
||||
if (sc?.modelTurn?.parts?.length) {
|
||||
mark('tFirstTokenMs');
|
||||
for (const p of sc.modelTurn.parts) if (typeof p.text === 'string' && !p.thought) r.text += p.text;
|
||||
}
|
||||
if (sc?.outputTranscription?.text) {
|
||||
mark('tFirstTokenMs');
|
||||
r.text += sc.outputTranscription.text;
|
||||
}
|
||||
if (m.toolCall?.functionCalls?.length) {
|
||||
mark('tFirstTokenMs');
|
||||
mark('tFirstCallMs');
|
||||
cur.toolCallThisGeneration = true;
|
||||
cur.toolRounds += 1;
|
||||
const functionResponses = m.toolCall.functionCalls.map((fc) => {
|
||||
const c = { name: fc.name, args: fc.args || {}, atMs: Date.now() - cur.t0 };
|
||||
r.calls.push(c);
|
||||
return { id: fc.id, name: fc.name, response: onCall(c, cur.turn) };
|
||||
});
|
||||
send({ toolResponse: { functionResponses } });
|
||||
}
|
||||
if (m.usageMetadata) {
|
||||
const u = m.usageMetadata;
|
||||
r.usage.inputTokens += u.promptTokenCount || 0;
|
||||
r.usage.cachedTokens += u.cachedContentTokenCount || 0;
|
||||
r.usage.outputTokens += u.responseTokenCount || 0;
|
||||
r.usage.reasoningTokens += u.thoughtsTokenCount || 0;
|
||||
r.costUsd += geminiUsageCostUsd(model, u);
|
||||
}
|
||||
if (m.goAway) r.errors.push('goAway');
|
||||
if (sc?.turnComplete) {
|
||||
r.responses += 1;
|
||||
if (cur.toolCallThisGeneration && cur.toolRounds < MAX_TOOL_ROUNDS) {
|
||||
cur.toolCallThisGeneration = false;
|
||||
if (r.text && !r.text.endsWith(' | ')) r.text += ' | ';
|
||||
// The follow-up generation normally starts within a second; if
|
||||
// the model has nothing more to say, end the turn at its last
|
||||
// message rather than charging the wait to its latency.
|
||||
const c = cur;
|
||||
c.idle = setTimeout(() => { if (cur === c) { c.r.tDoneMs = lastAt - c.t0; endTurn(); } }, IDLE_AFTER_TOOL_MS);
|
||||
return;
|
||||
}
|
||||
endTurn();
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Local-model adapters: stateless chat requests, so every phrase is a fresh
|
||||
* context by construction. Same system prompt (realtimeInstructions()) and
|
||||
* the production tools converted with toChatTools() (schema-normalized).
|
||||
*
|
||||
* openai-chat any OpenAI-compatible server (LM Studio :1234/v1, Ollama
|
||||
* :11434/v1). Streams SSE; timing is client-side.
|
||||
* ollama Ollama's native /api/chat. Same message and tool formats as
|
||||
* Chat Completions, but it can set num_ctx (the OpenAI shim
|
||||
* cannot — the ~15k-token prompt would be silently truncated
|
||||
* at a small default context) and turn thinking off, and it
|
||||
* reports server-side prefill/decode/load timings.
|
||||
*
|
||||
* Both mirror the Realtime loop: stub tool results, then continue until a
|
||||
* response has no tool calls, up to 4 continuations.
|
||||
*/
|
||||
import { realtimeInstructions, GEV_REALTIME_TOOLS } from '../productionConfig.mjs';
|
||||
import { toChatTools } from '../toolFormats.mjs';
|
||||
|
||||
const MAX_CONTINUATIONS = 4;
|
||||
|
||||
function emptyResult() {
|
||||
return {
|
||||
calls: [], text: '', errors: [], responses: 0,
|
||||
usage: { inputTokens: 0, cachedTokens: 0, outputTokens: 0, reasoningTokens: 0 },
|
||||
costUsd: 0, tFirstTokenMs: null, tFirstCallMs: null, tDoneMs: null, setupMs: 0, timeout: false,
|
||||
local: { prefillTokS: null, decodeTokS: null, promptEvalCount: null, evalCount: null, loadMs: null, firstResponseMs: null },
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(raw) {
|
||||
if (raw && typeof raw === 'object') return raw;
|
||||
try { return JSON.parse(raw || '{}'); } catch { return { __unparsed: String(raw).slice(0, 400) }; }
|
||||
}
|
||||
|
||||
async function* lines(body) {
|
||||
const decoder = new TextDecoder();
|
||||
let buf = '';
|
||||
for await (const chunk of body) {
|
||||
buf += decoder.decode(chunk, { stream: true });
|
||||
let i;
|
||||
while ((i = buf.indexOf('\n')) >= 0) {
|
||||
yield buf.slice(0, i);
|
||||
buf = buf.slice(i + 1);
|
||||
}
|
||||
}
|
||||
if (buf.trim()) yield buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared multi-response loop over one or more user turns in ONE growing
|
||||
* message history. `request(messages, r, t0, first, deadline)` performs one
|
||||
* streamed completion and returns `{ content, toolCalls:[{id,name,args}] }`.
|
||||
*/
|
||||
async function runChatDialogue({ turns, onCall, request, turnTimeoutMs, contexts = [] }) {
|
||||
const messages = [{ role: 'system', content: realtimeInstructions() }];
|
||||
const results = [];
|
||||
for (let turn = 0; turn < turns.length; turn += 1) {
|
||||
const r = emptyResult();
|
||||
if (contexts[turn]) messages.push({ role: 'system', content: JSON.stringify(contexts[turn]) });
|
||||
messages.push({ role: 'user', content: turns[turn] });
|
||||
const t0 = Date.now();
|
||||
const deadline = t0 + turnTimeoutMs;
|
||||
try {
|
||||
for (let round = 0; round <= MAX_CONTINUATIONS; round += 1) {
|
||||
if (Date.now() > deadline) { r.timeout = true; break; }
|
||||
const out = await request(messages, r, t0, round === 0, deadline);
|
||||
r.responses += 1;
|
||||
if (out.content) r.text += (r.text ? ' | ' : '') + out.content.trim();
|
||||
if (!out.toolCalls.length) {
|
||||
messages.push({ role: 'assistant', content: out.content || '' });
|
||||
break;
|
||||
}
|
||||
messages.push({ role: 'assistant', content: out.content || '', tool_calls: out.toolCalls.map((tc) => tc.wire) });
|
||||
for (const tc of out.toolCalls) {
|
||||
const c = { name: tc.name, args: tc.args, atMs: tc.atMs };
|
||||
r.calls.push(c);
|
||||
messages.push({ role: 'tool', tool_call_id: tc.id, tool_name: tc.name, name: tc.name, content: JSON.stringify(onCall(c, turn)) });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (e?.name === 'TimeoutError' || e?.name === 'AbortError') r.timeout = true;
|
||||
else r.errors.push(String(e?.message || e).slice(0, 300));
|
||||
}
|
||||
r.tDoneMs = Date.now() - t0;
|
||||
results.push(r);
|
||||
if (r.timeout || r.errors.length) {
|
||||
while (results.length < turns.length) {
|
||||
const skipped = emptyResult();
|
||||
skipped.errors.push('dialogue aborted after an earlier turn failed');
|
||||
results.push(skipped);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── OpenAI-compatible (SSE) ─────────────────────────────────────────────────
|
||||
|
||||
export function createOpenAiChatAdapter({ model, baseUrl, temperature = 0.8, extraBody = {}, turnTimeoutMs = 180_000, apiKey = 'local' }) {
|
||||
const tools = toChatTools(GEV_REALTIME_TOOLS);
|
||||
const request = async (messages, r, t0, first, deadline) => {
|
||||
const res = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(Math.max(1000, deadline - Date.now())),
|
||||
headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
|
||||
body: JSON.stringify({
|
||||
model, messages, tools, tool_choice: 'auto', temperature, stream: true,
|
||||
stream_options: { include_usage: true }, ...extraBody,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`);
|
||||
const tStart = Date.now();
|
||||
let tFirst = null;
|
||||
let content = '';
|
||||
const byIndex = new Map();
|
||||
let usage = null;
|
||||
let firstCallDoneAt = null;
|
||||
for await (const line of lines(res.body)) {
|
||||
if (!line.startsWith('data:')) continue;
|
||||
const data = line.slice(5).trim();
|
||||
if (data === '[DONE]') break;
|
||||
let m;
|
||||
try { m = JSON.parse(data); } catch { continue; }
|
||||
if (m.usage) usage = m.usage;
|
||||
const choice = m.choices?.[0];
|
||||
const d = choice?.delta || {};
|
||||
const now = Date.now();
|
||||
if ((d.content || d.reasoning_content || d.reasoning || d.tool_calls?.length) && tFirst === null) {
|
||||
tFirst = now;
|
||||
if (r.tFirstTokenMs === null) r.tFirstTokenMs = now - t0;
|
||||
}
|
||||
if (d.content) content += d.content;
|
||||
for (const tc of d.tool_calls || []) {
|
||||
const idx = tc.index ?? byIndex.size;
|
||||
if (idx > 0 && byIndex.has(0) && firstCallDoneAt === null) firstCallDoneAt = now;
|
||||
const cur = byIndex.get(idx) || { id: tc.id || `call_${idx}`, name: '', argText: '' };
|
||||
if (tc.id) cur.id = tc.id;
|
||||
if (tc.function?.name) cur.name += tc.function.name;
|
||||
if (tc.function?.arguments) cur.argText += typeof tc.function.arguments === 'string' ? tc.function.arguments : JSON.stringify(tc.function.arguments);
|
||||
byIndex.set(idx, cur);
|
||||
}
|
||||
if (choice?.finish_reason && byIndex.size && firstCallDoneAt === null) firstCallDoneAt = now;
|
||||
}
|
||||
const tEnd = Date.now();
|
||||
if (byIndex.size && firstCallDoneAt === null) firstCallDoneAt = tEnd;
|
||||
if (byIndex.size && r.tFirstCallMs === null) r.tFirstCallMs = firstCallDoneAt - t0;
|
||||
if (usage) {
|
||||
r.usage.inputTokens += usage.prompt_tokens || 0;
|
||||
r.usage.cachedTokens += usage.prompt_tokens_details?.cached_tokens || 0;
|
||||
r.usage.outputTokens += usage.completion_tokens || 0;
|
||||
r.usage.reasoningTokens += usage.completion_tokens_details?.reasoning_tokens || 0;
|
||||
}
|
||||
if (first) {
|
||||
r.local.firstResponseMs = tEnd - tStart;
|
||||
if (usage && tFirst) {
|
||||
r.local.promptEvalCount = usage.prompt_tokens ?? null;
|
||||
r.local.evalCount = usage.completion_tokens ?? null;
|
||||
// Client-side approximations: prefill ≈ prompt / (request → first token);
|
||||
// decode ≈ (completion − 1) / (first token → end of stream).
|
||||
r.local.prefillTokS = usage.prompt_tokens ? usage.prompt_tokens / Math.max(0.001, (tFirst - tStart) / 1000) : null;
|
||||
r.local.decodeTokS = usage.completion_tokens > 1 && tEnd > tFirst ? (usage.completion_tokens - 1) / ((tEnd - tFirst) / 1000) : null;
|
||||
r.local.timingSource = 'client';
|
||||
}
|
||||
}
|
||||
const toolCalls = [...byIndex.values()].map((tc) => ({
|
||||
id: tc.id, name: tc.name, args: parseArgs(tc.argText), atMs: firstCallDoneAt - t0,
|
||||
wire: { id: tc.id, type: 'function', function: { name: tc.name, arguments: tc.argText || '{}' } },
|
||||
}));
|
||||
return { content, toolCalls };
|
||||
};
|
||||
return {
|
||||
provider: 'openai-chat',
|
||||
model,
|
||||
settings: { baseUrl, temperature, tool_choice: 'auto', toolSchema: 'normalized (untyped → string|number|boolean)', context: 'stateless request per phrase', ...extraBody },
|
||||
async open() {},
|
||||
async close() {},
|
||||
runDialogue: ({ turns, onCall, contexts }) => runChatDialogue({ turns, onCall, contexts, request, turnTimeoutMs }),
|
||||
runTurn: async ({ text, onCall, context }) => (await runChatDialogue({ turns: [text], onCall, contexts: [context], request, turnTimeoutMs }))[0],
|
||||
};
|
||||
}
|
||||
|
||||
// ── Ollama native (/api/chat, NDJSON) ───────────────────────────────────────
|
||||
|
||||
export function createOllamaAdapter({ model, baseUrl = 'http://localhost:11434', temperature = 0.8, numCtx = 32768, think = false, keepAlive = '30m', turnTimeoutMs = 300_000 }) {
|
||||
const tools = toChatTools(GEV_REALTIME_TOOLS);
|
||||
let thinkParam = think;
|
||||
const request = async (messages, r, t0, first, deadline) => {
|
||||
const wireMessages = messages.map((m) => {
|
||||
if (m.role === 'assistant' && m.tool_calls) {
|
||||
return { role: 'assistant', content: m.content, tool_calls: m.tool_calls.map((tc) => ({ function: { name: tc.function.name, arguments: parseArgs(tc.function.arguments) } })) };
|
||||
}
|
||||
if (m.role === 'tool') return { role: 'tool', tool_name: m.tool_name, content: m.content };
|
||||
return m;
|
||||
});
|
||||
const body = { model, messages: wireMessages, tools, stream: true, keep_alive: keepAlive, options: { temperature, num_ctx: numCtx } };
|
||||
if (thinkParam !== null) body.think = thinkParam;
|
||||
const res = await fetch(`${baseUrl}/api/chat`, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(Math.max(1000, deadline - Date.now())),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const errText = await res.text();
|
||||
if (/does not support thinking/i.test(errText) && thinkParam !== null) {
|
||||
thinkParam = null;
|
||||
return request(messages, r, t0, first, deadline);
|
||||
}
|
||||
throw new Error(`HTTP ${res.status}: ${errText.slice(0, 300)}`);
|
||||
}
|
||||
let content = '';
|
||||
const toolCalls = [];
|
||||
let final = null;
|
||||
for await (const line of lines(res.body)) {
|
||||
if (!line.trim()) continue;
|
||||
let m;
|
||||
try { m = JSON.parse(line); } catch { continue; }
|
||||
if (m.error) throw new Error(m.error);
|
||||
const now = Date.now();
|
||||
const msg = m.message || {};
|
||||
if ((msg.content || msg.thinking || msg.tool_calls?.length) && r.tFirstTokenMs === null) r.tFirstTokenMs = now - t0;
|
||||
if (msg.content) content += msg.content;
|
||||
for (const tc of msg.tool_calls || []) {
|
||||
if (r.tFirstCallMs === null) r.tFirstCallMs = now - t0;
|
||||
const i = `${messages.length}_${toolCalls.length}`;
|
||||
const args = parseArgs(tc.function?.arguments);
|
||||
toolCalls.push({
|
||||
id: `call_${i}`, name: tc.function?.name, args, atMs: now - t0,
|
||||
wire: { id: `call_${i}`, type: 'function', function: { name: tc.function?.name, arguments: JSON.stringify(args) } },
|
||||
});
|
||||
}
|
||||
if (m.done) final = m;
|
||||
}
|
||||
if (final) {
|
||||
r.usage.inputTokens += final.prompt_eval_count || 0;
|
||||
r.usage.outputTokens += final.eval_count || 0;
|
||||
if (first) {
|
||||
const ns = 1e9;
|
||||
r.local.promptEvalCount = final.prompt_eval_count ?? null;
|
||||
r.local.evalCount = final.eval_count ?? null;
|
||||
r.local.prefillTokS = final.prompt_eval_duration ? final.prompt_eval_count / (final.prompt_eval_duration / ns) : null;
|
||||
r.local.decodeTokS = final.eval_duration ? final.eval_count / (final.eval_duration / ns) : null;
|
||||
r.local.loadMs = final.load_duration ? final.load_duration / 1e6 : null;
|
||||
r.local.firstResponseMs = final.total_duration ? final.total_duration / 1e6 : null;
|
||||
r.local.timingSource = 'ollama';
|
||||
}
|
||||
}
|
||||
return { content, toolCalls };
|
||||
};
|
||||
return {
|
||||
provider: 'ollama',
|
||||
model,
|
||||
get settings() {
|
||||
return { baseUrl, api: '/api/chat', temperature, num_ctx: numCtx, think: thinkParam === null ? 'unsupported by model (param omitted)' : thinkParam, keep_alive: keepAlive, toolSchema: 'normalized (untyped → string|number|boolean)', context: 'stateless request per phrase' };
|
||||
},
|
||||
async open() {},
|
||||
async close() {},
|
||||
runDialogue: ({ turns, onCall, contexts }) => runChatDialogue({ turns, onCall, contexts, request, turnTimeoutMs }),
|
||||
runTurn: async ({ text, onCall, context }) => (await runChatDialogue({ turns: [text], onCall, contexts: [context], request, turnTimeoutMs }))[0],
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* OpenAI Realtime adapter — text turns over the Realtime WebSocket.
|
||||
*
|
||||
* Each dialogue (one phrase, or a multi-turn dialogue) opens a FRESH session
|
||||
* configured with production's exact session body (captured from the token
|
||||
* handler) and sends its user text items one turn at a time. Each turn
|
||||
* mirrors qa-voice-routing's loop: every function call gets a stub output, and
|
||||
* a continuation response is requested after `response.done` (never while a
|
||||
* response is active), up to 4 continuations.
|
||||
*/
|
||||
import WebSocket from 'ws';
|
||||
import { providerKey } from '../keys.mjs';
|
||||
import { captureProductionSession } from '../productionConfig.mjs';
|
||||
import { openAiUsageCostUsd } from '../pricing.mjs';
|
||||
|
||||
const MAX_CONTINUATIONS = 4;
|
||||
|
||||
export function newTurnResult() {
|
||||
return {
|
||||
calls: [], text: '', errors: [], responses: 0,
|
||||
usage: { inputTokens: 0, cachedTokens: 0, outputTokens: 0, reasoningTokens: 0 },
|
||||
costUsd: 0, tFirstTokenMs: null, tFirstCallMs: null, tDoneMs: null, setupMs: null, timeout: false,
|
||||
};
|
||||
}
|
||||
|
||||
export function createOpenAiRealtimeAdapter({ model, turnTimeoutMs = 45_000 }) {
|
||||
let session = null;
|
||||
const adapter = {
|
||||
provider: 'openai-realtime',
|
||||
model,
|
||||
settings: {},
|
||||
async open() {
|
||||
session = await captureProductionSession(model);
|
||||
adapter.settings = {
|
||||
reasoning: session.reasoning,
|
||||
truncation: session.truncation,
|
||||
tool_choice: session.tool_choice,
|
||||
temperature: 'not settable (GA Realtime session has no temperature field)',
|
||||
output_modalities: ['text'],
|
||||
context: 'fresh session per phrase / dialogue',
|
||||
};
|
||||
},
|
||||
async close() {},
|
||||
async runTurn({ text, onCall, context }) {
|
||||
return (await adapter.runDialogue({ turns: [text], onCall, contexts: [context] }))[0];
|
||||
},
|
||||
runDialogue({ turns, onCall, contexts = [] }) {
|
||||
return new Promise((resolveAll) => {
|
||||
const tSetup = Date.now();
|
||||
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`, {
|
||||
headers: { Authorization: `Bearer ${providerKey('openai')}` },
|
||||
});
|
||||
const results = [];
|
||||
let setupMs = null;
|
||||
let cur = null; // { r, t0, continuations, needContinuation, turn, timer }
|
||||
let closed = false;
|
||||
const send = (o) => { if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify(o)); };
|
||||
const shutdown = (error) => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
if (cur) {
|
||||
clearTimeout(cur.timer);
|
||||
if (error) cur.r.errors.push(error);
|
||||
cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
cur = null;
|
||||
}
|
||||
while (results.length < turns.length) {
|
||||
const r = newTurnResult();
|
||||
r.errors.push(error || 'dialogue aborted');
|
||||
results.push(r);
|
||||
}
|
||||
try { ws.terminate(); } catch { /* noop */ }
|
||||
resolveAll(results);
|
||||
};
|
||||
const startTurn = (turn) => {
|
||||
const r = newTurnResult();
|
||||
r.setupMs = turn === 0 ? setupMs : 0;
|
||||
cur = { r, t0: Date.now(), continuations: 0, needContinuation: false, turn };
|
||||
cur.timer = setTimeout(() => { cur.r.timeout = true; endTurn(); }, turnTimeoutMs);
|
||||
// Optional inert system item first (the app's pointer_context shape).
|
||||
if (contexts[turn]) send({ type: 'conversation.item.create', item: { type: 'message', role: 'system', content: [{ type: 'input_text', text: JSON.stringify(contexts[turn]) }] } });
|
||||
send({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: turns[turn] }] } });
|
||||
send({ type: 'response.create', response: { output_modalities: ['text'] } });
|
||||
};
|
||||
const endTurn = () => {
|
||||
if (!cur) return;
|
||||
clearTimeout(cur.timer);
|
||||
cur.r.tDoneMs = Date.now() - cur.t0;
|
||||
results.push(cur.r);
|
||||
const next = cur.turn + 1;
|
||||
const timedOut = cur.r.timeout;
|
||||
cur = null;
|
||||
if (next < turns.length && !timedOut) startTurn(next);
|
||||
else shutdown();
|
||||
};
|
||||
const mark = (key) => { if (cur && cur.r[key] === null) cur.r[key] = Date.now() - cur.t0; };
|
||||
|
||||
ws.on('unexpected-response', (_req, res) => shutdown(`HTTP ${res.statusCode}`));
|
||||
ws.on('error', (e) => shutdown(`ws: ${e.message}`));
|
||||
ws.on('close', () => shutdown(results.length < turns.length ? 'socket closed' : undefined));
|
||||
ws.on('message', (raw) => {
|
||||
let m;
|
||||
try { m = JSON.parse(raw.toString()); } catch { return; }
|
||||
if (m.type === 'session.created') {
|
||||
send({ type: 'session.update', session });
|
||||
return;
|
||||
}
|
||||
if (m.type === 'session.updated' && setupMs === null) {
|
||||
setupMs = Date.now() - tSetup;
|
||||
startTurn(0);
|
||||
return;
|
||||
}
|
||||
if (m.type === 'error') {
|
||||
if (!cur) { shutdown(m.error?.message || 'realtime error'); return; }
|
||||
cur.r.errors.push(m.error?.message || 'realtime error');
|
||||
return;
|
||||
}
|
||||
if (!cur) return;
|
||||
const r = cur.r;
|
||||
if (
|
||||
m.type === 'response.output_text.delta' ||
|
||||
m.type === 'response.function_call_arguments.delta' ||
|
||||
m.type === 'response.output_audio_transcript.delta'
|
||||
) mark('tFirstTokenMs');
|
||||
if (m.type === 'response.output_text.delta' || m.type === 'response.output_audio_transcript.delta') {
|
||||
r.text += m.delta || '';
|
||||
}
|
||||
if (m.type === 'response.output_item.done' && m.item?.type === 'function_call') {
|
||||
mark('tFirstTokenMs');
|
||||
mark('tFirstCallMs');
|
||||
let args = {};
|
||||
try { args = JSON.parse(m.item.arguments || '{}'); } catch { args = { __unparsed: m.item.arguments }; }
|
||||
const c = { name: m.item.name, args, atMs: Date.now() - cur.t0 };
|
||||
r.calls.push(c);
|
||||
send({
|
||||
type: 'conversation.item.create',
|
||||
item: { type: 'function_call_output', call_id: m.item.call_id, output: JSON.stringify(onCall(c, cur.turn)) },
|
||||
});
|
||||
cur.needContinuation = true;
|
||||
}
|
||||
if (m.type === 'response.done') {
|
||||
r.responses += 1;
|
||||
const u = m.response?.usage;
|
||||
if (u) {
|
||||
r.usage.inputTokens += u.input_tokens || 0;
|
||||
r.usage.cachedTokens += u.input_token_details?.cached_tokens || 0;
|
||||
r.usage.outputTokens += u.output_tokens || 0;
|
||||
r.usage.reasoningTokens += u.output_token_details?.reasoning_tokens || 0;
|
||||
r.costUsd += openAiUsageCostUsd(model, u);
|
||||
}
|
||||
if (m.response?.status === 'failed') r.errors.push(JSON.stringify(m.response?.status_details).slice(0, 200));
|
||||
if (cur.needContinuation && cur.continuations < MAX_CONTINUATIONS) {
|
||||
cur.needContinuation = false;
|
||||
cur.continuations += 1;
|
||||
if (r.text) r.text += ' | ';
|
||||
send({ type: 'response.create', response: { output_modalities: ['text'] } });
|
||||
return;
|
||||
}
|
||||
endTurn();
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
return adapter;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
[{"type":"function","name":"fly_to_location","description":"Fly the God's Eye View camera to a known city, geocoded country/region/city/landmark, or explicit WGS84 coordinate. Countries/cities frame the whole place; landmarks/buildings use close framing.","parameters":{"type":"object","additionalProperties":false,"properties":{"locationId":{"type":"string","enum":["austin","sf","nyc","tokyo","london","paris","dubai","dc"],"description":"Known city preset ID. Use when the requested place matches one of these cities."},"query":{"type":"string","description":"Plain place search query, e.g. \"London\", \"Eiffel Tower\", or \"Dubai Marina\"."},"latitude":{"type":"number","minimum":-90,"maximum":90},"longitude":{"type":"number","minimum":-180,"maximum":180},"viewMode":{"type":"string","enum":["close","overview"],"description":"Optional framing intent. Usually omit this; GEV infers whole-place framing for countries/cities and close framing for landmarks."},"rangeM":{"type":"number","minimum":100,"maximum":20000000,"description":"Optional camera range from the target in meters. Omit it for automatic whole-country/whole-city or close-landmark framing; provide it only when the user explicitly requests a numeric height or distance."},"waitForArrival":{"type":"boolean","description":"Set true when a later tool depends on the destination viewport. The result then waits for the camera flight and returns arrived=true; cancellation returns ok=false."}}}},{"type":"function","name":"select_nearest_aircraft","description":"Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","enum":["flights","military"],"description":"Aircraft layer to enable and search. Use flights unless the user explicitly asks for military aircraft."},"locationId":{"type":"string","enum":["austin","sf","nyc","tokyo","london","paris","dubai","dc"],"description":"Known city preset ID when the place matches one of these cities."},"locationQuery":{"type":"string","maxLength":160,"description":"Free-form destination when no locationId matches."},"latitude":{"type":"number","minimum":-90,"maximum":90},"longitude":{"type":"number","minimum":-180,"maximum":180}},"required":["layerId"]}},{"type":"function","name":"adjust_camera_zoom","description":"Move the current Cesium camera closer to or farther from what it is presently looking at. Use for relative zoom requests without changing location.","parameters":{"type":"object","additionalProperties":false,"properties":{"direction":{"type":"string","enum":["in","out"]},"amount":{"type":"string","enum":["little","medium","lot"],"description":"Use little for phrases like \"a bit\" or \"a little\", medium for ordinary zoom requests, and lot for \"way out/in\"."}},"required":["direction","amount"]}},{"type":"function","name":"zoom_to_globe","description":"Pull the camera out to an ABSOLUTE full-Earth globe view (~18,000 km altitude, the whole planet in frame), keeping the current region centered. Use for \"globe view\", \"whole earth\", \"see the planet\", \"zoom all the way out\". Never use adjust_camera_zoom for these — its relative steps cannot reach the globe.","parameters":{"type":"object","additionalProperties":false,"properties":{}}},{"type":"function","name":"set_layer_visibility","description":"Enable or disable one registered God's Eye View data layer.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","description":"Common-name mapping for the non-obvious ids: space mission(s) → rocket-launches; fires/wildfires/active fires → local-firms (NASA FIRMS); ships/vessels/boats → ais-live-vessels; undersea/submarine cables → telegeography-submarine-cables; datacenters → local-datacenters; dams → local-dams; bikes/bike share → bikeshare; street traffic/congestion → traffic; traffic cameras → cctv; internet radio/stations → radio; ALPR/license plate readers/Flock cameras → alpr-cameras; local ADS-B/my receiver/my antenna (aircraft heard by a local RTL-SDR receiver) → local-adsb.","enum":["flights","military","earthquakes","satellites","rocket-launches","traffic","cctv","radio","bikeshare","ais-live-vessels","local-datacenters","local-dams","telegeography-submarine-cables","local-firms","alpr-cameras","local-adsb"]},"enabled":{"type":"boolean"}},"required":["layerId","enabled"]}},{"type":"function","name":"show_data_layers_menu","description":"Open the data layers dropdown/menu and optionally scroll to a specific layer row without toggling it.","parameters":{"type":"object","additionalProperties":false,"properties":{"layerId":{"type":"string","enum":["flights","military","earthquakes","satellites","traffic","ccLine truncated
|
||||
@@ -0,0 +1,289 @@
|
||||
/**
|
||||
* Pure grading for voice-routing runs. No I/O, no network.
|
||||
*
|
||||
* `gradeCore` reproduces the pass/fail semantics of the routing layer in
|
||||
* `scripts/qa-voice-routing.mjs` (which now calls it), plus a few extra
|
||||
* benchmark signals. `gradeCoverage` sorts coverage-probe turns into
|
||||
* correct / honest-refusal / hallucinated-capability / wrong-tool.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Spot-check `expected` against `actual` args: substring match (case
|
||||
* insensitive) for strings, exact match for everything else. An array value
|
||||
* is matched through its comma-joined string form, so `{ layers: 'earthquakes' }`
|
||||
* matches `layers: ['earthquakes']`.
|
||||
*/
|
||||
import { GEV_ACTION_SCHEMAS } from '../../src/voice/actionSchemas.js';
|
||||
export function matchArgs(expected, actual) {
|
||||
return Object.entries(expected).every(([k, v]) => {
|
||||
const got = actual?.[k];
|
||||
if (typeof v === 'string') return String(got ?? '').toLowerCase().includes(v.toLowerCase());
|
||||
return got === v;
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a call matcher: tool name plus optional arg spot-check. */
|
||||
export function call(name, args) {
|
||||
const matcher = (c) => c?.name === name && (!args || matchArgs(args, c.args));
|
||||
matcher.toolName = name;
|
||||
return matcher;
|
||||
}
|
||||
|
||||
/** Build a call matcher from a tool name and a predicate over its args. */
|
||||
export function where(name, predicate) {
|
||||
const matcher = (c) => c?.name === name && Boolean(predicate(c.args || {}));
|
||||
matcher.toolName = name;
|
||||
return matcher;
|
||||
}
|
||||
|
||||
/** Human label for a phrase's expectation. */
|
||||
export function expectLabel(phrase) {
|
||||
if (phrase.expectNone) return '(no tool)';
|
||||
if (Array.isArray(phrase.expect)) return phrase.expect.join('+');
|
||||
if (phrase.expect?.oneOf) return phrase.expect.oneOf.join('|');
|
||||
return String(phrase.expect);
|
||||
}
|
||||
|
||||
/** Every tool name that satisfies a phrase (union for oneOf / arrays). */
|
||||
export function expectedToolNames(phrase) {
|
||||
if (phrase.expectNone) return [];
|
||||
if (Array.isArray(phrase.expect)) return [...phrase.expect];
|
||||
if (phrase.expect?.oneOf) return [...phrase.expect.oneOf];
|
||||
return [phrase.expect];
|
||||
}
|
||||
|
||||
/**
|
||||
* Grade one core-suite turn.
|
||||
*
|
||||
* `ok` is exactly the qa-voice-routing verdict: expectNone passes only with no
|
||||
* calls; an array needs every tool; oneOf needs any; a string needs that tool;
|
||||
* then `args` / `argsByTool` spot checks. The rest are benchmark extras:
|
||||
* toolOk tool-name routing alone (before the args check)
|
||||
* argsOk null when the phrase has no arg spot check
|
||||
* firstCallOk the FIRST call was an expected tool (what a user sees first)
|
||||
* falsePositive a conversational turn produced a tool call
|
||||
* extraCalls calls to tools outside the expected set
|
||||
* typedArgsOk benchmark-only typed-argument check (`typedArgs`: exact
|
||||
* enum values, layer ids, numeric thresholds, sort order);
|
||||
* null when the phrase has none. Not part of `ok`.
|
||||
* fullCallOk toolOk and every arg check (args, argsByTool, typedArgs)
|
||||
*/
|
||||
export function gradeCore(phrase, calls) {
|
||||
const names = calls.map((c) => c.name);
|
||||
if (phrase.expectNone) {
|
||||
return {
|
||||
ok: names.length === 0,
|
||||
toolOk: names.length === 0,
|
||||
fullCallOk: names.length === 0,
|
||||
argsOk: null,
|
||||
typedArgsOk: null,
|
||||
firstCallOk: names.length === 0,
|
||||
falsePositive: names.length > 0,
|
||||
extraCalls: names,
|
||||
detail: names.length ? `unexpected calls: ${names.join(',')}` : 'clean conversational turn',
|
||||
};
|
||||
}
|
||||
const expected = expectedToolNames(phrase);
|
||||
let toolOk;
|
||||
if (Array.isArray(phrase.expect)) toolOk = phrase.expect.every((n) => names.includes(n));
|
||||
else if (phrase.expect?.oneOf) toolOk = names.some((n) => phrase.expect.oneOf.includes(n));
|
||||
else toolOk = names.includes(phrase.expect);
|
||||
|
||||
let detail = `called: ${names.join(',') || '(none)'}`;
|
||||
let argsOk = null;
|
||||
if (phrase.args) {
|
||||
const target = calls.find((c) => (Array.isArray(phrase.expect) ? true : expected.includes(c.name)));
|
||||
argsOk = Boolean(target) && matchArgs(phrase.args, target.args);
|
||||
if (toolOk && !argsOk) detail += ` args mismatch: ${JSON.stringify(target?.args)}`;
|
||||
}
|
||||
if (phrase.argsByTool) {
|
||||
let allOk = true;
|
||||
for (const [toolName, expectedArgs] of Object.entries(phrase.argsByTool)) {
|
||||
const target = calls.find((c) => c.name === toolName);
|
||||
if (!target || !matchArgs(expectedArgs, target.args)) {
|
||||
allOk = false;
|
||||
if (toolOk) detail += ` ${toolName} args mismatch: ${JSON.stringify(target?.args)}`;
|
||||
}
|
||||
}
|
||||
argsOk = argsOk === null ? allOk : argsOk && allOk;
|
||||
}
|
||||
const typedArgsOk = phrase.typedArgs ? phrase.typedArgs.every((m) => calls.some((c) => m(c))) : null;
|
||||
if (toolOk && typedArgsOk === false) detail += ` typed args miss: ${JSON.stringify(calls.map((c) => c.args)).slice(0, 200)}`;
|
||||
return {
|
||||
ok: toolOk && argsOk !== false,
|
||||
toolOk,
|
||||
fullCallOk: toolOk && argsOk !== false && typedArgsOk !== false,
|
||||
argsOk,
|
||||
typedArgsOk,
|
||||
firstCallOk: names.length > 0 && expected.includes(names[0]),
|
||||
falsePositive: false,
|
||||
extraCalls: names.filter((n) => !expected.includes(n)),
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Grade a two-turn dialogue from each turn's calls. The headline `ok` is
|
||||
* turn 2's full-call accuracy (tool + typed args), since turn 2 is the one
|
||||
* that depends on context; turn 1 is reported alongside.
|
||||
*/
|
||||
export function gradeDialogue(dialogue, turnCalls) {
|
||||
const turn1 = gradeCore(dialogue.first, turnCalls[0] || []);
|
||||
const turn2 = gradeCore(dialogue.second, turnCalls[1] || []);
|
||||
return {
|
||||
ok: turn2.fullCallOk,
|
||||
toolOk: turn2.toolOk,
|
||||
fullCallOk: turn2.fullCallOk,
|
||||
turn1Ok: turn1.fullCallOk,
|
||||
turn1,
|
||||
turn2,
|
||||
detail: `t1 ${turn1.detail} | t2 ${turn2.detail}`,
|
||||
};
|
||||
}
|
||||
|
||||
// ── JSON-schema argument validation (the subset GEV tools use) ─────────────
|
||||
|
||||
const TYPE_CHECKS = {
|
||||
string: (v) => typeof v === 'string',
|
||||
number: (v) => typeof v === 'number' && Number.isFinite(v),
|
||||
integer: (v) => Number.isInteger(v),
|
||||
boolean: (v) => typeof v === 'boolean',
|
||||
object: (v) => v !== null && typeof v === 'object' && !Array.isArray(v),
|
||||
array: (v) => Array.isArray(v),
|
||||
null: (v) => v === null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Validate `value` against a JSON schema (type / enum / anyOf / required /
|
||||
* properties / additionalProperties:false / items / min/max bounds).
|
||||
* Returns a list of human-readable violations; empty means valid.
|
||||
*/
|
||||
export function validateArgs(schema, value, where = 'args') {
|
||||
const errors = [];
|
||||
if (!schema || typeof schema !== 'object') return errors;
|
||||
if (schema.type) {
|
||||
const types = Array.isArray(schema.type) ? schema.type : [schema.type];
|
||||
if (!types.some((t) => TYPE_CHECKS[t]?.(value))) {
|
||||
errors.push(`${where} should be ${types.join('|')}`);
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
if (schema.enum && !schema.enum.includes(value)) {
|
||||
errors.push(`${where}=${JSON.stringify(value)} is not one of ${schema.enum.join(', ')}`);
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
if (typeof schema.minLength === 'number' && value.length < schema.minLength)
|
||||
errors.push(`${where} needs at least ${schema.minLength} characters`);
|
||||
if (typeof schema.pattern === 'string' && !new RegExp(schema.pattern).test(value))
|
||||
errors.push(`${where} does not match ${schema.pattern}`);
|
||||
}
|
||||
if (typeof value === 'number') {
|
||||
if (typeof schema.minimum === 'number' && value < schema.minimum) errors.push(`${where} below ${schema.minimum}`);
|
||||
if (typeof schema.maximum === 'number' && value > schema.maximum) errors.push(`${where} above ${schema.maximum}`);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (typeof schema.minItems === 'number' && value.length < schema.minItems) errors.push(`${where} needs ≥${schema.minItems} items`);
|
||||
if (schema.items) value.forEach((item, i) => errors.push(...validateArgs(schema.items, item, `${where}[${i}]`)));
|
||||
}
|
||||
if (TYPE_CHECKS.object(value) && (schema.properties || schema.required)) {
|
||||
for (const key of schema.required || []) {
|
||||
if (!(key in value)) errors.push(`${where}.${key} is required`);
|
||||
}
|
||||
for (const [key, v] of Object.entries(value)) {
|
||||
const sub = schema.properties?.[key];
|
||||
if (!sub) {
|
||||
if (schema.additionalProperties === false) errors.push(`${where}.${key} is not a known argument`);
|
||||
continue;
|
||||
}
|
||||
errors.push(...validateArgs(sub, v, `${where}.${key}`));
|
||||
}
|
||||
}
|
||||
if (Array.isArray(schema.anyOf)) {
|
||||
const branches = schema.anyOf.map((branch) =>
|
||||
validateArgs(branch, value, where),
|
||||
);
|
||||
if (!branches.some((branchErrors) => branchErrors.length === 0))
|
||||
errors.push(`${where} does not match any allowed shape`);
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/** Validate a tool call against the Realtime tool list. */
|
||||
export function validateCall(tools, c) {
|
||||
const tool = tools.find((t) => t.name === c.name);
|
||||
if (!tool) return [`unknown tool ${c.name}`];
|
||||
// The runner validates app actions against their full schema, including
|
||||
// top-level rules (anyOf) that the model-facing tools must omit.
|
||||
const action = GEV_ACTION_SCHEMAS.find((schema) => schema.name === c.name);
|
||||
const rules = action?.parameters
|
||||
? { ...tool.parameters, anyOf: action.parameters.anyOf }
|
||||
: tool.parameters;
|
||||
if (!rules.anyOf) delete rules.anyOf;
|
||||
return validateArgs(rules, c.args ?? {});
|
||||
}
|
||||
|
||||
// ── Coverage probe grading ─────────────────────────────────────────────────
|
||||
|
||||
/** Tools that only read state; they never count as acting on the request. */
|
||||
export const LOOKUP_TOOLS = Object.freeze([
|
||||
'get_entity_context',
|
||||
'get_current_view_state',
|
||||
'show_data_layers_menu',
|
||||
'set_panel_open',
|
||||
]);
|
||||
|
||||
export const REFUSAL_RE =
|
||||
/\b(can(?:no|')t|cannot|unable|not able|don'?t have|do not have|doesn'?t have|does not have|isn'?t (?:available|supported|something)|aren'?t (?:available|supported)|not (?:available|supported|possible|yet)|no (?:access|way|tool|data|layer|feed|live|transit|weather|wind|cyclone|radar|lightning|receiver)|n'?t (?:connected|see any|find)|(?:isn'?t|is no|are no) (?:an? )?(?:\w+ ){0,2}(?:layer|feed|tool|data source|integration)s?|outside what i can|beyond what i can|only (?:have|support|show|cover)|unfortunately|sorry)\b/i;
|
||||
|
||||
/**
|
||||
* Grade one coverage-probe turn.
|
||||
*
|
||||
* Order matters:
|
||||
* 1. a schema-invalid call (invented arg/tool) → hallucinated-capability
|
||||
* 2. a call pattern in `correct` → correct
|
||||
* 3. a pattern in `hallucinated` → hallucinated-capability
|
||||
* 4. any other acting call (not lookup/neutral) → wrong-tool
|
||||
* 5. no acting call: text matches `claimText`
|
||||
* and no refusal wording → hallucinated-capability
|
||||
* 6. no acting call: refusal wording or a question → honest-refusal
|
||||
* 7. no acting call, no refusal → hallucinated-capability
|
||||
* (it answered as if it had the data)
|
||||
*
|
||||
* `acceptable` = correct, or honest-refusal on an unsupported item.
|
||||
*/
|
||||
export function gradeCoverage(item, calls, text, tools = []) {
|
||||
// Models write typographic apostrophes ("can’t"); fold them before matching.
|
||||
const finalText = String(text || '').replace(/[\u2018\u2019\u02bc]/g, "'");
|
||||
const verdict = (v, reason) => ({
|
||||
verdict: v,
|
||||
reason,
|
||||
acceptable: v === 'correct' || (v === 'honest-refusal' && !item.supported),
|
||||
});
|
||||
if (tools.length) {
|
||||
for (const c of calls) {
|
||||
const errors = validateCall(tools, c);
|
||||
if (errors.length) return verdict('hallucinated-capability', `invalid ${c.name}: ${errors[0]}`);
|
||||
}
|
||||
}
|
||||
for (const alternative of item.correct || []) {
|
||||
if (alternative.every((m) => calls.some((c) => m(c)))) {
|
||||
return verdict('correct', `matched ${alternative.map((m) => m.toolName || 'pattern').join('+')}`);
|
||||
}
|
||||
}
|
||||
for (const m of item.hallucinated || []) {
|
||||
const hit = calls.find((c) => m(c));
|
||||
if (hit) return verdict('hallucinated-capability', `faked capability via ${hit.name}(${JSON.stringify(hit.args).slice(0, 80)})`);
|
||||
}
|
||||
const neutral = new Set([...LOOKUP_TOOLS, ...(item.neutral || [])]);
|
||||
const acting = calls.filter((c) => !neutral.has(c.name));
|
||||
if (acting.length) return verdict('wrong-tool', `called ${acting.map((c) => c.name).join(',')}`);
|
||||
const refused = REFUSAL_RE.test(finalText);
|
||||
if (item.claimText && item.claimText.test(finalText) && !refused) {
|
||||
return verdict('hallucinated-capability', 'asserted data it cannot have');
|
||||
}
|
||||
if (refused || /\?\s*$/.test(finalText.trim())) return verdict('honest-refusal', 'refused or asked');
|
||||
if (!finalText.trim().replace(/\|/g, '').trim()) {
|
||||
return verdict('wrong-tool', calls.length ? `only lookups/neutral calls (${calls.map((c) => c.name).join(',')}) and no answer` : 'no tool call and no answer');
|
||||
}
|
||||
return verdict('hallucinated-capability', 'answered without a tool or a refusal');
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Runtime key lookup for the voice benchmark. Keys are read from the
|
||||
* environment, then the macOS Keychain, at call time. They are never written
|
||||
* to results, logs, or files.
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const KEYCHAIN = {
|
||||
openai: { env: 'OPENAI_API_KEY', service: 'openai-api', account: 'api-key' },
|
||||
gemini: { env: 'GEMINI_API_KEY', service: 'gemini-api', account: 'api-key' },
|
||||
};
|
||||
|
||||
const cache = new Map();
|
||||
|
||||
/** Resolve a provider key from the environment, then the Keychain. */
|
||||
export function providerKey(provider) {
|
||||
if (cache.has(provider)) return cache.get(provider);
|
||||
const spec = KEYCHAIN[provider];
|
||||
if (!spec) throw new Error(`unknown key provider: ${provider}`);
|
||||
let value = process.env[spec.env] || '';
|
||||
if (!value && process.platform === 'darwin') {
|
||||
try {
|
||||
value = execFileSync(
|
||||
'security',
|
||||
['find-generic-password', '-s', spec.service, '-a', spec.account, '-w'],
|
||||
{ encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] },
|
||||
).trim();
|
||||
} catch {
|
||||
value = '';
|
||||
}
|
||||
}
|
||||
if (!value) throw new Error(`${spec.env} is not available (env or Keychain)`);
|
||||
cache.set(provider, value);
|
||||
return value;
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
/**
|
||||
* Shared voice-routing phrase tables.
|
||||
*
|
||||
* CORE_PHRASES is the routing table used by `scripts/qa-voice-routing.mjs`
|
||||
* (layer 1) and the core suite of `scripts/qa-voice-bench.mjs`.
|
||||
*
|
||||
* `expect` tool name the model must call (string), every tool in an
|
||||
* array, or `{ oneOf: [...] }` for documented acceptable variance.
|
||||
* `expectNone` conversational turns that must NOT call tools.
|
||||
* `args` spot-check subset matched against the first matching call's
|
||||
* args (substring match for strings, exact for booleans/numbers).
|
||||
* `argsByTool` the same spot check, keyed by tool, for multi-intent phrases.
|
||||
* `typedArgs` benchmark-only matchers for typed arguments (exact enum
|
||||
* values, layer ids, thresholds, sort direction). They feed the
|
||||
* benchmark's full-call accuracy and never change the
|
||||
* qa-voice-routing verdict.
|
||||
*
|
||||
* COVERAGE_PROBE is the newer-layers coverage probe: utterances about layers
|
||||
* and questions that today's tool surface only partly supports. Each item is
|
||||
* graded into correct / honest-refusal / hallucinated-capability / wrong-tool
|
||||
* by `gradeCoverage` in ./grade.mjs.
|
||||
*/
|
||||
|
||||
import { call, where } from './grade.mjs';
|
||||
|
||||
export const CORE_PHRASES = Object.freeze([
|
||||
// — navigation & framing —
|
||||
{ phrase: 'Take me to Tokyo', expect: 'fly_to_location', typedArgs: [where('fly_to_location', (a) => a.locationId === 'tokyo' || /tokyo/i.test(a.query || ''))] },
|
||||
{ phrase: 'Fly to the Golden Gate Bridge', expect: 'fly_to_location' },
|
||||
{ phrase: 'Go to Sixth Street in Austin', expect: 'fly_to_location' },
|
||||
{ phrase: 'Show me the Alps from above', expect: { oneOf: ['fly_to_location', 'frame_overhead'] } },
|
||||
{ phrase: 'Zoom in a bit', expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'in' && a.amount === 'little')] },
|
||||
{ phrase: 'Zoom out a little', expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'out' && a.amount === 'little')] },
|
||||
{ phrase: 'Zoom out to a globe view', expect: 'zoom_to_globe' },
|
||||
{ phrase: 'Show me the whole earth', expect: 'zoom_to_globe' },
|
||||
{ phrase: 'Frame the aircraft near us from overhead', expect: 'frame_overhead', typedArgs: [where('frame_overhead', (a) => a.target === 'flights')] },
|
||||
|
||||
// — the satellites trap: data layer, never basemap —
|
||||
{ phrase: 'Show me the satellites', expect: { oneOf: ['set_layer_visibility', 'frame_overhead'] } },
|
||||
{ phrase: 'Turn off the satellites', expect: 'set_layer_visibility', args: { layerId: 'satellites' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'satellites' && a.enabled === false)] },
|
||||
{ phrase: 'Switch to Bing aerial', expect: 'set_map_stack', typedArgs: [where('set_map_stack', (a) => a.stack === 'bing-aerial')] },
|
||||
{ phrase: 'Switch the basemap to OSM', expect: 'set_map_stack', typedArgs: [where('set_map_stack', (a) => a.stack === 'osm')] },
|
||||
|
||||
// — layers —
|
||||
{ phrase: 'Turn on the flights layer', expect: 'set_layer_visibility', args: { layerId: 'flights' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'flights' && a.enabled === true)] },
|
||||
{ phrase: 'Show me live vessels', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'ais-live-vessels' && a.enabled === true)] },
|
||||
{ phrase: 'Turn on the fires layer', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'local-firms' && a.enabled === true)] },
|
||||
{ phrase: 'Turn on street traffic', expect: 'set_layer_visibility', args: { layerId: 'traffic' }, typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'traffic' && a.enabled === true)] },
|
||||
{ phrase: 'Open the data layers menu', expect: 'show_data_layers_menu' },
|
||||
{ phrase: 'Show me the datacenter layers', expect: 'show_data_layers_menu', typedArgs: [where('show_data_layers_menu', (a) => a.layerId === 'local-datacenters')] },
|
||||
{ phrase: 'Turn on the datacenters layer', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'local-datacenters' && a.enabled === true)] },
|
||||
|
||||
// — visual styles & post-fx —
|
||||
{ phrase: 'Give me night vision', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'surveillance')] },
|
||||
{ phrase: 'Switch to thermal view', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'thermal')] },
|
||||
{ phrase: 'Back to the normal look', expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'normal')] },
|
||||
{ phrase: 'Turn on bloom', expect: 'set_post_processing', typedArgs: [where('set_post_processing', (a) => a.bloom?.enabled === true)] },
|
||||
{ phrase: 'Sharpen the image a touch', expect: 'set_post_processing', typedArgs: [where('set_post_processing', (a) => a.sharpen && a.sharpen.enabled !== false)] },
|
||||
|
||||
// — HUD / detection / panels —
|
||||
{ phrase: 'Turn the HUD off', expect: 'set_hud', typedArgs: [where('set_hud', (a) => a.visible === 'off')] },
|
||||
{ phrase: 'Switch to the tactical layout', expect: 'set_hud', typedArgs: [where('set_hud', (a) => a.layout === 'tactical')] },
|
||||
{ phrase: 'Turn on detection', expect: 'set_detection', typedArgs: [where('set_detection', (a) => a.enabled !== false)] },
|
||||
{ phrase: 'Set detection density to fifty percent', expect: 'set_detection', typedArgs: [where('set_detection', (a) => a.densityPct === 50 || a.mode === 'balanced')] },
|
||||
|
||||
// — context questions —
|
||||
{ phrase: 'What am I looking at right now?', expect: 'get_entity_context' },
|
||||
{ phrase: 'What city is this below us?', expect: 'get_entity_context' },
|
||||
{ phrase: 'Is there anything interesting in view?', expect: 'get_entity_context' },
|
||||
|
||||
// — tracking —
|
||||
// Context-free text turns may reasonably look before tracking; either
|
||||
// routing is correct (production sessions always carry screen context).
|
||||
{ phrase: 'Track that plane', expect: { oneOf: ['track_entity', 'get_entity_context'] } },
|
||||
// The session instructions route "nearest" through analyst_query first
|
||||
// (sortBy distance, limit 1); stubbed results make that first hop the whole
|
||||
// observable turn.
|
||||
{ phrase: 'Follow the nearest aircraft', expect: { oneOf: ['track_entity', 'get_entity_context', 'analyst_query'] } },
|
||||
{ phrase: 'Stop tracking', expect: 'stop_tracking' },
|
||||
|
||||
// — CCTV / scenes / ISS —
|
||||
{ phrase: 'Show me the nearest traffic camera', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'nearest')] },
|
||||
{ phrase: 'Turn on the camera viewsheds', expect: 'control_cctv', typedArgs: [where('control_cctv', (a) => a.action === 'viewshed' && a.enabled !== false)] },
|
||||
{ phrase: 'Play a news radio station near Austin', expect: 'control_radio', args: { action: 'select', category: 'news', locationId: 'austin' }, typedArgs: [where('control_radio', (a) => a.action === 'select' && a.category === 'news' && a.locationId === 'austin')] },
|
||||
{ phrase: 'Turn on the radio', expect: 'control_radio', args: { action: 'play' }, typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
{ phrase: 'Set the radio volume to thirty percent', expect: 'control_radio', args: { action: 'volume', volumePct: 30 } },
|
||||
{ phrase: 'Pause the radio', expect: 'control_radio', args: { action: 'pause' }, typedArgs: [where('control_radio', (a) => a.action === 'pause')] },
|
||||
{ phrase: 'Stop the radio', expect: 'control_radio', args: { action: 'stop' }, typedArgs: [where('control_radio', (a) => a.action === 'stop')] },
|
||||
{ phrase: 'When does the ISS pass over next?', expect: 'next_iss_pass' },
|
||||
|
||||
// — annotations —
|
||||
{ phrase: 'Annotate the Texas State Capitol and its grounds', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Outline the state of Texas', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Outline Lady Bird Lake', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'area'))] },
|
||||
{ phrase: 'Draw the walking route from the Capitol to Zilker Park', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'route' && (x.mode ?? 'walking') === 'walking'))] },
|
||||
{ phrase: 'How far is the Eiffel Tower from the Louvre?', expect: 'annotate_map', typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) => x.type === 'arrow'))] },
|
||||
{ phrase: 'Clear the map', expect: 'clear_annotations' },
|
||||
|
||||
// — multi-intent (assert ALL tools fire before speech) —
|
||||
{
|
||||
phrase: 'Switch to night vision and turn on the flights layer',
|
||||
expect: ['set_visual_style', 'set_layer_visibility'],
|
||||
typedArgs: [where('set_visual_style', (a) => a.style === 'surveillance'), where('set_layer_visibility', (a) => a.layerId === 'flights' && a.enabled === true)],
|
||||
},
|
||||
{
|
||||
phrase: 'Turn off the HUD and take me to Paris',
|
||||
expect: ['set_hud', 'fly_to_location'],
|
||||
typedArgs: [where('set_hud', (a) => a.visible === 'off'), where('fly_to_location', (a) => a.locationId === 'paris' || /paris/i.test(a.query || ''))],
|
||||
},
|
||||
{
|
||||
phrase: 'Go to full planet view and then turn on the radio',
|
||||
expect: ['zoom_to_globe', 'control_radio'],
|
||||
argsByTool: { control_radio: { action: 'play' } },
|
||||
},
|
||||
|
||||
// — camera verbs (tools #23/#24) —
|
||||
{ phrase: 'Orbit around this area slowly', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'orbit' && a.speed === 'slow')] },
|
||||
{ phrase: 'Pan left a bit', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'pan' && a.direction === 'left')] },
|
||||
{ phrase: 'Stop moving the camera', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'stop')] },
|
||||
{ phrase: 'Fly the route we just drew', expect: 'fly_route' },
|
||||
|
||||
// — analyst queries (tool #22) —
|
||||
{ phrase: 'How many flights are over Texas right now?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('flights') && a.scope?.kind === 'region' && /texas/i.test(a.scope?.name || ''))] },
|
||||
{ phrase: 'Which ships are headed to Oakland?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('ais-live-vessels') && (a.filters || []).some((f) => f.field === 'destination' && /oakland/i.test(String(f.value))))] },
|
||||
{ phrase: 'What is the biggest fire near Los Angeles?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('local-firms') && ['frp', 'acres'].includes(a.sortBy) && a.sortDir !== 'asc')] },
|
||||
{ phrase: 'Is anything flying above forty thousand feet?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).some((l) => l === 'flights' || l === 'military') && (a.filters || []).some((f) => f.field === 'altitudeM' && (f.op === 'gt' || f.op === 'gte') && Math.abs(Number(f.value) - 12192) <= 50))] },
|
||||
|
||||
// — negative controls: conversation must NOT tool-call —
|
||||
{ phrase: 'How is your evening going?', expectNone: true },
|
||||
{ phrase: 'Tell me a fun fact about maps', expectNone: true },
|
||||
]);
|
||||
|
||||
// ── Coverage probe ──────────────────────────────────────────────────────────
|
||||
//
|
||||
// Text-mode port of the voice-coverage routing probe. The live probe ran each
|
||||
// utterance against a real app page (setup steps, real tool results); this
|
||||
// benchmark has no app, so every item states the scene it assumes in `scene`
|
||||
// and the model sees only stub tool results. Grading therefore judges the
|
||||
// ROUTING DECISION and the honesty of the final words, not app behaviour.
|
||||
//
|
||||
// supported true when today's tools can satisfy the ask (a refusal is a
|
||||
// miss); false when the best possible outcome is an honest
|
||||
// refusal, optionally after a neutral helper call.
|
||||
// correct call patterns that satisfy the ask (any alternative).
|
||||
// neutral extra tools that neither help nor hurt (navigation to the
|
||||
// named place, enabling a related layer that exists).
|
||||
// hallucinated call patterns that fake the capability (e.g. toggling the
|
||||
// satellites DATA layer for "recent satellite imagery").
|
||||
// claimText final-text pattern that asserts data the model cannot have.
|
||||
//
|
||||
// Any call whose arguments violate the tool schema (an invented layer id) is
|
||||
// graded hallucinated-capability regardless of the item.
|
||||
//
|
||||
// Two rubrics, so a before/after comparison stays honest:
|
||||
// v1 the original rubric, graded against the tool schema of 4b56d0e9
|
||||
// (baseline-tools-4b56d0e9.json) — "by the old yardstick".
|
||||
// v2 `item.v2` overrides (supported / correct / hallucinated / neutral),
|
||||
// graded against the CURRENT production tools after the voice layer
|
||||
// manifest and point-and-ask landed. Use
|
||||
// coverageRubric(item, 'v2') to get the merged item.
|
||||
|
||||
const hasRoute = (c) =>
|
||||
c.name === 'annotate_map' &&
|
||||
Array.isArray(c.args?.annotations) &&
|
||||
c.args.annotations.some((a) => a?.type === 'route');
|
||||
|
||||
/** The coverage item as graded under rubric 'v1' (original) or 'v2' (current tools). */
|
||||
export function coverageRubric(item, version = 'v1') {
|
||||
if (version !== 'v2' || !item.v2) return item;
|
||||
return { ...item, ...item.v2 };
|
||||
}
|
||||
|
||||
export const COVERAGE_PROBE = Object.freeze([
|
||||
{
|
||||
id: 'quakes-m5-week',
|
||||
v2: { supported: true },
|
||||
utterance: 'Show me earthquakes above magnitude 5 this week',
|
||||
scene: 'globe, no layers on',
|
||||
expect: 'enable earthquakes + analyst_query magnitude>=5; must say feed covers last 24h only (no week)',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('analyst_query', { layers: 'earthquakes' })],
|
||||
[call('set_layer_visibility', { layerId: 'earthquakes', enabled: true })],
|
||||
],
|
||||
neutral: ['zoom_to_globe'],
|
||||
},
|
||||
{
|
||||
id: 'trains-london',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'transit' })], [call('set_layer_visibility', { layerId: 'transit', enabled: true })]], hallucinated: [] },
|
||||
utterance: 'How many trains are moving in London right now?',
|
||||
scene: 'London',
|
||||
expect: 'honest "no transit data for London / transit not queryable" — no invented count',
|
||||
supported: false,
|
||||
neutral: ['fly_to_location'],
|
||||
hallucinated: [call('analyst_query')],
|
||||
claimText: /\b\d+\s+(trains?|services)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'transit-on',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'transit', enabled: true })]] },
|
||||
utterance: 'Turn on transit',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility transit (no transit id in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('set_layer_visibility', { layerId: 'traffic', enabled: true })],
|
||||
},
|
||||
{
|
||||
id: 'ship-fastest-singapore',
|
||||
v2: { supported: true },
|
||||
utterance: 'Which ship near Singapore is fastest?',
|
||||
scene: 'Singapore Strait, vessels layer on',
|
||||
expect: 'analyst_query ais-live-vessels near Singapore sortBy speedKts desc',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'ais-live-vessels' })]],
|
||||
neutral: ['fly_to_location', 'set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'recent-imagery',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'recent-imagery', enabled: true })]] },
|
||||
utterance: 'Show recent satellite imagery of this area',
|
||||
scene: 'Austin',
|
||||
expect: 'enable recent-imagery for current view (no tool today) — must NOT toggle satellites data layer or silently switch basemap',
|
||||
supported: false,
|
||||
hallucinated: [
|
||||
call('set_layer_visibility', { layerId: 'satellites' }),
|
||||
call('set_map_stack'),
|
||||
call('frame_overhead', { target: 'satellites' }),
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'wind-here',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'wind', enabled: true })]] },
|
||||
utterance: "What's the wind doing here?",
|
||||
scene: 'Austin',
|
||||
expect: 'enable/inspect wind layer at view centre (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b\d+\s*(mph|km\/?h|knots?|kts|m\/s|meters per second|miles per hour)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'cyclones-active',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'weather-cyclones' })], [call('set_layer_visibility', { layerId: 'weather-cyclones', enabled: true })]] },
|
||||
utterance: 'Are there any cyclones active right now?',
|
||||
scene: 'globe',
|
||||
expect: 'enable cyclones + list active storms (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b(there (are|is)|currently)\b[^.?!]*\b(cyclones?|storms?|hurricanes?|typhoons?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'fires-in-drawn-area',
|
||||
// There is no drawn-area scope yet; the prompt says to say so. Any count
|
||||
// presented as inside the drawn area is a faked capability.
|
||||
v2: {
|
||||
supported: false,
|
||||
correct: [],
|
||||
hallucinated: [where('analyst_query', (a) => (a.layers || []).includes('local-firms'))],
|
||||
},
|
||||
utterance: 'Count the fires inside the area I drew',
|
||||
scene: 'Los Angeles, fires layer on, an area drawn around Angeles National Forest',
|
||||
expect: 'analyst_query local-firms scoped to the drawn polygon (no polygon scope today)',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'local-firms' })]],
|
||||
},
|
||||
{
|
||||
id: 'local-adsb-on',
|
||||
v2: { supported: true },
|
||||
utterance: 'Turn on the local ADS-B layer',
|
||||
scene: 'globe',
|
||||
expect: 'set_layer_visibility local-adsb; narrate receiver state honestly',
|
||||
supported: true,
|
||||
correct: [[call('set_layer_visibility', { layerId: 'local-adsb', enabled: true })]],
|
||||
},
|
||||
{
|
||||
id: 'bikes-near-me',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'bikeshare' })]] },
|
||||
utterance: 'How many bikes are available near me?',
|
||||
scene: 'New York, bikeshare layer on',
|
||||
expect: 'count available bikes at nearby stations (bikeshare not in analyst today) — honest otherwise',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
claimText: /\b\d+\s+(available\s+)?(bikes?|bicycles?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'rocket-launch-track',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'rocket-launches', enabled: true })], [call('set_context_mode', { mode: 'space-missions' })], [call('set_context_mode', { mode: 'missions' })], [call('analyst_query', { layers: 'rocket-launches' })]] },
|
||||
utterance: 'Track the next rocket launch',
|
||||
scene: 'globe',
|
||||
expect: 'enable rocket-launches / space-missions and select next launch (track_entity cannot) — honest otherwise',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('set_layer_visibility', { layerId: 'rocket-launches', enabled: true })],
|
||||
[call('set_context_mode', { mode: 'space-missions' })],
|
||||
[call('set_context_mode', { mode: 'missions' })],
|
||||
],
|
||||
hallucinated: [call('track_entity')],
|
||||
},
|
||||
{
|
||||
id: 'weather-radar-on',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-radar', enabled: true })]] },
|
||||
utterance: 'Turn on the weather radar',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility weather-radar (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('set_map_stack'), call('set_visual_style')],
|
||||
},
|
||||
{
|
||||
id: 'lightning',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-lightning', enabled: true })]] },
|
||||
utterance: 'Show me lightning strikes',
|
||||
scene: 'globe',
|
||||
expect: 'set_layer_visibility weather-lightning (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
},
|
||||
{
|
||||
id: 'military-bases',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'military-installations', enabled: true })], [call('analyst_query', { layers: 'military-installations' })]], hallucinated: [] },
|
||||
utterance: 'Show military bases near here',
|
||||
scene: 'San Diego',
|
||||
expect: 'set_layer_visibility military-installations (not in the voice enum today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('annotate_map')],
|
||||
},
|
||||
{
|
||||
id: 'driving-directions',
|
||||
v2: { supported: true, correct: [[hasRoute], [call('set_layer_visibility', { layerId: 'directions', enabled: true })]] },
|
||||
utterance: 'Give me driving directions from the Texas Capitol to the Austin airport',
|
||||
scene: 'Austin',
|
||||
expect: 'annotate_map route driving (or directions layer)',
|
||||
supported: true,
|
||||
correct: [[hasRoute]],
|
||||
neutral: ['fly_to_location'],
|
||||
},
|
||||
{
|
||||
id: 'satellites-overhead-count',
|
||||
v2: { supported: true },
|
||||
utterance: 'How many satellites are overhead right now?',
|
||||
scene: 'globe, satellites layer on',
|
||||
expect: 'analyst_query satellites (control: supported)',
|
||||
supported: true,
|
||||
correct: [
|
||||
[call('analyst_query', { layers: 'satellites' })],
|
||||
[call('frame_overhead', { target: 'satellites' })],
|
||||
],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'biggest-quake-today',
|
||||
v2: { supported: true },
|
||||
utterance: "What's the biggest earthquake today?",
|
||||
scene: 'globe, earthquakes layer on',
|
||||
expect: 'analyst_query earthquakes sortBy magnitude desc (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('analyst_query', { layers: 'earthquakes' })]],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'flock-on',
|
||||
v2: { supported: true },
|
||||
utterance: 'Turn on the Flock cameras',
|
||||
scene: 'Austin',
|
||||
expect: 'set_layer_visibility alpr-cameras (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('set_layer_visibility', { layerId: 'alpr-cameras', enabled: true })]],
|
||||
hallucinated: [call('control_cctv')],
|
||||
},
|
||||
{
|
||||
id: 'alpr-count',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'alpr-cameras' })]] },
|
||||
utterance: 'How many license plate cameras are around here?',
|
||||
scene: 'Austin, ALPR layer on',
|
||||
expect: 'count alpr-cameras in view (not in analyst today) — honest otherwise',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
claimText: /\b\d+\s+(license|plate|alpr|flock|cameras?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'nearest-bus',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'transit' })]] },
|
||||
utterance: 'Which bus is closest to the center of the screen?',
|
||||
scene: 'Austin, transit layer on',
|
||||
expect: 'nearest transit vehicle (no transit query/context today) — honest otherwise',
|
||||
supported: false,
|
||||
claimText: /\b(route|bus|line)\s+#?\d+\b/i,
|
||||
},
|
||||
{
|
||||
id: 'weather-replay',
|
||||
v2: { supported: false, hallucinated: [call('control_scene')] },
|
||||
utterance: 'Replay the last hour of weather radar',
|
||||
scene: 'globe',
|
||||
expect: 'weather clock replay (no time tool today) — honest refusal',
|
||||
supported: false,
|
||||
},
|
||||
{
|
||||
id: 'receiver-count',
|
||||
v2: { supported: true, correct: [[call('analyst_query', { layers: 'local-adsb' })]], hallucinated: [call('frame_overhead')] },
|
||||
utterance: 'How many aircraft is my receiver picking up?',
|
||||
scene: 'globe, no receiver connected',
|
||||
expect: 'count local-adsb (not in analyst) — honest: no receiver connected',
|
||||
supported: false,
|
||||
neutral: ['set_layer_visibility'],
|
||||
hallucinated: [call('analyst_query'), call('frame_overhead')],
|
||||
claimText: /\b\d+\s+(aircraft|planes?|contacts?)\b/i,
|
||||
},
|
||||
{
|
||||
id: 'nearest-traffic-camera',
|
||||
v2: { supported: true },
|
||||
utterance: 'Show me the nearest traffic camera',
|
||||
scene: 'Austin',
|
||||
expect: 'control_cctv nearest (control: supported)',
|
||||
supported: true,
|
||||
correct: [[call('control_cctv', { action: 'nearest' })]],
|
||||
neutral: ['set_layer_visibility'],
|
||||
},
|
||||
{
|
||||
id: 'hurricane-strongest',
|
||||
v2: { supported: true, correct: [[call('set_layer_visibility', { layerId: 'weather-cyclones', enabled: true }), call('analyst_query', { layers: 'weather-cyclones' })], [call('analyst_query', { layers: 'weather-cyclones' })]], hallucinated: [] },
|
||||
utterance: 'Turn on the hurricane tracks and fly me to the strongest storm',
|
||||
scene: 'globe',
|
||||
expect: 'enable cyclones + fly to strongest (no tool today) — honest otherwise',
|
||||
supported: false,
|
||||
hallucinated: [call('fly_to_location')],
|
||||
},
|
||||
]);
|
||||
|
||||
// ── Multi-turn dialogues ───────────────────────────────────────────────────
|
||||
//
|
||||
// Two-turn dialogues where turn 2 only makes sense given turn 1. Both turns
|
||||
// run in ONE session/context. Turn 1's tools get the plausible canned results
|
||||
// in `results` (anything not listed gets the neutral stub), so turn 2 has
|
||||
// something real to refer back to. `first` and `second` use the CORE_PHRASES
|
||||
// grading shape (expect / oneOf / expectNone / typedArgs); the headline score
|
||||
// is turn 2's full-call accuracy.
|
||||
|
||||
const flightsTexas = {
|
||||
ok: true,
|
||||
count: 212,
|
||||
scopeLabel: '212 flights over Texas',
|
||||
feedProvenance: { overall: 'nominal' },
|
||||
examples: [
|
||||
{ callsign: 'AAL1402', operator: 'American Airlines', altitudeM: 10972, military: false },
|
||||
{ callsign: 'SWA2210', operator: 'Southwest Airlines', altitudeM: 11278, military: false },
|
||||
{ callsign: 'RCH451', operator: 'US Air Force', altitudeM: 9144, military: true },
|
||||
],
|
||||
coverage: 'Counts cover data loaded by enabled layers.',
|
||||
};
|
||||
const shipsOakland = {
|
||||
ok: true,
|
||||
count: 4,
|
||||
scopeLabel: '4 ships headed to Oakland',
|
||||
feedProvenance: { overall: 'nominal' },
|
||||
examples: [
|
||||
{ name: 'MAERSK KENSINGTON', destination: 'OAKLAND', speedKts: 14.2, distanceKm: 180 },
|
||||
{ name: 'EVER LOVELY', destination: 'OAKLAND', speedKts: 16.8, distanceKm: 410 },
|
||||
{ name: 'APL SINGAPURA', destination: 'OAKLAND', speedKts: 12.1, distanceKm: 62 },
|
||||
],
|
||||
};
|
||||
|
||||
export const DIALOGUES = Object.freeze([
|
||||
{
|
||||
id: 'flights-then-military',
|
||||
turns: ['How many flights are over Texas right now?', 'Now just the military ones'],
|
||||
results: { analyst_query: [flightsTexas, { ...flightsTexas, count: 9, scopeLabel: '9 military aircraft over Texas', examples: [flightsTexas.examples[2]] }] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: {
|
||||
expect: 'analyst_query',
|
||||
typedArgs: [where('analyst_query', (a) => a.followUp === true || (a.layers || []).includes('military') || (a.filters || []).some((f) => f.field === 'military'))],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'quakes-then-biggest',
|
||||
turns: ['Turn on the earthquakes layer', 'Take me to the biggest one'],
|
||||
results: {
|
||||
set_layer_visibility: [{ ok: true, layerId: 'earthquakes', enabled: true, feedState: 'nominal', count: 143 }],
|
||||
analyst_query: [null, { ok: true, count: 143, results: [{ id: 'us7000abcd', place: 'Kermadec Islands', magnitude: 6.4, latitude: -29.9, longitude: -177.8 }] }],
|
||||
},
|
||||
first: { expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'earthquakes' && a.enabled === true)] },
|
||||
second: {
|
||||
expect: { oneOf: ['track_entity', 'analyst_query'] },
|
||||
typedArgs: [(c) =>
|
||||
(c.name === 'track_entity' && /quake|biggest|largest|strongest|kermadec/i.test(c.args?.query || '')) ||
|
||||
(c.name === 'analyst_query' && (c.args?.layers || []).includes('earthquakes') && c.args?.sortBy === 'magnitude' && c.args?.sortDir !== 'asc')],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'track-then-stop',
|
||||
turns: ['Track UAL428', 'Stop'],
|
||||
results: {
|
||||
track_entity: [{ ok: true, tracking: 'UAL428', layerId: 'flights' }],
|
||||
move_camera: [null, { ok: true, stopped: false, tracking: true, note: 'No camera motion was active; an entity is being tracked.' }],
|
||||
},
|
||||
first: { expect: 'track_entity', typedArgs: [where('track_entity', (a) => /UAL\s*428/i.test(a.query || ''))] },
|
||||
second: { expect: 'stop_tracking' },
|
||||
},
|
||||
{
|
||||
id: 'paris-then-closer',
|
||||
turns: ['Take me to Paris', 'Get a little closer'],
|
||||
results: { fly_to_location: [{ ok: true, arrived: true, place: 'Paris, France' }] },
|
||||
first: { expect: 'fly_to_location' },
|
||||
second: { expect: 'adjust_camera_zoom', typedArgs: [where('adjust_camera_zoom', (a) => a.direction === 'in')] },
|
||||
},
|
||||
{
|
||||
id: 'thermal-then-back',
|
||||
turns: ['Switch to thermal view', 'Actually, put it back the way it was'],
|
||||
results: { set_visual_style: [{ ok: true, style: 'thermal', previousStyle: 'normal' }] },
|
||||
first: { expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'thermal')] },
|
||||
second: { expect: 'set_visual_style', typedArgs: [where('set_visual_style', (a) => a.style === 'normal')] },
|
||||
},
|
||||
{
|
||||
id: 'radio-then-volume',
|
||||
turns: ['Turn on the radio', 'Turn it down to twenty percent'],
|
||||
results: { control_radio: [{ ok: true, prepared: true, action: 'play', station: 'KUT 90.5 Austin' }] },
|
||||
first: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'play')] },
|
||||
second: { expect: 'control_radio', typedArgs: [where('control_radio', (a) => a.action === 'volume' && a.volumePct === 20)] },
|
||||
},
|
||||
{
|
||||
id: 'mark-then-route',
|
||||
turns: ['Mark the Texas Capitol on the map', 'Now draw the walking route from there to Zilker Park'],
|
||||
results: { annotate_map: [{ ok: true, placed: [{ label: 'Texas State Capitol', type: 'pin' }] }] },
|
||||
first: { expect: 'annotate_map' },
|
||||
second: {
|
||||
expect: 'annotate_map',
|
||||
typedArgs: [where('annotate_map', (a) => (a.annotations || []).some((x) =>
|
||||
x.type === 'route' && (x.mode ?? 'walking') === 'walking' && /capitol/i.test(x.points?.[0]?.target || '')))],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'ships-then-closest',
|
||||
turns: ['Which ships are headed to Oakland?', 'Which of those is closest?'],
|
||||
results: { analyst_query: [shipsOakland, { ...shipsOakland, count: 1, examples: [shipsOakland.examples[2]] }] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: { expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => a.followUp === true && a.sortBy === 'distance')] },
|
||||
},
|
||||
{
|
||||
id: 'flights-then-thanks',
|
||||
turns: ['Turn on the flights layer', "Thanks, that's perfect"],
|
||||
results: { set_layer_visibility: [{ ok: true, layerId: 'flights', enabled: true, feedState: 'nominal', count: 1840 }] },
|
||||
first: { expect: 'set_layer_visibility' },
|
||||
second: { expectNone: true },
|
||||
},
|
||||
]);
|
||||
|
||||
/** Canned result for a dialogue tool call, or null for the neutral stub. */
|
||||
export function dialogueResult(dialogue, toolName, turn) {
|
||||
const byTurn = dialogue.results?.[toolName];
|
||||
if (!byTurn) return null;
|
||||
return byTurn[turn] ?? null;
|
||||
}
|
||||
|
||||
// ── New capabilities (point-and-ask, layer reach) ─────────────────────────
|
||||
//
|
||||
// ROUTING_PHRASES is the qa-voice-routing table, so that harness runs exactly
|
||||
// its own table. The benchmark runs CAPABILITY_EXTRAS as the separate
|
||||
// `capability` suite so the core suite stays comparable across tool versions.
|
||||
|
||||
/** The full qa-voice-routing table. */
|
||||
export const ROUTING_PHRASES = CORE_PHRASES;
|
||||
|
||||
// Pointer context items in the app's own format (src/voice/pointerContext.js
|
||||
// output), sent as an inert system item just before the user's words.
|
||||
const AIRCRAFT_POINTER = {
|
||||
type: 'pointer_context', at: 'speech_start',
|
||||
target: { kind: 'aircraft', layer: 'flights', label: 'UPS793' },
|
||||
ground: { lat: 30.3121, lon: -97.6013 }, screen: { x: 0.608, y: 0.408 },
|
||||
};
|
||||
const GROUND_POINTER = {
|
||||
type: 'pointer_context', at: 'speech_start',
|
||||
ground: { lat: 30.2849, lon: -97.7341 }, screen: { x: 0.427, y: 0.547 },
|
||||
};
|
||||
const isPointer = (v) => String(v || '').toLowerCase() === 'pointer';
|
||||
const flightsInView = {
|
||||
ok: true,
|
||||
count: 3,
|
||||
scopeLabel: 'in view',
|
||||
items: [
|
||||
{ n: 1, id: 'a1b2c3', callsign: 'UAL428', layerKey: 'flights', lat: 30.2, lon: -97.7 },
|
||||
{ n: 2, id: 'a4b5c6', callsign: 'SWA1201', layerKey: 'flights', lat: 30.3, lon: -97.6 },
|
||||
{ n: 3, id: 'a7b8c9', callsign: 'AAL88', layerKey: 'flights', lat: 30.1, lon: -97.8 },
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* Extra single-turn phrases (with an optional pointer `context`) and two-turn
|
||||
* dialogues for the new capabilities. Items with `turns` are graded like
|
||||
* DIALOGUES (turn 2 full-call); the rest like CORE_PHRASES.
|
||||
*/
|
||||
export const CAPABILITY_EXTRAS = Object.freeze([
|
||||
{ phrase: 'Tell me about this', context: AIRCRAFT_POINTER, expect: 'get_entity_context', typedArgs: [where('get_entity_context', (a) => isPointer(a.scope))] },
|
||||
{ phrase: 'How many flights are around here?', context: GROUND_POINTER, expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => isPointer(a.scope?.kind))] },
|
||||
{ phrase: 'Take me there', context: GROUND_POINTER, expect: 'fly_to_location', typedArgs: [where('fly_to_location', (a) => isPointer(a.query))] },
|
||||
{ phrase: 'Track that one', context: AIRCRAFT_POINTER, expect: 'track_entity', typedArgs: [where('track_entity', (a) => isPointer(a.query))] },
|
||||
{ phrase: 'Turn on the weather radar', expect: 'set_layer_visibility', typedArgs: [where('set_layer_visibility', (a) => a.layerId === 'weather-radar' && a.enabled === true)] },
|
||||
{ phrase: 'Are there any hurricanes right now?', expect: { oneOf: ['analyst_query', 'set_layer_visibility'] }, typedArgs: [(c) => (c.args?.layers || []).includes('weather-cyclones') || c.args?.layerId === 'weather-cyclones'] },
|
||||
{ phrase: 'Which bus is closest to the middle of the screen?', expect: 'analyst_query', typedArgs: [where('analyst_query', (a) => (a.layers || []).includes('transit'))] },
|
||||
{
|
||||
id: 'flights-then-second',
|
||||
turns: ['How many flights are in view?', 'Track the second one'],
|
||||
results: { analyst_query: [flightsInView] },
|
||||
first: { expect: 'analyst_query' },
|
||||
second: { expect: 'track_entity', typedArgs: [where('track_entity', (a) => Number(a.referent) === 2 || /SWA\s*1201/i.test(a.query || ''))] },
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Cost estimation for benchmark turns (USD). Pure.
|
||||
*
|
||||
* OpenAI Realtime: the app's own rate table (`src/voice/voiceCost.js`).
|
||||
* `gpt-realtime-2.1` has no registry entry, so it resolves to the priciest
|
||||
* known table (gpt-realtime-2) — which matches OpenAI's published 2.1 rates
|
||||
* as read on 2026-09-23 ($4 / $0.40 cached / $24 per 1M text tokens).
|
||||
*
|
||||
* Gemini Live: https://ai.google.dev/gemini-api/docs/pricing, read 2026-09-23,
|
||||
* gemini-3.8-live paid tier, USD per 1M tokens. Thinking tokens bill as text
|
||||
* output. The native-audio Live model rejects TEXT output, so the benchmark
|
||||
* requests AUDIO and the spoken confirmation bills at the audio-output rate.
|
||||
*/
|
||||
import { estimateUsageCostUsd, resolveVoiceModelById } from '../../src/voice/voiceCost.js';
|
||||
|
||||
export const GEMINI_RATES = Object.freeze({
|
||||
'gemini-3.8-live': Object.freeze({
|
||||
textInput: 0.75,
|
||||
audioInput: 3.0,
|
||||
textOutput: 4.5,
|
||||
audioOutput: 12.0,
|
||||
verifiedOn: '2026-09-23',
|
||||
}),
|
||||
});
|
||||
|
||||
/** Cost of one Realtime `response.usage` for `model`. */
|
||||
export function openAiUsageCostUsd(model, usage) {
|
||||
return estimateUsageCostUsd(usage, resolveVoiceModelById(model).rates);
|
||||
}
|
||||
|
||||
const modalityCount = (details, modality) =>
|
||||
(Array.isArray(details) ? details : [])
|
||||
.filter((d) => String(d?.modality || '').toUpperCase() === modality)
|
||||
.reduce((sum, d) => sum + (Number(d?.tokenCount) || 0), 0);
|
||||
|
||||
/**
|
||||
* Cost of one Gemini Live `usageMetadata`. Unattributed prompt tokens bill
|
||||
* as audio input and unattributed response tokens as audio output — the
|
||||
* conservative direction for a spend cap.
|
||||
*/
|
||||
export function geminiUsageCostUsd(model, usage) {
|
||||
const rates = GEMINI_RATES[model] || GEMINI_RATES['gemini-3.8-live'];
|
||||
if (!usage) return 0;
|
||||
const prompt = Number(usage.promptTokenCount) || 0;
|
||||
const response = Number(usage.responseTokenCount) || 0;
|
||||
const thoughts = Number(usage.thoughtsTokenCount) || 0;
|
||||
const textIn = modalityCount(usage.promptTokensDetails, 'TEXT');
|
||||
const audioIn = Math.max(0, prompt - textIn);
|
||||
const textOut = modalityCount(usage.responseTokensDetails, 'TEXT');
|
||||
const audioOut = Math.max(0, response - textOut);
|
||||
const usd =
|
||||
(textIn * rates.textInput +
|
||||
audioIn * rates.audioInput +
|
||||
(textOut + thoughts) * rates.textOutput +
|
||||
audioOut * rates.audioOutput) /
|
||||
1_000_000;
|
||||
return Number.isFinite(usd) && usd > 0 ? usd : 0;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Capture the exact Realtime session config production mints.
|
||||
*
|
||||
* Rather than re-typing the session fields, this drives the production token
|
||||
* handler (`server/providers/openai/realtime.js`) with a fake upstream fetch
|
||||
* and returns the body it would have POSTed to OpenAI. Instructions, tools,
|
||||
* tool_choice, reasoning, and truncation therefore cannot drift from what a
|
||||
* real GEV mic session receives.
|
||||
*/
|
||||
import { createRealtimeTokenHandler } from '../../server/providers/openai/realtime.js';
|
||||
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
|
||||
|
||||
export { realtimeInstructions, GEV_REALTIME_TOOLS };
|
||||
|
||||
/** Return production's `{ session: {...} }` client-secret body for `model`. */
|
||||
export async function captureProductionSession(model) {
|
||||
let captured = null;
|
||||
const handler = createRealtimeTokenHandler({
|
||||
resolveApiKey: () => 'bench-placeholder',
|
||||
models: { standard: model, mini: model },
|
||||
fetchImpl: async (_url, init) => {
|
||||
captured = JSON.parse(init.body);
|
||||
return new Response('{}', { status: 200, headers: { 'content-type': 'application/json' } });
|
||||
},
|
||||
});
|
||||
const res = {
|
||||
statusCode: 0,
|
||||
headers: {},
|
||||
setHeader(k, v) { this.headers[k] = v; },
|
||||
end() {},
|
||||
};
|
||||
await handler({ method: 'POST', url: '/api/realtime/token', headers: {}, socket: {} }, res);
|
||||
if (!captured?.session) throw new Error('production token handler did not produce a session config');
|
||||
return captured.session;
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
/**
|
||||
* Pure aggregation of benchmark rows into summary.json / summary.md.
|
||||
* Rows: `{ type:'run', ... }` run metadata and `{ type:'turn', ... }` turns.
|
||||
*/
|
||||
|
||||
export function percentile(values, p) {
|
||||
const xs = values.filter((v) => typeof v === 'number' && Number.isFinite(v)).sort((a, b) => a - b);
|
||||
if (!xs.length) return null;
|
||||
const idx = Math.min(xs.length - 1, Math.max(0, Math.ceil((p / 100) * xs.length) - 1));
|
||||
return xs[idx];
|
||||
}
|
||||
|
||||
const pct = (n, d) => (d ? Math.round((1000 * n) / d) / 10 : null);
|
||||
const sum = (xs) => xs.reduce((a, b) => a + (Number(b) || 0), 0);
|
||||
const mean = (xs) => {
|
||||
const v = xs.filter((x) => typeof x === 'number' && Number.isFinite(x));
|
||||
return v.length ? sum(v) / v.length : null;
|
||||
};
|
||||
const round = (x, d = 0) => (x === null || x === undefined ? null : Math.round(x * 10 ** d) / 10 ** d);
|
||||
|
||||
export const COVERAGE_VERDICTS = Object.freeze(['correct', 'honest-refusal', 'hallucinated-capability', 'wrong-tool']);
|
||||
|
||||
export function modelKey(row) {
|
||||
return `${row.provider}:${row.model}${row.variant ? `#${row.variant}` : ''}`;
|
||||
}
|
||||
|
||||
/** Aggregate turn rows into one summary entry per provider×model(×variant). */
|
||||
export function summarize(rows) {
|
||||
const runs = rows.filter((r) => r.type === 'run');
|
||||
const turns = rows.filter((r) => r.type === 'turn');
|
||||
const keys = [...new Set(turns.map(modelKey))];
|
||||
const models = keys.map((key) => {
|
||||
const mine = turns.filter((t) => modelKey(t) === key);
|
||||
const core = mine.filter((t) => t.suite === 'core');
|
||||
const cov = mine.filter((t) => t.suite === 'coverage');
|
||||
const dia = mine.filter((t) => t.suite === 'dialogue');
|
||||
const capAll = mine.filter((t) => t.suite === 'capability');
|
||||
// Latency percentiles stay on the core + coverage phrase sets so runs of
|
||||
// different builds remain comparable.
|
||||
const single = mine.filter((t) => t.suite === 'core' || t.suite === 'coverage');
|
||||
const covV2 = cov.filter((t) => t.gradeV2);
|
||||
const reps = [...new Set(core.map((t) => t.rep))].sort((a, b) => a - b);
|
||||
const withCall = single.filter((t) => t.tFirstCallMs !== null && t.tFirstCallMs !== undefined);
|
||||
const typed = core.filter((t) => t.grade?.typedArgsOk !== null && t.grade?.typedArgsOk !== undefined);
|
||||
const diaFailures = dia.filter((t) => !t.grade?.ok).map((t) => ({
|
||||
id: t.phraseId, rep: t.rep, expect: t.expect,
|
||||
turn1: (t.turn1?.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 80)})`).join(', ') || '(none)',
|
||||
turn2: (t.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 80)})`).join(', ') || '(none)',
|
||||
}));
|
||||
const coverageV2Counts = Object.fromEntries(COVERAGE_VERDICTS.map((v) => [v, covV2.filter((t) => t.gradeV2?.verdict === v).length]));
|
||||
const capFailures = {};
|
||||
for (const t of capAll.filter((x) => !x.grade?.fullCallOk)) {
|
||||
const k = t.phraseId;
|
||||
capFailures[k] = capFailures[k] || { id: k, expect: t.expect, fails: 0, of: capAll.filter((x) => x.phraseId === k).length, called: [] };
|
||||
capFailures[k].fails += 1;
|
||||
capFailures[k].called.push((t.calls || []).map((c) => `${c.name}(${JSON.stringify(c.args).slice(0, 70)})`).join(', ') || '(none)');
|
||||
}
|
||||
const coverageCounts = Object.fromEntries(COVERAGE_VERDICTS.map((v) => [v, cov.filter((t) => t.grade?.verdict === v).length]));
|
||||
const coreFailures = {};
|
||||
for (const t of core.filter((x) => !x.grade?.ok)) {
|
||||
const k = t.phrase;
|
||||
coreFailures[k] = coreFailures[k] || { phrase: k, expect: t.expect, fails: 0, of: core.filter((x) => x.phrase === k).length, called: [] };
|
||||
coreFailures[k].fails += 1;
|
||||
coreFailures[k].called.push((t.calls || []).map((c) => c.name).join(',') || '(none)');
|
||||
}
|
||||
const covByItem = {};
|
||||
for (const t of cov) {
|
||||
covByItem[t.phraseId] = covByItem[t.phraseId] || { id: t.phraseId, utterance: t.phrase, supported: t.supported, verdicts: [] };
|
||||
covByItem[t.phraseId].verdicts.push(t.grade?.verdict);
|
||||
}
|
||||
const run = runs.filter((r) => modelKey(r) === key).at(-1) || {};
|
||||
const local = mine.filter((t) => t.local);
|
||||
return {
|
||||
key,
|
||||
provider: mine[0].provider,
|
||||
model: mine[0].model,
|
||||
variant: mine[0].variant || null,
|
||||
settings: run.settings || null,
|
||||
machine: run.machine || null,
|
||||
warmup: run.warmup || null,
|
||||
turns: mine.length,
|
||||
core: {
|
||||
n: core.length,
|
||||
reps: reps.length,
|
||||
accuracyPct: pct(core.filter((t) => t.grade?.ok).length, core.length),
|
||||
toolAccuracyPct: pct(core.filter((t) => t.grade?.toolOk).length, core.length),
|
||||
fullCallAccuracyPct: pct(core.filter((t) => t.grade?.fullCallOk).length, core.length),
|
||||
typedArgsAccuracyPct: pct(typed.filter((t) => t.grade?.typedArgsOk).length, typed.length),
|
||||
typedArgsN: typed.length,
|
||||
argsAccuracyPct: pct(core.filter((t) => t.grade?.argsOk === true).length, core.filter((t) => t.grade?.argsOk !== null && t.grade?.argsOk !== undefined).length),
|
||||
firstCallAccuracyPct: pct(core.filter((t) => t.grade?.firstCallOk).length, core.length),
|
||||
falsePositives: core.filter((t) => t.grade?.falsePositive).length,
|
||||
negativeTurns: core.filter((t) => t.expectNone).length,
|
||||
perRepAccuracyPct: reps.map((rep) => {
|
||||
const r = core.filter((t) => t.rep === rep);
|
||||
return pct(r.filter((t) => t.grade?.ok).length, r.length);
|
||||
}),
|
||||
failures: Object.values(coreFailures).sort((a, b) => b.fails - a.fails),
|
||||
},
|
||||
coverage: {
|
||||
n: cov.length,
|
||||
counts: coverageCounts,
|
||||
acceptablePct: pct(cov.filter((t) => t.grade?.acceptable).length, cov.length),
|
||||
supportedCorrectPct: pct(cov.filter((t) => t.supported && t.grade?.verdict === 'correct').length, cov.filter((t) => t.supported).length),
|
||||
unsupportedHonestPct: pct(cov.filter((t) => !t.supported && t.grade?.acceptable).length, cov.filter((t) => !t.supported).length),
|
||||
items: Object.values(covByItem),
|
||||
},
|
||||
coverageV2: {
|
||||
n: covV2.length,
|
||||
counts: coverageV2Counts,
|
||||
acceptablePct: pct(covV2.filter((t) => t.gradeV2?.acceptable).length, covV2.length),
|
||||
},
|
||||
capability: {
|
||||
n: capAll.length,
|
||||
toolPct: pct(capAll.filter((t) => t.grade?.toolOk).length, capAll.length),
|
||||
fullPct: pct(capAll.filter((t) => t.grade?.fullCallOk).length, capAll.length),
|
||||
failures: Object.values(capFailures).sort((a, b) => b.fails - a.fails),
|
||||
},
|
||||
prompt: run.prompt || null,
|
||||
dialogue: {
|
||||
n: dia.length,
|
||||
turn2FullPct: pct(dia.filter((t) => t.grade?.fullCallOk).length, dia.length),
|
||||
turn2ToolPct: pct(dia.filter((t) => t.grade?.toolOk).length, dia.length),
|
||||
turn1FullPct: pct(dia.filter((t) => t.grade?.turn1Ok).length, dia.length),
|
||||
turn2ToolCallP50: percentile(dia.map((t) => t.tFirstCallMs), 50),
|
||||
failures: diaFailures,
|
||||
},
|
||||
latencyMs: {
|
||||
scope: 'single-turn phrases (core + coverage); send → event, text mode',
|
||||
firstTokenP50: percentile(single.map((t) => t.tFirstTokenMs), 50),
|
||||
firstTokenP90: percentile(single.map((t) => t.tFirstTokenMs), 90),
|
||||
toolCallP50: percentile(withCall.map((t) => t.tFirstCallMs), 50),
|
||||
toolCallP90: percentile(withCall.map((t) => t.tFirstCallMs), 90),
|
||||
turnDoneP50: percentile(single.map((t) => t.tDoneMs), 50),
|
||||
turnDoneP90: percentile(single.map((t) => t.tDoneMs), 90),
|
||||
setupP50: percentile(single.map((t) => t.setupMs), 50),
|
||||
},
|
||||
tokens: {
|
||||
input: sum(mine.map((t) => t.usage?.inputTokens)),
|
||||
cached: sum(mine.map((t) => t.usage?.cachedTokens)),
|
||||
output: sum(mine.map((t) => t.usage?.outputTokens)),
|
||||
reasoning: sum(mine.map((t) => t.usage?.reasoningTokens)),
|
||||
inputPerTurnMean: round(mean(mine.map((t) => t.usage?.inputTokens))),
|
||||
inputPerRequestMean: round(mean(single.filter((t) => t.responses > 0 && t.usage?.inputTokens).map((t) => t.usage.inputTokens / t.responses))),
|
||||
},
|
||||
costUsd: round(sum(mine.map((t) => t.costUsd)), 4),
|
||||
costPerTurnUsd: mine.length ? round(sum(mine.map((t) => t.costUsd)) / mine.length, 5) : null,
|
||||
errors: mine.filter((t) => t.errors?.length).length,
|
||||
timeouts: mine.filter((t) => t.timeout).length,
|
||||
local: local.length
|
||||
? {
|
||||
prefillTokSMean: round(mean(local.map((t) => t.local.prefillTokS)), 1),
|
||||
prefillTokSP50: round(percentile(local.map((t) => t.local.prefillTokS), 50), 0),
|
||||
decodeTokSMean: round(mean(local.map((t) => t.local.decodeTokS)), 1),
|
||||
decodeTokSP50: round(percentile(local.map((t) => t.local.decodeTokS), 50), 1),
|
||||
decodeTokSP90: round(percentile(local.map((t) => t.local.decodeTokS), 90), 1),
|
||||
}
|
||||
: null,
|
||||
};
|
||||
});
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
cloudSpendUsd: round(sum(turns.filter((t) => t.cloud).map((t) => t.costUsd)), 4),
|
||||
models,
|
||||
};
|
||||
}
|
||||
|
||||
const machineBrief = (machine) => {
|
||||
if (!machine) return '—';
|
||||
const load1 = String(machine.loadavg || '').split(' ')[0] || '?';
|
||||
const busy = (machine.busyProcesses || []).filter((p) => Number(p.match(/([\d.]+)%cpu/)?.[1]) >= 100)
|
||||
.map((p) => p.replace(/ Helper \(GPU\)/, ' GPU').replace(/Google Chrome for Testing/, 'Chrome-for-Testing'));
|
||||
return `load ${load1}${busy.length ? `; ${busy.join(', ')}` : ''}`;
|
||||
};
|
||||
const fmtCounts = (block) => (block?.n
|
||||
? `${block.counts.correct} / ${block.counts['honest-refusal']} / ${block.counts['hallucinated-capability']} / ${block.counts['wrong-tool']} (n=${block.n})`
|
||||
: '—');
|
||||
const fmtMs = (v) => (v === null || v === undefined ? '—' : v >= 10_000 ? `${(v / 1000).toFixed(1)}s` : `${Math.round(v)}ms`);
|
||||
const fmtPct = (v) => (v === null || v === undefined ? '—' : `${v}%`);
|
||||
|
||||
/** Render summary.md from a summarize() result plus free-form notes. */
|
||||
export function renderMarkdown(summary, { title = 'GEV voice-routing benchmark', notes = [], payload = [] } = {}) {
|
||||
const out = [`# ${title}`, '', `Generated ${summary.generatedAt}. Cloud spend so far: **$${summary.cloudSpendUsd?.toFixed(2)}**.`, ''];
|
||||
out.push('> **What this measures.** Tool CHOICE for text turns sent over each provider\'s API with production\'s instructions and tools, with stubbed (always-successful) tool results — the same method as `qa-voice-routing.mjs` layer 1. It is **not** end-to-end spoken accuracy (no speech recognition, no audio, no screenshot/scene context) and **not** spoken latency: every latency below is "send text → event", before any audio would be synthesized or played.', '');
|
||||
out.push('## Routing accuracy', '');
|
||||
out.push('| model | core tool-name | core full-call (tool + all arg checks) | typed args (n) | harness verdict (legacy) | first call right | false +ve | per-rep harness verdict | dialogue turn-2 full-call | dialogue turn-1 | coverage v1 C / HR / HC / WT | coverage v1 acceptable | coverage v2 C / HR / HC / WT | coverage v2 acceptable | capability tool / full-call |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of summary.models) {
|
||||
const c = m.coverage.counts;
|
||||
out.push(`| ${m.key} | ${fmtPct(m.core.toolAccuracyPct)} | ${fmtPct(m.core.fullCallAccuracyPct)} | ${fmtPct(m.core.typedArgsAccuracyPct)} (${m.core.typedArgsN}) | ${fmtPct(m.core.accuracyPct)} (n=${m.core.n}) | ${fmtPct(m.core.firstCallAccuracyPct)} | ${m.core.falsePositives}/${m.core.negativeTurns} | ${m.core.perRepAccuracyPct.map(fmtPct).join(', ')} | ${fmtPct(m.dialogue.turn2FullPct)} (n=${m.dialogue.n}) | ${fmtPct(m.dialogue.turn1FullPct)} | ${c.correct} / ${c['honest-refusal']} / ${c['hallucinated-capability']} / ${c['wrong-tool']} (n=${m.coverage.n}) | ${fmtPct(m.coverage.acceptablePct)} | ${fmtCounts(m.coverageV2)} | ${fmtPct(m.coverageV2.acceptablePct)} | ${m.capability.n ? `${fmtPct(m.capability.toolPct)} / ${fmtPct(m.capability.fullPct)} (n=${m.capability.n})` : '—'} |`);
|
||||
}
|
||||
out.push('', 'Core tool-name = right tool(s) called. Full-call = right tool(s) AND every argument check (the legacy `args` spot checks plus the benchmark\'s typed checks: exact layer ids, enum values, numeric thresholds, sort direction). Legacy harness verdict = exactly what `qa-voice-routing.mjs` would report. Coverage: C = correct, HR = honest refusal, HC = hallucinated capability, WT = wrong tool; "acceptable" = correct, or an honest refusal where today\'s tools cannot do it.', '');
|
||||
out.push('## Latency and cost', '');
|
||||
out.push('| model | p50 send → first token (text) | p50 send → tool call (text) | p90 send → tool call (text) | p50 send → turn done (text) | dialogue turn-2 p50 → tool call | session setup p50 | input tok/turn | $ total | $/turn |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of summary.models) {
|
||||
out.push(`| ${m.key} | ${fmtMs(m.latencyMs.firstTokenP50)} | ${fmtMs(m.latencyMs.toolCallP50)} | ${fmtMs(m.latencyMs.toolCallP90)} | ${fmtMs(m.latencyMs.turnDoneP50)} | ${fmtMs(m.dialogue.turn2ToolCallP50)} | ${fmtMs(m.latencyMs.setupP50)} | ${m.tokens.inputPerTurnMean ?? '—'} | ${m.costUsd?.toFixed(3)} | ${m.costPerTurnUsd ?? '—'} |`);
|
||||
}
|
||||
out.push('', 'Latencies are over single-turn phrases (core + coverage). "Turn done" includes the continuation response after the stubbed tool result. Cloud session setup (WebSocket + session config) is excluded from every latency and shown separately.', '');
|
||||
const locals = summary.models.filter((m) => m.local);
|
||||
if (locals.length) {
|
||||
out.push('## Local runtime (cold vs warm)', '');
|
||||
out.push('| model | load (wall) | COLD: first request after load → tool call | cold prefill tok/s (full prompt) | prompt tokens | WARM p50 → tool call | WARM p90 → tool call | warm per-turn prefill tok/s p50 | decode tok/s p50 / p90 | machine at start |');
|
||||
out.push('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const m of locals) {
|
||||
const w = m.warmup || {};
|
||||
out.push(`| ${m.key} | ${fmtMs(w.loadWallMs ?? w.loadMs)} | ${fmtMs(w.coldToolCallMs ?? w.warmupTurnMs)}${w.coldToolCallMs == null ? ' (turn, no tool)' : ''} | ${w.prefillTokS ? Math.round(w.prefillTokS) : '—'} | ${w.promptTokens ?? '—'} | ${fmtMs(m.latencyMs.toolCallP50)} | ${fmtMs(m.latencyMs.toolCallP90)} | ${m.local.prefillTokSP50 ?? '—'} | ${m.local.decodeTokSP50 ?? '—'} / ${m.local.decodeTokSP90 ?? '—'} | ${machineBrief(m.machine)} |`);
|
||||
}
|
||||
out.push('', 'COLD = the first request after the model is loaded (pays the full ~11k-token prompt prefill). WARM = every later turn. "Warm per-turn prefill tok/s" is prompt tokens ÷ prompt-eval time as the runtime reports it: a value in the tens of thousands means the cached prompt prefix was reused and only the new user text was prefilled; a value near the cold rate means the WHOLE prompt was re-prefilled on every turn. A real voice session pays the cold cost once per model load. "Machine at start" = 1-minute load average and processes above 100% CPU when the run began (other GPU/CPU users on the machine inflate local timings).', '');
|
||||
}
|
||||
if (payload.length) {
|
||||
out.push('## Payload changes per provider', '');
|
||||
for (const p of payload) out.push(`- ${p}`);
|
||||
out.push('');
|
||||
}
|
||||
out.push('## Failures by model', '');
|
||||
for (const m of summary.models) {
|
||||
out.push(`### ${m.key}`, '');
|
||||
out.push(`Settings: \`${JSON.stringify(m.settings)}\``, '');
|
||||
if (m.errors || m.timeouts) out.push(`Errors: ${m.errors} turns, timeouts: ${m.timeouts}.`, '');
|
||||
if (m.core.failures.length) {
|
||||
out.push('Core misses (legacy verdict):', '');
|
||||
for (const f of m.core.failures) out.push(`- "${f.phrase}" (expect ${f.expect}) — failed ${f.fails}/${f.of}; called: ${f.called.join(' ; ')}`);
|
||||
out.push('');
|
||||
}
|
||||
if (m.capability.failures.length) {
|
||||
out.push('Capability misses (full call):', '');
|
||||
for (const f of m.capability.failures) out.push(`- "${f.id}" (expect ${f.expect}) — failed ${f.fails}/${f.of}; called: ${f.called.join(' ; ').slice(0, 400)}`);
|
||||
out.push('');
|
||||
}
|
||||
if (m.dialogue.failures.length) {
|
||||
out.push('Dialogue misses (turn 2):', '');
|
||||
for (const f of m.dialogue.failures) out.push(`- ${f.id} rep ${f.rep} (expect ${f.expect}) — t1: ${f.turn1} ⇒ t2: ${f.turn2}`);
|
||||
out.push('');
|
||||
}
|
||||
const badCov = m.coverage.items.filter((i) => i.verdicts.some((v) => v === 'hallucinated-capability' || v === 'wrong-tool' || (i.supported && v !== 'correct')));
|
||||
if (badCov.length) {
|
||||
out.push('Coverage misses:', '');
|
||||
for (const i of badCov) out.push(`- "${i.utterance}" (${i.supported ? 'supported' : 'unsupported'}) — ${i.verdicts.join(', ')}`);
|
||||
out.push('');
|
||||
}
|
||||
}
|
||||
if (notes.length) {
|
||||
out.push('## Notes', '');
|
||||
for (const n of notes) out.push(`- ${n}`);
|
||||
out.push('');
|
||||
}
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Before → after table for two summarize() results (matched by model key).
|
||||
* Coverage "before" is the v1 rubric (what the old tools could do); "after"
|
||||
* shows both v1 (same yardstick) and v2 (the current tool surface).
|
||||
*/
|
||||
export function renderComparison(before, after, { beforeLabel = 'before', afterLabel = 'after' } = {}) {
|
||||
const out = ['## Before → after', '', `Before = ${beforeLabel}; after = ${afterLabel}. Core accuracy = tool + all argument checks (full call). Coverage OK % = correct, or an honest refusal where the tools of that build cannot do it. HC = hallucinated-capability count. p50 = send → tool call (text). Prompt tokens = mean input tokens per model request (cloud: as billed, before cache discounts; local: the cold full-prompt count).`, ''];
|
||||
out.push('| model | core full-call | coverage OK % (v1 → v1 / v2) | HC (v1 → v1 / v2) | dialogue turn-2 | capability full-call (after) | p50 → tool call | prompt tokens |');
|
||||
out.push('|---|---|---|---|---|---|---|---|');
|
||||
const hc = (block) => (block?.n ? block.counts['hallucinated-capability'] : null);
|
||||
const promptTok = (m) => (m?.warmup?.promptTokens ?? m?.tokens?.inputPerRequestMean ?? null);
|
||||
for (const a of after.models) {
|
||||
const b = before.models.find((x) => x.key === a.key);
|
||||
const arrow = (x, y, f) => `${f(x)} → ${f(y)}`;
|
||||
const pctF = (v) => (v === null || v === undefined ? '—' : `${v}%`);
|
||||
const msF = (v) => (v === null || v === undefined ? '—' : `${Math.round(v)}ms`);
|
||||
const num = (v) => (v === null || v === undefined ? '—' : String(v));
|
||||
out.push(`| ${a.key} | ${arrow(b?.core.fullCallAccuracyPct, a.core.fullCallAccuracyPct, pctF)} | ${pctF(b?.coverage.acceptablePct)} → ${pctF(a.coverage.acceptablePct)} / ${pctF(a.coverageV2.acceptablePct)} | ${num(hc(b?.coverage))} → ${num(hc(a.coverage))} / ${num(hc(a.coverageV2))} | ${arrow(b?.dialogue.turn2FullPct, a.dialogue.turn2FullPct, pctF)} | ${pctF(a.capability.fullPct)} | ${arrow(b?.latencyMs.toolCallP50, a.latencyMs.toolCallP50, msF)} | ${arrow(promptTok(b), promptTok(a), num)} |`);
|
||||
}
|
||||
out.push('');
|
||||
return out.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* Stub tool results fed back to the model after each call. Pure.
|
||||
*
|
||||
* core suite: the neutral `{ ok:true }` stub qa-voice-routing has always
|
||||
* used, so continuations behave exactly as in that harness.
|
||||
* coverage suite: the same stub, except a call whose arguments violate the
|
||||
* tool schema gets `{ ok:false, error }` — what the app's runner returns
|
||||
* for an unknown layer id — so the model's follow-up words can be judged
|
||||
* for honesty.
|
||||
*/
|
||||
import { validateCall } from './grade.mjs';
|
||||
|
||||
export const STUB_OK = Object.freeze({ ok: true, note: 'qa-harness stub result' });
|
||||
|
||||
export function stubResult(suite, c, tools) {
|
||||
if (suite === 'coverage') {
|
||||
const errors = validateCall(tools, c);
|
||||
if (errors.length) return { ok: false, error: `Invalid arguments: ${errors.join('; ')}` };
|
||||
}
|
||||
return { ...STUB_OK };
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Pure converters from the production Realtime tool list
|
||||
* (`{ type:'function', name, description, parameters }`) to other providers'
|
||||
* tool formats. Descriptions and schemas are carried over unchanged except
|
||||
* where a target cannot represent them; every such change is listed below.
|
||||
*/
|
||||
|
||||
const PERMISSIVE_SCALAR = Object.freeze(['string', 'number', 'boolean']);
|
||||
|
||||
const clone = (value) => JSON.parse(JSON.stringify(value));
|
||||
|
||||
/**
|
||||
* Normalize a JSON schema for OpenAI-compatible local servers.
|
||||
*
|
||||
* Only one change: a property schema with no `type` (and no enum / anyOf /
|
||||
* oneOf / $ref / const that would imply one) gets a permissive scalar union.
|
||||
* `analyst_query.filters[].value` is `{}` in production — valid JSON Schema,
|
||||
* but chat templates that print `type` for every property (Gemma's among
|
||||
* them) crash on it.
|
||||
*/
|
||||
export function normalizeSchemaForChat(schema) {
|
||||
const out = clone(schema);
|
||||
const visit = (node, isRoot) => {
|
||||
if (!node || typeof node !== 'object' || Array.isArray(node)) return;
|
||||
const implied = ['type', 'enum', 'anyOf', 'oneOf', 'allOf', '$ref', 'const'].some((k) => k in node);
|
||||
if (!isRoot && !implied) node.type = [...PERMISSIVE_SCALAR];
|
||||
if (node.properties) for (const sub of Object.values(node.properties)) visit(sub, false);
|
||||
if (node.items) visit(node.items, false);
|
||||
for (const key of ['anyOf', 'oneOf', 'allOf']) if (Array.isArray(node[key])) node[key].forEach((s) => visit(s, false));
|
||||
};
|
||||
visit(out, true);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every schema location normalizeSchemaForChat changes, as
|
||||
* `{ tool, path, from, to }` — logged in summary.md so the per-provider
|
||||
* payload difference is explicit rather than implied.
|
||||
*/
|
||||
export function chatNormalizationDiff(realtimeTools) {
|
||||
const changes = [];
|
||||
const walk = (before, after, tool, where) => {
|
||||
if (!before || typeof before !== 'object' || Array.isArray(before)) return;
|
||||
if (JSON.stringify(before.type) !== JSON.stringify(after.type)) {
|
||||
changes.push({ tool, path: where, from: before.type === undefined ? '(untyped)' : before.type, to: after.type });
|
||||
}
|
||||
for (const [k, v] of Object.entries(before.properties || {})) walk(v, after.properties[k], tool, `${where}.${k}`);
|
||||
if (before.items) walk(before.items, after.items, tool, `${where}[]`);
|
||||
};
|
||||
for (const tool of realtimeTools) walk(tool.parameters, normalizeSchemaForChat(tool.parameters), tool.name, tool.name);
|
||||
return changes;
|
||||
}
|
||||
|
||||
/** Realtime tools → Chat Completions tools (`{type:'function', function:{...}}`). */
|
||||
export function toChatTools(realtimeTools, { normalize = true } = {}) {
|
||||
return realtimeTools.map((tool) => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: normalize ? normalizeSchemaForChat(tool.parameters) : clone(tool.parameters),
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
// Keywords the Gemini OpenAPI-subset `Schema` does not accept.
|
||||
const GEMINI_DROP = new Set(['additionalProperties', '$schema', 'default', 'examples', '$id', 'const']);
|
||||
|
||||
/**
|
||||
* Convert a JSON schema to the Gemini OpenAPI-subset `Schema`.
|
||||
* Drops unsupported keywords (additionalProperties, $schema, default, ...);
|
||||
* an untyped property becomes `anyOf` string|number|boolean; a type array
|
||||
* becomes `anyOf` of single types.
|
||||
*/
|
||||
export function toGeminiSchema(schema) {
|
||||
if (!schema || typeof schema !== 'object' || Array.isArray(schema)) return schema;
|
||||
const out = {};
|
||||
for (const [key, value] of Object.entries(schema)) {
|
||||
if (GEMINI_DROP.has(key)) continue;
|
||||
if (key === 'properties') {
|
||||
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
|
||||
} else if (key === 'items') {
|
||||
out.items = toGeminiSchema(value);
|
||||
} else if (key === 'anyOf' || key === 'oneOf') {
|
||||
out.anyOf = value.map((s) => toGeminiSchema(s));
|
||||
} else {
|
||||
out[key] = clone(value);
|
||||
}
|
||||
}
|
||||
if (Array.isArray(out.type)) {
|
||||
const types = out.type;
|
||||
delete out.type;
|
||||
out.anyOf = types.map((t) => ({ type: t }));
|
||||
}
|
||||
const implied = ['type', 'enum', 'anyOf'].some((k) => k in out);
|
||||
if (!implied) out.anyOf = PERMISSIVE_SCALAR.map((t) => ({ type: t }));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Realtime tools → Gemini `functionDeclarations`.
|
||||
* mode 'jsonSchema' (default): pass the production schema verbatim through
|
||||
* `parametersJsonSchema` — full JSON Schema, closest to prompt parity.
|
||||
* mode 'openapi': convert to the OpenAPI-subset `parameters` field.
|
||||
* A tool with no properties gets no parameters field in openapi mode (Gemini
|
||||
* rejects an empty OBJECT schema there).
|
||||
*/
|
||||
export function toGeminiDeclarations(realtimeTools, { mode = 'jsonSchema' } = {}) {
|
||||
return realtimeTools.map((tool) => {
|
||||
const decl = { name: tool.name, description: tool.description };
|
||||
const params = tool.parameters;
|
||||
if (mode === 'jsonSchema') {
|
||||
decl.parametersJsonSchema = clone(params);
|
||||
} else if (params && Object.keys(params.properties || {}).length) {
|
||||
decl.parameters = toGeminiSchema(params);
|
||||
}
|
||||
return decl;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Summarize voice turn spans from the Realtime debug log.
|
||||
*
|
||||
* Every closed voice turn writes one `turn.span` record (see
|
||||
* src/voice/turnMetrics.js). This prints each span and p50/p90 figures for
|
||||
* end-of-speech → first audio, the longest silence, tool time, preamble rate,
|
||||
* spoken words and hedge hits.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/voice-turn-spans.mjs [log.jsonl] [--since <ISO time>] [--session <id>] [--json]
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const args = process.argv.slice(2);
|
||||
const option = (flag) => {
|
||||
const index = args.indexOf(flag);
|
||||
return index >= 0 ? args[index + 1] : null;
|
||||
};
|
||||
const positional = args.filter(
|
||||
(arg, index) => !arg.startsWith('--') && !args[index - 1]?.startsWith('--'),
|
||||
);
|
||||
const logPath =
|
||||
positional[0] || path.join(root, '.gev-logs', 'realtime-conversations.jsonl');
|
||||
const since = option('--since') ? Date.parse(option('--since')) : null;
|
||||
const session = option('--session');
|
||||
|
||||
const spans = [];
|
||||
for (const line of fs.readFileSync(logPath, 'utf8').split('\n')) {
|
||||
if (!line.includes('"turn.span"')) continue;
|
||||
let record;
|
||||
try {
|
||||
record = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (record.event !== 'turn.span') continue;
|
||||
if (since && Date.parse(record.timestamp) < since) continue;
|
||||
if (session && record.sessionId !== session) continue;
|
||||
spans.push({
|
||||
at: record.timestamp,
|
||||
session: record.sessionId,
|
||||
...record.payload,
|
||||
});
|
||||
}
|
||||
|
||||
const quantile = (values, q) => {
|
||||
const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
|
||||
if (!sorted.length) return null;
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))];
|
||||
};
|
||||
const withTools = spans.filter((span) => span.tools?.length);
|
||||
const slowToolTurns = withTools.filter((span) =>
|
||||
span.tools.some((tool) => tool.ms >= 700),
|
||||
);
|
||||
const summary = {
|
||||
turns: spans.length,
|
||||
toolTurns: withTools.length,
|
||||
firstAudioMs: {
|
||||
p50: quantile(spans.map((s) => s.first_audio_ms), 0.5),
|
||||
p90: quantile(spans.map((s) => s.first_audio_ms), 0.9),
|
||||
},
|
||||
maxSilenceMs: {
|
||||
p50: quantile(spans.map((s) => s.max_silence_ms), 0.5),
|
||||
p90: quantile(spans.map((s) => s.max_silence_ms), 0.9),
|
||||
},
|
||||
toolMs: {
|
||||
p50: quantile(withTools.flatMap((s) => s.tools.map((t) => t.ms)), 0.5),
|
||||
p90: quantile(withTools.flatMap((s) => s.tools.map((t) => t.ms)), 0.9),
|
||||
},
|
||||
preambleRate: withTools.length
|
||||
? withTools.filter((s) => s.preamble).length / withTools.length
|
||||
: null,
|
||||
slowToolPreambleOrProgressRate: slowToolTurns.length
|
||||
? slowToolTurns.filter((s) => s.preamble || s.progress_lines > 0).length /
|
||||
slowToolTurns.length
|
||||
: null,
|
||||
answerWords: {
|
||||
p50: quantile(spans.map((s) => s.answer_words), 0.5),
|
||||
p90: quantile(spans.map((s) => s.answer_words), 0.9),
|
||||
},
|
||||
spokenWordsTotal: spans.reduce((sum, s) => sum + (s.spoken_words || 0), 0),
|
||||
hedgeHits: spans.reduce((sum, s) => sum + (s.hedge_hits?.length || 0), 0),
|
||||
};
|
||||
|
||||
if (args.includes('--json')) {
|
||||
console.log(JSON.stringify({ summary, spans }, null, 2));
|
||||
} else {
|
||||
for (const span of spans) {
|
||||
const tools = (span.tools || [])
|
||||
.map((tool) => `${tool.name}:${tool.ms}ms`)
|
||||
.join(',');
|
||||
console.log(
|
||||
[
|
||||
span.at,
|
||||
`first=${span.first_audio_ms ?? '-'}ms`,
|
||||
`maxSilence=${span.max_silence_ms ?? '-'}ms`,
|
||||
`preamble=${span.preamble ? 'y' : 'n'}`,
|
||||
`progress=${span.progress_lines}`,
|
||||
`words=${span.answer_words}/${span.spoken_words}`,
|
||||
`hedges=${span.hedge_hits?.length || 0}`,
|
||||
tools || '(no tools)',
|
||||
span.answer_text ? `"${span.answer_text.slice(0, 80)}"` : '',
|
||||
].join(' '),
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
}
|
||||
Reference in new issue
Block a user