mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
German Khokhlov
b8f4d097c4
Preserve the camera, map source and layer selection across engine sessions. Restore the previous engine if startup fails and retain local history navigation.
668 lines
27 KiB
JavaScript
668 lines
27 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Deterministic QA for the detection overlay's bounded two-lane label pipeline.
|
|
*
|
|
* Run:
|
|
* node scripts/qa-labels.mjs --url http://localhost:4173 [--engine=cesium|navara]
|
|
*
|
|
* Engine-neutral: the camera and frames go through the globe engine and the
|
|
* layers through the QA probe (window.__godsEyeView.layers). On an engine
|
|
* without the flights, military and satellites layers (Navara today) the
|
|
* field checks are SKIPPED and the settings checks still run.
|
|
*
|
|
* The harness replaces only the three detection read APIs in the loaded page.
|
|
* It does not enable their network pollers or mutate repository data. The field
|
|
* contains 12,000 stable observations and is sampled while the global camera
|
|
* makes a small orbit.
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import puppeteer from 'puppeteer';
|
|
|
|
const argv = process.argv.slice(2);
|
|
const getFlag = (name) => argv.includes(name);
|
|
const getOpt = (name, fallback) => {
|
|
const index = argv.indexOf(name);
|
|
return index >= 0 && argv[index + 1] ? argv[index + 1] : fallback;
|
|
};
|
|
|
|
const ENGINE = argv.find((value) => value.startsWith('--engine='))?.slice(9) ?? 'cesium';
|
|
const APP_URL = (() => {
|
|
const url = new URL(getOpt('--url', 'http://localhost:4173'));
|
|
url.searchParams.set('engine', ENGINE);
|
|
return url.href;
|
|
})();
|
|
const APP_ORIGIN = new URL(APP_URL).origin;
|
|
const HEADFUL = getFlag('--headful');
|
|
const SHOT_DIR = path.resolve('qa-shots/labels');
|
|
const STORAGE_KEY = 'gev:detection-allocation:v1';
|
|
const OBSERVATION_COUNT = 12000;
|
|
const SYNTHETIC_SAMPLE_MS = 10000;
|
|
const NORMAL_SAMPLE_MS = 6000;
|
|
const SAMPLE_TIMEOUT_MS = 120000;
|
|
const MIN_SYNTHETIC_SOLVES = 20;
|
|
const MIN_SYNTHETIC_SPAN_MS = SYNTHETIC_SAMPLE_MS;
|
|
const MIN_NORMAL_FRAMES = 20;
|
|
const SETTLED_SOLVE_COUNT = 3;
|
|
const GLOBAL_CAMERA_MIN_HEIGHT_M = 20_000_000;
|
|
const GLOBAL_CAMERA_MAX_HEIGHT_M = 30_000_000;
|
|
const FIELD_COUNTS = Object.freeze({
|
|
flights: 7200,
|
|
military: 1000,
|
|
satellites: 3800,
|
|
});
|
|
const NORMAL_COUNTS = Object.freeze({
|
|
flights: 4300,
|
|
military: 70,
|
|
satellites: 830,
|
|
});
|
|
|
|
const CHROME_EXECUTABLE_CANDIDATES = [
|
|
process.env.PUPPETEER_EXECUTABLE_PATH,
|
|
// Prefer puppeteer's version-pinned Chrome-for-Testing over the system
|
|
// Chrome: /Applications auto-updates underneath the harnesses, and its
|
|
// software-GL behavior shifts across majors (system Chrome 150 blew the
|
|
// tile-gated drain budget under SwiftShader on 2026-07-30 — six
|
|
// false-negative qa-cctv-v2 runs against a healthy build). A deterministic
|
|
// pinned browser beats the newest one for regression harnesses.
|
|
await puppeteer.executablePath().catch(() => null),
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
'/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
|
|
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
].filter(Boolean);
|
|
|
|
function findChromeExecutable() {
|
|
for (const candidate of CHROME_EXECUTABLE_CANDIDATES) {
|
|
try {
|
|
if (fs.existsSync(candidate)) return candidate;
|
|
} catch {
|
|
// Let Puppeteer use its bundled browser.
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const results = [];
|
|
let skippedChecks = 0;
|
|
let skippedField = false;
|
|
function record(name, ok, detail) {
|
|
results.push({ name, ok, detail });
|
|
const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
|
|
console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`);
|
|
}
|
|
|
|
function percentile(values, pct) {
|
|
const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
|
|
if (sorted.length === 0) return Number.NaN;
|
|
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * pct) - 1)];
|
|
}
|
|
|
|
function analyzeMembership(samples) {
|
|
let replacements = 0;
|
|
let exposure = 0;
|
|
const transitions = new Map();
|
|
|
|
for (let index = 0; index < samples.length; index++) {
|
|
const sample = samples[index];
|
|
const current = new Set(sample.labeledKeys);
|
|
if (index > 0) {
|
|
const previous = new Set(samples[index - 1].labeledKeys);
|
|
const removed = [...previous].filter((key) => !current.has(key)).length;
|
|
const added = [...current].filter((key) => !previous.has(key)).length;
|
|
replacements += Math.max(removed, added);
|
|
const elapsedSec = Math.max(0.001, (sample.t - samples[index - 1].t) / 1000);
|
|
exposure += Math.max(1, (previous.size + current.size) / 2) * elapsedSec;
|
|
}
|
|
for (const key of current) {
|
|
if (!transitions.has(key)) transitions.set(key, []);
|
|
transitions.get(key).push(sample.t);
|
|
}
|
|
}
|
|
|
|
let showHideShow = 0;
|
|
for (const times of transitions.values()) {
|
|
for (let index = 1; index < times.length; index++) {
|
|
const previousIndex = samples.findIndex((sample) => sample.t === times[index - 1]);
|
|
const nextIndex = samples.findIndex((sample) => sample.t === times[index]);
|
|
if (nextIndex - previousIndex > 1 && times[index] - times[index - 1] <= 2000) {
|
|
showHideShow++;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
const churnPctPerSec = exposure > 0
|
|
? replacements / exposure * 100
|
|
: 0;
|
|
return {
|
|
churnPctPerSec,
|
|
showHideShow,
|
|
replacements,
|
|
exposure,
|
|
transitions: Math.max(0, samples.length - 1),
|
|
minimumPopulation: samples.length
|
|
? Math.min(...samples.map((sample) => sample.labeledKeys.length))
|
|
: 0,
|
|
maximumPopulation: samples.length
|
|
? Math.max(...samples.map((sample) => sample.labeledKeys.length))
|
|
: 0,
|
|
};
|
|
}
|
|
|
|
async function waitForConclusiveSamples(page, {
|
|
minimumElapsedMs,
|
|
minimumSolves = 0,
|
|
minimumSolveSpanMs = 0,
|
|
minimumNormalFrames = 0,
|
|
}) {
|
|
const deadline = Date.now() + SAMPLE_TIMEOUT_MS;
|
|
while (Date.now() < deadline) {
|
|
const state = await page.evaluate(() => {
|
|
const qa = window.__LABEL_QA;
|
|
const solves = qa?.solve || [];
|
|
const matchingFrames = (qa?.frames || []).filter(
|
|
(sample) => sample.observationCount === 5200,
|
|
);
|
|
return {
|
|
elapsedMs: performance.now() - (qa?.startedAt || performance.now()),
|
|
solveCount: solves.length,
|
|
solveSpanMs: solves.length > 1 ? solves.at(-1).t - solves[0].t : 0,
|
|
normalFrameCount: matchingFrames.length,
|
|
};
|
|
});
|
|
if (state.elapsedMs >= minimumElapsedMs
|
|
&& state.solveCount >= minimumSolves
|
|
&& state.solveSpanMs >= minimumSolveSpanMs
|
|
&& state.normalFrameCount >= minimumNormalFrames) return state;
|
|
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function waitForSettledSyntheticField(page) {
|
|
await page.waitForFunction(
|
|
({ observationCount, settledSolveCount, minHeight, maxHeight }) => {
|
|
const solves = window.__LABEL_QA?.solve || [];
|
|
if (solves.length < settledSolveCount) return false;
|
|
return solves.slice(-settledSolveCount).every((sample) => (
|
|
sample.observationCount === observationCount
|
|
&& sample.labeledKeys.length > 0
|
|
&& sample.selectedCount === sample.labeledKeys.length
|
|
&& sample.cameraHeight >= minHeight
|
|
&& sample.cameraHeight <= maxHeight
|
|
));
|
|
},
|
|
{ timeout: SAMPLE_TIMEOUT_MS, polling: 100 },
|
|
{
|
|
observationCount: OBSERVATION_COUNT,
|
|
settledSolveCount: SETTLED_SOLVE_COUNT,
|
|
minHeight: GLOBAL_CAMERA_MIN_HEIGHT_M,
|
|
maxHeight: GLOBAL_CAMERA_MAX_HEIGHT_M,
|
|
},
|
|
);
|
|
}
|
|
|
|
async function main() {
|
|
console.log('\nDetection Label QA');
|
|
console.log(` App URL : ${APP_URL}`);
|
|
console.log(` Observations : ${OBSERVATION_COUNT}`);
|
|
console.log(` Sample window: >=${SYNTHETIC_SAMPLE_MS / 1000}s synthetic + >=${NORMAL_SAMPLE_MS / 1000}s normal\n`);
|
|
|
|
try {
|
|
const response = await fetch(APP_URL);
|
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
} catch (error) {
|
|
console.error(`Dev server not reachable at ${APP_URL}: ${error.message}`);
|
|
process.exit(2);
|
|
}
|
|
|
|
fs.mkdirSync(SHOT_DIR, { recursive: true });
|
|
const executablePath = findChromeExecutable();
|
|
const browser = await puppeteer.launch({
|
|
headless: HEADFUL ? false : 'new',
|
|
...(executablePath ? { executablePath } : {}),
|
|
args: [
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--use-gl=angle',
|
|
`--use-angle=${process.env.QA_ANGLE || 'swiftshader'}`,
|
|
'--disable-dev-shm-usage',
|
|
'--disable-web-security',
|
|
'--disable-background-timer-throttling',
|
|
'--disable-renderer-backgrounding',
|
|
'--window-size=1280,800',
|
|
],
|
|
});
|
|
|
|
const consoleErrors = [];
|
|
const failedResponses = [];
|
|
try {
|
|
const page = await browser.newPage();
|
|
await page.setViewport({ width: 1280, height: 800, deviceScaleFactor: 1 });
|
|
page.on('console', (message) => {
|
|
if (message.type() !== 'error') return;
|
|
const text = message.text();
|
|
const sourceUrl = message.location()?.url || '';
|
|
if (!/Failed to load resource.*404/i.test(text)) {
|
|
consoleErrors.push(sourceUrl ? `${text} [${sourceUrl}]` : text);
|
|
}
|
|
});
|
|
page.on('pageerror', (error) => consoleErrors.push(`pageerror: ${error.message}`));
|
|
page.on('response', (response) => {
|
|
if (response.status() >= 500) {
|
|
failedResponses.push(`HTTP ${response.status()} ${response.url()}`);
|
|
}
|
|
});
|
|
|
|
await page.evaluateOnNewDocument((storageKey, appOrigin) => {
|
|
localStorage.setItem(storageKey, 'WEIGHTED');
|
|
const realFetch = window.fetch.bind(window);
|
|
window.fetch = (input, init) => {
|
|
const requestUrl = typeof input === 'string' || input instanceof URL
|
|
? String(input)
|
|
: input?.url;
|
|
const url = new URL(requestUrl, window.location.href);
|
|
if (url.origin === appOrigin && url.pathname === '/api/openai/hud-summary') {
|
|
return Promise.resolve(new Response(JSON.stringify({ summary: 'QA globe ready' }), {
|
|
status: 200,
|
|
headers: { 'Content-Type': 'application/json' },
|
|
}));
|
|
}
|
|
return realFetch(input, init);
|
|
};
|
|
}, STORAGE_KEY, APP_ORIGIN);
|
|
|
|
await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
await page.waitForFunction(
|
|
() => window.__godsEyeView?.layers && window.__godsEyeView?.styleManager,
|
|
{ timeout: 60000, polling: 100 },
|
|
);
|
|
|
|
// flyToAustin schedules its 600 m arrival 500 ms after initialization.
|
|
// Let that callback start, then cancel it before establishing the global
|
|
// measurement camera so startup motion cannot invalidate the sample.
|
|
await new Promise((resolve) => setTimeout(resolve, 600));
|
|
await page.evaluate(() => window.__godsEyeView.engine.camera.cancelFlight());
|
|
|
|
const stateChecks = await page.evaluate((storageKey) => {
|
|
const manager = window.__godsEyeView.styleManager;
|
|
const restored = manager.getDetectionState();
|
|
manager._syncDetectionUiFromEngine();
|
|
const storedAfterPassiveSync = localStorage.getItem(storageKey);
|
|
const off = manager.setDetection({ enabled: false, densityPct: 25 });
|
|
const restoredFromOff = manager.setDetection({ enabled: true });
|
|
return { restored, storedAfterPassiveSync, off, restoredFromOff };
|
|
}, STORAGE_KEY);
|
|
|
|
record(
|
|
'allocation preference survives passive UI sync',
|
|
stateChecks.storedAfterPassiveSync === 'WEIGHTED',
|
|
`share/engine=${stateChecks.restored.allocationStrategy}, stored=${stateChecks.storedAfterPassiveSync}`,
|
|
);
|
|
record(
|
|
'density update while OFF restores the matching profile',
|
|
stateChecks.off.detectionMode === 'OFF'
|
|
&& stateChecks.off.densityPct === 25
|
|
&& stateChecks.restoredFromOff.detectionMode === 'SPARSE'
|
|
&& stateChecks.restoredFromOff.densityPct === 25,
|
|
`OFF=${stateChecks.off.densityPct}%, restore=${stateChecks.restoredFromOff.detectionMode}/${stateChecks.restoredFromOff.densityPct}%`,
|
|
);
|
|
|
|
const fieldLayers = ['flights', 'military', 'satellites'];
|
|
const missingLayers = await page.evaluate(
|
|
(ids) => ids.filter((id) => !window.__godsEyeView.layers.available(id)),
|
|
fieldLayers,
|
|
);
|
|
// An engine with flights but not the other field layers (Navara, W1 so
|
|
// far): the same 12,000 / 5,200 observations, all as aircraft, so the
|
|
// detection lane is measured on the layers the engine has.
|
|
let fieldCounts = FIELD_COUNTS;
|
|
let normalCounts = NORMAL_COUNTS;
|
|
if (missingLayers.length && !missingLayers.includes('flights')) {
|
|
const sum = (counts) =>
|
|
Object.values(counts).reduce((total, count) => total + count, 0);
|
|
fieldCounts = Object.fromEntries(
|
|
fieldLayers.map((id) => [
|
|
id,
|
|
missingLayers.includes(id)
|
|
? 0
|
|
: id === 'flights'
|
|
? sum(FIELD_COUNTS) - fieldLayers.filter((other) => other !== 'flights' && !missingLayers.includes(other)).reduce((total, other) => total + FIELD_COUNTS[other], 0)
|
|
: FIELD_COUNTS[id],
|
|
]),
|
|
);
|
|
normalCounts = Object.fromEntries(
|
|
fieldLayers.map((id) => [
|
|
id,
|
|
missingLayers.includes(id)
|
|
? 0
|
|
: id === 'flights'
|
|
? sum(NORMAL_COUNTS) - fieldLayers.filter((other) => other !== 'flights' && !missingLayers.includes(other)).reduce((total, other) => total + NORMAL_COUNTS[other], 0)
|
|
: NORMAL_COUNTS[id],
|
|
]),
|
|
);
|
|
console.log(
|
|
` [INFO] the field as ${JSON.stringify(fieldCounts)} (no ${missingLayers.join(', ')} on ${ENGINE})`,
|
|
);
|
|
} else if (missingLayers.length) {
|
|
console.log(` [SKIP] the deterministic field checks — no ${missingLayers.join(', ')} on ${ENGINE}`);
|
|
skippedChecks += 1;
|
|
record(
|
|
'console remains clean',
|
|
consoleErrors.length === 0,
|
|
consoleErrors.slice(0, 3).join(' | ') || '0 errors',
|
|
);
|
|
skippedField = true;
|
|
return;
|
|
}
|
|
const injected = await page.evaluate(async ({ fieldCounts, normalCounts }) => {
|
|
const { engine, dataManager, styleManager } = window.__godsEyeView;
|
|
const geo = await import('/src/geo/index.js');
|
|
engine.camera.cancelFlight();
|
|
// A frame counter on the engine (the samples are one per drawn frame).
|
|
window.__LABEL_QA_FRAME = 0;
|
|
engine.frames.postRender.addEventListener(() => {
|
|
window.__LABEL_QA_FRAME += 1;
|
|
});
|
|
const Cartesian3 = { fromDegrees: (lon, lat, height) => geo.cartesian3.fromDegrees(lon, lat, height) };
|
|
const field = { flights: [], military: [], satellites: [] };
|
|
const layerIds = Object.keys(field);
|
|
const goldenAngle = Math.PI * (3 - Math.sqrt(5));
|
|
|
|
for (const layerId of layerIds) {
|
|
const count = fieldCounts[layerId];
|
|
for (let localIndex = 0; localIndex < count; localIndex++) {
|
|
const radius = 57 * Math.sqrt((localIndex + 0.5) / count);
|
|
const angle = localIndex * goldenAngle + layerIds.indexOf(layerId) * 0.7;
|
|
const lat = Math.max(-65, Math.min(75, 30 + radius * Math.sin(angle)));
|
|
const lon = -97 + radius * Math.cos(angle) / Math.max(0.4, Math.cos(lat * Math.PI / 180));
|
|
const sourceId = `${layerId.slice(0, 1)}-${String(localIndex).padStart(4, '0')}`;
|
|
field[layerId].push({
|
|
sourceId,
|
|
id: sourceId.toUpperCase(),
|
|
metric: layerId === 'satellites' ? 'LEO' : `${250 + (localIndex % 450)}KT`,
|
|
position: Cartesian3.fromDegrees(lon, lat, layerId === 'satellites' ? 550000 : 9000),
|
|
type: layerId === 'satellites' ? 'SAT' : 'AIR',
|
|
tier: layerId === 'military' ? 'military' : (layerId === 'satellites' ? 'space' : 'civil'),
|
|
});
|
|
}
|
|
}
|
|
|
|
window.__LABEL_QA_FIELD = field;
|
|
window.__LABEL_QA_NORMAL_COUNTS = normalCounts;
|
|
for (const layerId of layerIds) {
|
|
const entry = dataManager.layers.get(layerId);
|
|
if (entry)
|
|
entry.module.getDetectableObjects = () => window.__LABEL_QA_FIELD[layerId];
|
|
}
|
|
|
|
// The global camera, orbiting west to east (the old rotateRight about
|
|
// the globe centre at 25 000 km).
|
|
let orbitLon = -97;
|
|
const placeCamera = () =>
|
|
engine.camera.setView({
|
|
destination: Cartesian3.fromDegrees(orbitLon, 30, 25000000),
|
|
orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
|
|
});
|
|
placeCamera();
|
|
styleManager.setDetection({
|
|
enabled: true,
|
|
densityPct: 100,
|
|
allocationStrategy: 'elastic',
|
|
});
|
|
|
|
window.__LABEL_QA = {
|
|
solve: [],
|
|
frames: [],
|
|
lastRevision: -1,
|
|
lastFrameNumber: -1,
|
|
startedAt: performance.now(),
|
|
};
|
|
let lastOrbitAt = performance.now();
|
|
window.__LABEL_QA_ORBIT = setInterval(() => {
|
|
const now = performance.now();
|
|
orbitLon -= ((now - lastOrbitAt) * 0.0000012 * 180) / Math.PI;
|
|
lastOrbitAt = now;
|
|
placeCamera();
|
|
engine.render.request('qa-labels');
|
|
}, 100);
|
|
const sample = () => {
|
|
const diagnostics = styleManager.getDetectionDiagnostics();
|
|
const frameNumber = window.__LABEL_QA_FRAME;
|
|
if (diagnostics && frameNumber !== window.__LABEL_QA.lastFrameNumber) {
|
|
window.__LABEL_QA.lastFrameNumber = frameNumber;
|
|
window.__LABEL_QA.frames.push({
|
|
didSolve: diagnostics.didSolve,
|
|
placementBuildCount: diagnostics.placementBuildCount,
|
|
selectedCount: diagnostics.selectedCount,
|
|
fadingCount: diagnostics.fadingCount,
|
|
cohortCount: diagnostics.cohortCount,
|
|
observationCount: diagnostics.observationCount,
|
|
frameTotalMs: diagnostics.frameTotalMs,
|
|
paintMs: diagnostics.paintMs,
|
|
throttleSkipCount: diagnostics.throttleSkipCount,
|
|
});
|
|
}
|
|
const dataset = document.querySelector('#world-overlay-canvas')?.dataset;
|
|
const revision = Number(dataset?.solveRevision || 0);
|
|
if (dataset && revision > 0 && revision !== window.__LABEL_QA.lastRevision) {
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.solve.push({
|
|
t: performance.now(),
|
|
revision,
|
|
solveMs: Number(dataset.solveMs),
|
|
frameTotalMs: Number(dataset.frameTotalMs),
|
|
paintMs: Number(dataset.paintMs),
|
|
throttleSkipCount: Number(dataset.throttleSkipCount),
|
|
placementBuildCount: Number(dataset.placementBuildCount),
|
|
observationCount: Number(dataset.observationCount),
|
|
selectedCount: Number(dataset.selectedCount),
|
|
fadingCount: Number(dataset.fadingCount),
|
|
cameraHeight: engine.camera.height(),
|
|
demandByLayer: JSON.parse(dataset.demandByLayer || '{}'),
|
|
cohortByLayer: JSON.parse(dataset.cohortByLayer || '{}'),
|
|
entitlementByLayer: JSON.parse(dataset.entitlementByLayer || '{}'),
|
|
labeledKeys: JSON.parse(dataset.labeledKeys || '[]'),
|
|
});
|
|
}
|
|
requestAnimationFrame(sample);
|
|
};
|
|
requestAnimationFrame(sample);
|
|
return Object.fromEntries(layerIds.map((layerId) => [layerId, field[layerId].length]));
|
|
}, { fieldCounts, normalCounts });
|
|
|
|
record(
|
|
'deterministic field contains exactly 12,000 observations',
|
|
Object.values(injected).reduce((sum, count) => sum + count, 0) === OBSERVATION_COUNT,
|
|
JSON.stringify(injected),
|
|
);
|
|
|
|
// Warm-up before steady-state p95: the first solves after field
|
|
// injection run on cold-tier code paths and skew p95 on a fresh server.
|
|
// (perf wave 2 added a 2.5 s sleep for this; main's settle+conclusive
|
|
// -samples gates supersede it — same fix, deterministic instead of timed.)
|
|
await waitForSettledSyntheticField(page);
|
|
await page.evaluate(() => {
|
|
const revision = Number(document.querySelector('#world-overlay-canvas')?.dataset?.solveRevision || 0);
|
|
window.__LABEL_QA.solve = [];
|
|
window.__LABEL_QA.frames = [];
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.lastFrameNumber = window.__LABEL_QA_FRAME;
|
|
window.__LABEL_QA.startedAt = performance.now();
|
|
});
|
|
const syntheticSampling = await waitForConclusiveSamples(page, {
|
|
minimumElapsedMs: SYNTHETIC_SAMPLE_MS,
|
|
minimumSolves: MIN_SYNTHETIC_SOLVES,
|
|
minimumSolveSpanMs: MIN_SYNTHETIC_SPAN_MS,
|
|
});
|
|
|
|
await page.screenshot({
|
|
path: path.join(SHOT_DIR, 'deterministic-12000.png'),
|
|
fullPage: false,
|
|
});
|
|
|
|
const syntheticSamples = await page.evaluate(() => {
|
|
const completed = window.__LABEL_QA;
|
|
const counts = window.__LABEL_QA_NORMAL_COUNTS;
|
|
for (const layerId of Object.keys(window.__LABEL_QA_FIELD)) {
|
|
window.__LABEL_QA_FIELD[layerId] = window.__LABEL_QA_FIELD[layerId].slice(0, counts[layerId]);
|
|
}
|
|
window.__LABEL_QA = {
|
|
solve: [],
|
|
frames: [],
|
|
lastRevision: -1,
|
|
lastFrameNumber: -1,
|
|
startedAt: performance.now(),
|
|
};
|
|
return completed;
|
|
});
|
|
|
|
await page.waitForFunction(
|
|
() => window.__godsEyeView.styleManager.getDetectionDiagnostics()?.observationCount === 5200,
|
|
{ timeout: SAMPLE_TIMEOUT_MS, polling: 100 },
|
|
);
|
|
await page.evaluate(() => {
|
|
const revision = Number(document.querySelector('#world-overlay-canvas')?.dataset?.solveRevision || 0);
|
|
window.__LABEL_QA.solve = [];
|
|
window.__LABEL_QA.frames = [];
|
|
window.__LABEL_QA.lastRevision = revision;
|
|
window.__LABEL_QA.lastFrameNumber = window.__LABEL_QA_FRAME;
|
|
window.__LABEL_QA.startedAt = performance.now();
|
|
});
|
|
const normalSampling = await waitForConclusiveSamples(page, {
|
|
minimumElapsedMs: NORMAL_SAMPLE_MS,
|
|
minimumNormalFrames: MIN_NORMAL_FRAMES,
|
|
});
|
|
const normalSamples = await page.evaluate(() => {
|
|
clearInterval(window.__LABEL_QA_ORBIT);
|
|
return window.__LABEL_QA;
|
|
});
|
|
|
|
await page.screenshot({
|
|
path: path.join(SHOT_DIR, 'normal-global-5200.png'),
|
|
fullPage: false,
|
|
});
|
|
|
|
const solveSamples = syntheticSamples.solve;
|
|
const steadySolveSamples = solveSamples.filter((sample) => (
|
|
sample.observationCount === OBSERVATION_COUNT
|
|
&& sample.labeledKeys.length > 0
|
|
&& sample.selectedCount === sample.labeledKeys.length
|
|
&& sample.cameraHeight >= GLOBAL_CAMERA_MIN_HEIGHT_M
|
|
&& sample.cameraHeight <= GLOBAL_CAMERA_MAX_HEIGHT_M
|
|
));
|
|
const normalFieldFrames = normalSamples.frames.filter(
|
|
(sample) => sample.observationCount === 5200,
|
|
);
|
|
const normalFrames = normalFieldFrames.filter((sample) => !sample.didSolve);
|
|
const solveP95 = percentile(solveSamples.map((sample) => sample.solveMs), 0.95);
|
|
const renderP95 = percentile(normalFrames.map((sample) => sample.frameTotalMs), 0.95);
|
|
// SwiftShader can render the complete Cesium scene below 8 FPS, making
|
|
// every sampled overlay frame eligible for the 125 ms solve tick. Paint is
|
|
// timed separately from solve, so it remains the deterministic frame-lane
|
|
// measurement even when there are no non-solve frames in this backend.
|
|
const paintP95 = percentile(
|
|
(normalFrames.length ? normalFrames : normalFieldFrames).map((sample) => sample.paintMs),
|
|
0.95,
|
|
);
|
|
const throttles = normalSamples.frames.map((sample) => sample.throttleSkipCount).filter(Number.isFinite);
|
|
const throttleDelta = throttles.length ? Math.max(...throttles) - Math.min(...throttles) : 0;
|
|
const normalPlacementOverflow = normalFieldFrames.filter(
|
|
(sample) => sample.placementBuildCount > sample.selectedCount + sample.fadingCount
|
|
+ (sample.didSolve ? sample.cohortCount : 0),
|
|
);
|
|
const solvePlacementOverflow = solveSamples.filter((sample) => {
|
|
const cohort = Object.values(sample.cohortByLayer).reduce((sum, count) => sum + count, 0);
|
|
return sample.placementBuildCount > cohort + sample.selectedCount + sample.fadingCount;
|
|
});
|
|
const cohortOverflow = solveSamples.filter((sample) => Object.entries(sample.cohortByLayer)
|
|
.some(([layerId, count]) => {
|
|
const quota = sample.entitlementByLayer[layerId] || 0;
|
|
return count > Math.min(256, Math.max(64, 4 * quota));
|
|
}));
|
|
const membership = analyzeMembership(solveSamples);
|
|
|
|
record(
|
|
'sampling is conclusive across the pathological and normal fields',
|
|
Boolean(syntheticSampling && normalSampling)
|
|
&& solveSamples.length >= MIN_SYNTHETIC_SOLVES
|
|
&& syntheticSampling.solveSpanMs >= MIN_SYNTHETIC_SPAN_MS
|
|
&& normalFieldFrames.length >= MIN_NORMAL_FRAMES,
|
|
`synthetic solves=${solveSamples.length}, span=${syntheticSampling?.solveSpanMs?.toFixed(0) || 0}ms; normal frames=${normalFieldFrames.length}`,
|
|
);
|
|
|
|
record(
|
|
'synthetic churn samples remain non-empty at the 12,000-contact global view',
|
|
steadySolveSamples.length === solveSamples.length,
|
|
`steady=${steadySolveSamples.length}/${solveSamples.length}; population=${membership.minimumPopulation}-${membership.maximumPopulation}`,
|
|
);
|
|
|
|
record(
|
|
'solve cohort remains bounded per layer',
|
|
cohortOverflow.length === 0,
|
|
`${solveSamples.length} solves checked`,
|
|
);
|
|
record(
|
|
'placement construction is bounded in both lanes',
|
|
normalPlacementOverflow.length === 0 && solvePlacementOverflow.length === 0,
|
|
`normal overflow=${normalPlacementOverflow.length}, solve overflow=${solvePlacementOverflow.length}`,
|
|
);
|
|
record(
|
|
'solve p95 is below approximately 4 ms',
|
|
solveP95 < 4.5,
|
|
`${solveP95.toFixed(2)} ms`,
|
|
);
|
|
record(
|
|
'normal-field overlay paint p95 is at or below 10 ms',
|
|
paintP95 <= 10,
|
|
`paint=${paintP95.toFixed(2)} ms${Number.isFinite(renderP95) ? `, non-solve total=${renderP95.toFixed(2)} ms` : ', every sampled frame included a solve'}`,
|
|
);
|
|
record(
|
|
'normal scene does not trigger adaptive throttling',
|
|
throttleDelta === 0,
|
|
`skip delta=${throttleDelta}`,
|
|
);
|
|
record(
|
|
'label churn remains below 8% per second',
|
|
membership.churnPctPerSec < 8,
|
|
`${membership.churnPctPerSec.toFixed(2)}%/s; replacements=${membership.replacements}, exposure=${membership.exposure.toFixed(1)} label-s, transitions=${membership.transitions}, population=${membership.minimumPopulation}-${membership.maximumPopulation}`,
|
|
);
|
|
record(
|
|
'no show-hide-show cycle occurs within two seconds',
|
|
membership.showHideShow === 0,
|
|
`cycles=${membership.showHideShow}`,
|
|
);
|
|
record(
|
|
'console remains clean',
|
|
consoleErrors.length === 0,
|
|
consoleErrors.slice(0, 3).join(' | ') || '0 errors',
|
|
);
|
|
record(
|
|
'no HTTP 5xx responses occur',
|
|
failedResponses.length === 0,
|
|
failedResponses.slice(0, 3).join(' | ') || '0 responses',
|
|
);
|
|
|
|
console.log(`\n Screenshot: ${path.join(SHOT_DIR, 'deterministic-12000.png')}`);
|
|
console.log(` Screenshot: ${path.join(SHOT_DIR, 'normal-global-5200.png')}`);
|
|
console.log(` Synthetic solves: ${solveSamples.length}; normal frames: ${normalFieldFrames.length} (${normalFrames.length} non-solve)`);
|
|
} finally {
|
|
await browser.close();
|
|
if (skippedField) summarize();
|
|
}
|
|
|
|
summarize();
|
|
}
|
|
|
|
function summarize() {
|
|
const failures = results.filter((result) => !result.ok);
|
|
console.log(`\n${results.length - failures.length}/${results.length} checks passed${skippedChecks ? `, ${skippedChecks} skipped` : ''} (engine ${ENGINE}).`);
|
|
if (failures.length) process.exit(1);
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(error);
|
|
process.exit(1);
|
|
});
|