Files
gods-eye-view/scripts/qa-all-layer-switches.mjs
T
German Khokhlov b4556f9763 Test layers on one engine and share selection and shader helpers
Layer display tests no longer re-check Cesium primitives in every layer:
they run on the fake engine, and one Cesium translation check per ground
kind lives in the Cesium engine's ground tests. Their harnesses are shared,
and the fake-engine, Cesium-engine and flights tests that repeated the
contract suite are gone, 340 tests in all.

The press-travel measurement is shared by the selection dispatcher and
CCTV, which loses its ScreenSpaceEventHandler shim and world-click module.
ALPR, installations and bikeshare share one selection keeper and selected
marker write. Layers with their own click rules keep their handlers.

Navara's pulse is a class like the ground pass, the ground and imagery
passes share the surface uniforms, and the colour, log-depth and occlusion
GLSL lives in one chunk module, each caller keeping its depth bias. The
ground suite runs inside the contract suite (--only=ground:), and fixture
and script names say what they test.

Behaviour is unchanged on both engines.
2026-11-08 17:06:10 +04:00

618 lines
21 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { execFileSync, spawn, spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
readdirSync,
rmSync,
statSync,
symlinkSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import assert from 'node:assert/strict';
import puppeteer from 'puppeteer';
import { APPLICATION_LAYER_DIRECTORY } from '../src/app/layers/directory.js';
import {
assertPreviewPortAvailable,
createStrict,
ENGINES,
localPreview,
waitForEngine,
} from './qa-strict-harness.mjs';
// The strict acceptance: every browser scenario against one keyless local
// preview, under the strict network policy (scripts/qa-strict-harness.mjs),
// writing <output>/acceptance.json, one log and one assertion manifest per
// scenario.
//
// node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 <output>
// [--serve[=<commit>]] build <commit> (default HEAD) keyless in a clean
// worktree and serve it with every .env key blank
// [--scratch=DIR] where --serve puts its checkout and build
// [--only=a,b] run only these labels
// [--quick] the soak with 2 runs of 10 switches and 100
// switches for the long run
//
// The run fails when a scenario fails, when any catalog layer (read from
// the live page and cross-checked with src/app/layers/directory.js) has no
// assertion on either engine, or when any network policy violation shows.
const base = localPreview(process.argv[2]);
const output = process.argv[3]?.startsWith('--')
? mkdtempSync(join(tmpdir(), 'gev-acceptance-'))
: resolve(process.argv[3] ?? mkdtempSync(join(tmpdir(), 'gev-acceptance-')));
const flag = (name) =>
process.argv
.find((arg) => arg === `--${name}` || arg.startsWith(`--${name}=`))
?.split('=')[1] ?? (process.argv.includes(`--${name}`) ? true : null);
const only = flag('only') ? String(flag('only')).split(',') : null;
const quick = Boolean(flag('quick'));
mkdirSync(join(output, 'evidence'), { recursive: true });
const root = fileURLToPath(new URL('..', import.meta.url));
const git = (...args) =>
execFileSync('git', args, { cwd: root, encoding: 'utf8' });
const dirty = () =>
git('status', '--porcelain')
.split('\n')
.filter(Boolean)
.map((line) => line.slice(3));
// Fail once, with the fix, instead of once per scenario.
let chrome;
try {
const browser = await puppeteer.launch({ headless: true });
chrome = await browser.version();
await browser.close();
} catch (error) {
console.error(
'Puppeteer cannot start Chrome: ' +
String(error?.message)
.split('\n')[0]
.replace(/ This can occur.*/, '') +
'\nPoint PUPPETEER_EXECUTABLE_PATH at an installed Chrome, for example\n' +
' PUPPETEER_EXECUTABLE_PATH="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"\n' +
'or download the pinned build with `npx puppeteer browsers install chrome`.',
);
process.exit(2);
}
// Every name the checkout's .env files define, plus key-like names in this
// environment: all blank for the build and the preview.
const KEY_LIKE = /(_KEY|_TOKEN|_SECRET|_CLIENT_ID|PASSWORD)$/;
/** The [name, value] pairs of the checkout's .env and .env.local, in order. */
const dotenv = () =>
['.env', '.env.local']
.filter((file) => existsSync(join(root, file)))
.flatMap((file) => readFileSync(join(root, file), 'utf8').split('\n'))
.map((line) => line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)/))
.filter(Boolean)
.map(([, name, raw]) => [name, raw.trim().replace(/^["']|["']$/g, '')]);
const dotenvNames = () => [...new Set(dotenv().map(([name]) => name))].sort();
const envNames = () =>
[
...new Set([
...dotenvNames(),
...Object.keys(process.env).filter((name) => KEY_LIKE.test(name)),
]),
].sort();
// Credentials only (model names and URLs are settings the code may also
// carry as defaults).
const envValues = () =>
dotenv()
.filter(([name, value]) => KEY_LIKE.test(name) && value.length >= 8)
.map(([, value]) => value);
const scrubbed = () => ({
...process.env,
...Object.fromEntries(envNames().map((name) => [name, ''])),
});
// The report names no local directory: the output, the served checkout and
// this repository become placeholders (logs carry stack traces).
const machinePaths = (text) => {
let out = String(text);
for (const [path, name] of [
[served?.scratch, '<scratch>'],
[output, '<output>'],
[root.replace(/\/$/, ''), '<repo>'],
])
if (path) out = out.split(path).join(name);
return out;
};
let served = null;
let preview = null;
const cleanup = () => {
if (preview && !preview.killed) preview.kill('SIGTERM');
if (served?.worktree)
try {
git('worktree', 'remove', '--force', served.worktree);
served.worktree = null;
} catch (error) {
console.error('Could not remove', served.worktree, error.message);
}
};
process.on('exit', cleanup);
process.on('SIGINT', () => process.exit(130));
if (flag('serve')) {
const commit = git(
'rev-parse',
flag('serve') === true ? 'HEAD' : String(flag('serve')),
).trim();
// --scratch=DIR keeps the checkout and build out of the system temp.
const scratchRoot = flag('scratch')
? resolve(String(flag('scratch')))
: tmpdir();
mkdirSync(scratchRoot, { recursive: true });
await assertPreviewPortAvailable(base.href);
const scratch = mkdtempSync(join(scratchRoot, 'gev-acceptance-build-'));
const worktree = join(scratch, 'checkout');
const dist = join(scratch, 'dist');
git('worktree', 'add', '--detach', worktree, commit);
symlinkSync(join(root, 'node_modules'), join(worktree, 'node_modules'));
served = {
scratch,
commit,
worktree,
dist,
// Names from .env files only; other key-like variables of this shell
// are blanked too but only counted.
blankedEnv: dotenvNames(),
otherKeyLikeVariablesBlanked: envNames().filter(
(name) => !dotenvNames().includes(name),
).length,
envFilesInCheckout: ['.env', '.env.local'].filter((f) =>
existsSync(join(worktree, f)),
),
};
console.log('Building', commit, 'keyless into', dist);
const build = spawnSync(
process.execPath,
[join(root, 'node_modules/vite/bin/vite.js'), 'build', '--outDir', dist],
{ cwd: worktree, env: scrubbed(), encoding: 'utf8', maxBuffer: 1 << 26 },
);
if (build.status !== 0) {
console.error(build.stdout + build.stderr);
process.exit(2);
}
// No .env credential value may reach the built files.
const values = envValues();
let leaks = 0;
const walk = (dir) => {
for (const name of readdirSync(dir)) {
const file = join(dir, name);
if (statSync(file).isDirectory()) walk(file);
else if (/\.(js|html|json|css|webmanifest)$/.test(name)) {
const text = readFileSync(file, 'utf8');
for (const value of values) if (text.includes(value)) leaks++;
}
}
};
walk(dist);
served.distCredentialValuesFound = leaks;
served.mainChunk = JSON.parse(
spawnSync(
process.execPath,
[join(root, 'scripts/check-main-chunk.mjs'), `--dist=${dist}`, '--json'],
{ cwd: root, encoding: 'utf8' },
).stdout || 'null',
);
preview = spawn(
process.execPath,
[
join(root, 'node_modules/vite/bin/vite.js'),
'preview',
'--outDir',
dist,
'--host',
base.hostname,
'--port',
base.port,
'--strictPort',
],
{ cwd: worktree, env: scrubbed(), stdio: 'ignore' },
);
for (let i = 0; i < 60; i++) {
try {
if ((await fetch(base.origin + '/')).ok) break;
} catch {
// Not up yet.
}
await new Promise((r) => setTimeout(r, 500));
}
}
try {
const response = await fetch(base.origin + '/', {
signal: AbortSignal.timeout(10000),
});
if (!response.ok) throw Error('HTTP ' + response.status);
} catch (error) {
console.error(
'No preview answers at ' + base.origin + ' (' + error.message + ').',
);
process.exit(2);
}
// The preview reports no paid key: its own keyless status routes.
const probe = async (path) => {
try {
const r = await fetch(base.origin + path, {
headers: { 'Sec-Fetch-Site': 'same-origin' },
});
return { status: r.status, body: await r.json().catch(() => null) };
} catch (error) {
return { error: error.message };
}
};
const keys = {
openaiHudSummary: await probe('/api/openai/hud-summary/status'),
firms: await probe('/api/firms/status'),
tomtom: await probe('/api/tomtom/status'),
mapillary: await probe('/api/mapillary/status'),
};
const paidKeysConfigured = [
keys.openaiHudSummary.body?.configured !== false && 'OpenAI',
keys.firms.body?.hasKey !== false && 'FIRMS',
keys.tomtom.body?.hasKey !== false && 'TomTom',
keys.mapillary.body?.configured !== false && 'Mapillary',
].filter(Boolean);
const dirtyFiles = dirty();
const soakRuns = quick
? ['--runs=2', '--swaps=10']
: ['--runs=6', '--swaps=50'];
const longRun = quick ? '--swaps=100' : '--swaps=500';
// label, script, layers, extra arguments, timeout in seconds.
const runs = [
['engine-switch', 'engine-switch', ['earthquakes']],
['director-engine-switch', 'director-engine-switch', []],
['layer-handoff', 'layer-handoff', ['flights']],
['layer-handoff--military', 'layer-handoff', ['military'], ['--military']],
[
'ground-layer-switch',
'ground-layer-switch',
['ais-live-vessels', 'air-quality', 'gnss-integrity', 'fire-perimeters'],
],
[
'contact-switch',
'contact-switch',
['military-awareness', 'military-installations'],
],
['local-adsb-engine-switch', 'local-adsb-engine-switch', ['local-adsb']],
['ghostmaps-switch', 'ghostmaps-switch', ['s2-ghostmaps']],
['wildlife-switch', 'wildlife-switch', ['wildlife-sightings']],
[
'open-layers-switch',
'open-layers-switch',
['drone-airspace', 'osm-places', 'sea-marks', 'ski-pistes'],
],
[
'weather-engine-switch',
'weather-engine-switch',
[
'weather-radar',
'weather-satellite',
'weather-lightning',
'weather-cyclones',
],
],
['wind-engine-switch', 'wind-engine-switch', ['wind']],
[
'infrastructure-engine-switch',
'infrastructure-engine-switch',
['local-dams', 'local-datacenters'],
],
[
'submarine-cables-engine-switch',
'submarine-cables-engine-switch',
['telegeography-submarine-cables'],
],
['directions-engine-switch', 'directions-engine-switch', ['directions']],
['nepal-locator-switch', 'nepal-locator-switch', ['bhote-koshi-locator']],
['nepal-event-switch', 'nepal-event-switch', ['bhote-koshi-2026']],
['traffic-engine-switch', 'traffic-engine-switch', ['traffic']],
[
'traffic-engine-switch--simulation',
'traffic-engine-switch',
['traffic'],
['--simulation'],
],
[
'street-level-engine-switch',
'street-level-engine-switch',
['street-level'],
],
[
'street-level-engine-switch--panoramax',
'street-level-engine-switch',
['street-level'],
['--panoramax'],
],
['radio-engine-switch', 'radio-engine-switch', ['radio']],
['imagery-contract', 'imagery-contract', ['recent-imagery']],
['fire-camera-switch', 'fire-camera-switch', ['alpr-cameras']],
[
'fire-camera-switch--fires',
'fire-camera-switch',
['local-firms'],
['--fires'],
],
['urban-engine-switch', 'urban-engine-switch', ['bikeshare']],
[
'urban-engine-switch--transit',
'urban-engine-switch',
['transit'],
['--transit'],
],
['cctv-engine-switch', 'cctv-engine-switch', ['cctv']],
['space-engine-switch', 'space-engine-switch', ['satellites']],
[
'space-engine-switch--missions',
'space-engine-switch',
['rocket-launches'],
['--missions'],
],
// One application per page, and voice across a switch: these block paid
// routes themselves but do not run under the strict harness, so they add
// no assertion manifest and no network ledger.
['app-once', 'app-once', [], [], 600, true],
['voice-engine-switch', 'voice-engine-switch', [], [], 300, true],
['strict-rapid-switch', 'strict-rapid-switch', []],
['strict-hidden-switch', 'strict-hidden-switch', []],
['strict-context-loss', 'strict-context-loss', []],
['strict-parked-release', 'strict-parked-release', []],
['strict-navara-hd', 'strict-navara-hd', []],
['strict-failure-matrix', 'strict-failure-matrix', [], [], 600],
['strict-cold-chunks', 'strict-cold-chunks', []],
['strict-prod-smoke', 'strict-prod-smoke', []],
// The app's visual styles, bloom, contact sonar, cockpit clouds, and the
// annotations, Draw, Import and the scope mask with the celestial ring on
// Navara (those are tools and display overlays, not catalog layers:
// TOOL_IDS).
['navara-visual-styles', 'navara-visual-styles', []],
['navara-bloom', 'navara-bloom', []],
['navara-sonar', 'navara-sonar', []],
['navara-cockpit-clouds', 'navara-cockpit-clouds', []],
['navara-annotations', 'navara-annotations', ['annotations']],
['navara-draw', 'navara-draw', ['draw']],
['navara-geo-import', 'navara-geo-import', ['geo-import']],
['navara-scope', 'navara-scope', ['scope-mask', 'celestial-ring']],
['strict-soak', 'strict-soak', [], soakRuns, 1200],
[
'strict-soak--long',
'strict-soak',
[],
['--runs=1', longRun, '--assert'],
1800,
],
];
// Ids a scenario covers that are tools, not catalog layers: their
// assertions are recorded, but the catalog comparison leaves them out.
const TOOL_IDS = new Set([
'annotations',
'draw',
'geo-import',
'scope-mask',
'celestial-ring',
]);
const scenarioIds = [
...new Set(
runs.flatMap(([, , ids]) => ids).filter((id) => !TOOL_IDS.has(id)),
),
].sort();
const directoryIds = APPLICATION_LAYER_DIRECTORY.map((l) => l.id).sort();
// The catalog as the live page has it, on both engines.
const liveCatalog = {};
{
const strict = createStrict({
origin: base.origin,
layers: [],
name: 'catalog',
});
const browser = await strict.launch();
try {
for (const engine of ENGINES) {
const page = await browser.newPage();
await page.goto(
`${base.origin}/?engine=${engine}&welcome=0#v=2&lat=30.27&lon=-97.74&alt=12000&pitch=-90&map=osm&l=`,
);
await waitForEngine(page, engine);
liveCatalog[engine] = await page.evaluate(() =>
[...window.__godsEyeView.dataManager.layers.keys()].sort(),
);
await page.close();
}
} finally {
await browser.close();
}
liveCatalog.violations = strict.ledger.violations;
}
const scenarios = [];
const report = {
commit: git('rev-parse', 'HEAD').trim(),
dirty: dirtyFiles.length > 0,
dirtyFiles,
dirtyFilesAtEnd: null,
chrome,
puppeteerExecutable: process.env.PUPPETEER_EXECUTABLE_PATH || null,
node: process.version,
platform: process.platform + ' ' + process.arch,
baseUrl: base.origin,
preview: served
? {
servedByRunner: true,
appCommit: served.commit,
appCheckoutClean: true,
scenariosFrom: 'the same clean checkout',
envFilesInCheckout: served.envFilesInCheckout,
blankedEnv: served.blankedEnv,
otherKeyLikeVariablesBlanked: served.otherKeyLikeVariablesBlanked,
distCredentialValuesFound: served.distCredentialValuesFound,
}
: { servedByRunner: false },
keyProbes: keys,
paidKeysConfigured,
mainChunk: served?.mainChunk ?? null,
networkPolicy: {
blockedFirst:
'Google (*.googleapis.com, *.google.com, maps.google.*, *.gstatic.com), Cesium ion (*.cesium.com), preview /api/openai/*, /api/realtime/*, /api/google/*',
allowed:
'the preview origin, each scenario’s declared fixture URLs, keyless probes answered by the harness, and the free tile hosts answered from the disk cache',
keylessProbes: (await import('./qa-map-policy.mjs')).KEYLESS_PROBES.map(
(p) => p.pattern,
),
tileHosts: (await import('./qa-map-policy.mjs')).STRICT_TILE_HOSTS,
},
catalog: {
directory: directoryIds,
live: liveCatalog,
scenarios: scenarioIds,
},
startedAt: new Date().toISOString(),
finishedAt: null,
passed: 0,
failed: 0,
networkViolations: 0,
coverage: null,
uncovered: null,
scenarios,
};
const save = () =>
writeFileSync(
join(output, 'acceptance.json'),
machinePaths(JSON.stringify(report, null, 2)) + '\n',
);
save();
for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
if (only && !only.includes(label)) continue;
const script = 'scripts/qa-' + name + '.mjs';
const evidence = join(output, 'evidence', label + '.json');
rmSync(evidence, { force: true });
const args = [base.origin, ...extra];
if (external) args.splice(0, 1, base.origin + '/');
else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`);
const start = Date.now();
// With --serve the scenarios run from the same clean checkout the app was
// built from, so other work in this tree cannot change them.
const scripts = served?.worktree ?? root;
const run = spawnSync(process.execPath, [join(scripts, script), ...args], {
encoding: 'utf8',
timeout: seconds * 1000,
maxBuffer: 1 << 26,
env: scrubbed(),
});
const status =
run.status === 0
? 'pass'
: run.error?.code === 'ETIMEDOUT'
? 'timeout'
: 'fail';
const log = machinePaths(
(run.stdout || '') + (run.stderr || '') + (run.error?.message || ''),
);
writeFileSync(join(output, label + '.log'), log);
const manifest = existsSync(evidence)
? JSON.parse(machinePaths(readFileSync(evidence, 'utf8')))
: null;
if (manifest)
writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n');
scenarios.push({
label,
script,
args: args.map((arg) =>
arg.includes(output) ? arg.replace(output, '<output>') : arg,
),
layers,
external: Boolean(external),
status,
exitCode: run.status,
seconds: Math.round((Date.now() - start) / 100) / 10,
log: label + '.log',
logSha256: createHash('sha256').update(log).digest('hex'),
evidence: manifest ? 'evidence/' + label + '.json' : null,
assertions: manifest?.assertions.length ?? 0,
coverage: manifest?.coverage ?? null,
picks: manifest?.picks ?? [],
network: manifest
? {
requests: manifest.network.requests,
violations: manifest.network.violations,
external: manifest.network.external,
tileCache: manifest.network.tileCache,
probes: manifest.network.probes,
routes: manifest.network.routes,
previewApi: manifest.network.previewApi,
fixtures: manifest.network.fixtures,
}
: null,
contracts: manifest?.contracts ?? null,
error: manifest?.error ?? null,
});
if (manifest?.soak) scenarios.at(-1).soak = manifest.soak;
report.passed = scenarios.filter((row) => row.status === 'pass').length;
report.failed = scenarios.length - report.passed;
report.networkViolations =
scenarios.reduce(
(sum, row) => sum + (row.network?.violations.length ?? 0),
0,
) + liveCatalog.violations.length;
save();
console.log(status.toUpperCase(), label, scenarios.at(-1).seconds);
}
// Coverage that cannot be faked: every catalog id needs an assertion on each
// engine from some scenario's manifest.
const catalogIds = [
...new Set([...directoryIds, ...(liveCatalog.navara ?? [])]),
].sort();
report.coverage = Object.fromEntries(
catalogIds.map((id) => [
id,
Object.fromEntries(
ENGINES.map((engine) => [
engine,
scenarios.reduce(
(sum, row) => sum + (row.coverage?.[id]?.[engine] ?? 0),
0,
),
]),
),
]),
);
report.uncovered = Object.entries(report.coverage).flatMap(([id, counts]) =>
ENGINES.filter((engine) => !counts[engine]).map(
(engine) => `${id}@${engine}`,
),
);
report.catalogMatches =
JSON.stringify(liveCatalog.navara) === JSON.stringify(directoryIds) &&
JSON.stringify(liveCatalog.cesium) === JSON.stringify(directoryIds) &&
JSON.stringify(scenarioIds) === JSON.stringify(directoryIds);
report.finishedAt = new Date().toISOString();
report.dirtyFilesAtEnd = dirty();
save();
console.log(
`${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` +
`${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` +
`paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`,
);
if (
report.failed ||
(!only && report.uncovered.length) ||
report.networkViolations ||
paidKeysConfigured.length ||
!report.catalogMatches
)
process.exitCode = 1;
// The preview child would keep this process alive: stop it and drop the
// checkout now.
cleanup();