From ed6ae54ef669831c5709611bf9670c89bd2c77ee Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 10 Nov 2026 14:10:46 +0400 Subject: [PATCH] Stop the strict acceptance from waiting for itself A scenario whose body ran past about 19 s idled until 45 s after its start, and twice one never exited and hit the runner's 300 s timeout after it had passed: something of Chrome outlived Browser.close and kept its stdout and stderr open. - scripts/qa/browser.mjs closes Chrome within 5 s, kills its process group and closes its pipes; the harness then finishes and exits with the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and prints the open handles and Chrome's process tree 5 s after finish. - Every strict page starts with its clock at the 18:30 UTC nearest to the real time, so Navara's sun is up over the test cities whenever the acceptance runs; a scenario can ask for its own instant or the real clock (Wildlife does, for its freshness fixtures). - The runner records a scenario that passed but did not exit as hung, kills the whole process group on a timeout, runs a failed scenario once more and reports one that passes then as flaky, and appends each run to --history=FILE; scripts/report-acceptance-flake.mjs reads it. --- docs/CURRENT-STATE.md | 11 +- scripts/qa-all-layer-switches.mjs | 128 ++++++++--- scripts/qa-app-once.mjs | 7 +- scripts/qa-imagery-contract.mjs | 6 +- scripts/qa-navara-bloom.mjs | 2 - scripts/qa-navara-cockpit-clouds.mjs | 17 +- scripts/qa-navara-visual-styles.mjs | 2 - scripts/qa-strict-acceptance.test.mjs | 191 ++++++++++++++++ scripts/qa-strict-harness.mjs | 108 ++++++--- scripts/qa-strict-harness.test.mjs | 305 ++++++++++++++++++++++++++ scripts/qa-strict-soak.mjs | 8 +- scripts/qa-voice-engine-switch.mjs | 7 +- scripts/qa-wildlife-switch.mjs | 2 + scripts/qa/acceptance.mjs | 84 +++++++ scripts/qa/browser.mjs | 73 ++++++ scripts/qa/navaraVisual.mjs | 18 +- scripts/qa/process.mjs | 119 ++++++++++ scripts/qa/time.mjs | 40 ++++ scripts/report-acceptance-flake.mjs | 15 ++ 19 files changed, 1039 insertions(+), 104 deletions(-) create mode 100644 scripts/qa-strict-acceptance.test.mjs create mode 100644 scripts/qa-strict-harness.test.mjs create mode 100644 scripts/qa/acceptance.mjs create mode 100644 scripts/qa/browser.mjs create mode 100644 scripts/qa/process.mjs create mode 100644 scripts/qa/time.mjs create mode 100644 scripts/report-acceptance-flake.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 34002bdf2..a66e8e6d9 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -416,7 +416,16 @@ one application, and only the globe-bound parts move to the next engine. --serve` builds the commit keyless in a clean worktree and runs every layer and switch scenario under a deny-by-default network policy, with free map tiles from a disk cache and a trap that fails any read of Cesium before it - has loaded. Every catalog layer must be asserted on both engines. Point + has loaded. Every catalog layer must be asserted on both engines. + Page clocks start at `2026-11-03T18:30:00Z` unless a scenario supplies + `time` (an ISO instant or `'real'`); `QA_EXIT_DIAGNOSTICS=1` preserves + shutdown resources and prints them five seconds after completion. + A timed-out scenario with passing evidence is `hung`; non-passing scenarios + retry once unless `--no-retry` is set, and a passing retry is reported as + `flaky` without failing the run. + `--history=FILE` appends run summaries, and + `node scripts/report-acceptance-flake.mjs FILE` reports each scenario's + share of non-pass runs, including flaky runs. Point `PUPPETEER_EXECUTABLE_PATH` at an installed Chrome if Puppeteer's pinned build is missing. Navara lights the globe by the real sun; every base map, street maps included, takes the night fill after dusk, so a scenario over diff --git a/scripts/qa-all-layer-switches.mjs b/scripts/qa-all-layer-switches.mjs index 7bd36b237..9759cbb7c 100644 --- a/scripts/qa-all-layer-switches.mjs +++ b/scripts/qa-all-layer-switches.mjs @@ -1,6 +1,7 @@ import { execFileSync, spawn, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import { + appendFileSync, existsSync, mkdirSync, mkdtempSync, @@ -24,17 +25,31 @@ import { localPreview, waitForEngine, } from './qa-strict-harness.mjs'; +import { closeBrowser, exitAfterFlush } from './qa/browser.mjs'; +import { runScenario } from './qa/process.mjs'; +import { + classifyAttempt, + historyLine, + retryStatus, + statusCounts, +} from './qa/acceptance.mjs'; // The strict acceptance: every browser scenario against one keyless local // preview, under the strict network policy (scripts/qa-strict-harness.mjs), // writing /acceptance.json, one log and one assertion manifest per -// scenario. +// scenario. A timed-out pass is hung; a retry that passes is flaky and does +// not fail the run. QA_EXIT_DIAGNOSTICS=1 preserves Chrome resources and +// reports them five seconds after strict completion instead of forcing exit. +// Pages start at 2026-11-03T18:30:00Z unless createStrict's time overrides it. // // node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 // [--serve[=]] build (default HEAD) keyless in a clean // worktree and serve it with every .env key blank // [--scratch=DIR] where --serve puts its checkout and build // [--only=a,b] run only these labels +// [--no-retry] disable the one retry of a non-passing scenario +// [--history=FILE] append a JSONL run summary; inspect it with +// node scripts/report-acceptance-flake.mjs FILE // [--quick] the soak with 2 runs of 10 switches and 100 // switches for the long run // @@ -42,6 +57,7 @@ import { // the live page and cross-checked with src/app/layers/directory.js) has no // assertion on either engine, or when any network policy violation shows. +const startedAt = new Date().toISOString(); const base = localPreview(process.argv[2]); const output = process.argv[3]?.startsWith('--') ? mkdtempSync(join(tmpdir(), 'gev-acceptance-')) @@ -67,7 +83,7 @@ let chrome; try { const browser = await puppeteer.launch({ headless: true }); chrome = await browser.version(); - await browser.close(); + await closeBrowser(browser); } catch (error) { console.error( 'Puppeteer cannot start Chrome: ' + @@ -429,7 +445,7 @@ const liveCatalog = {}; await page.close(); } } finally { - await browser.close(); + await closeBrowser(browser); } liveCatalog.violations = strict.ledger.violations; } @@ -475,10 +491,11 @@ const report = { live: liveCatalog, scenarios: scenarioIds, }, - startedAt: new Date().toISOString(), + startedAt, finishedAt: null, passed: 0, failed: 0, + counts: statusCounts([]), networkViolations: 0, coverage: null, uncovered: null, @@ -495,7 +512,6 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { if (only && !only.includes(label)) continue; const script = 'scripts/qa-' + name + '.mjs'; const evidence = join(output, 'evidence', label + '.json'); - rmSync(evidence, { force: true }); const args = [base.origin, ...extra]; if (external) args.splice(0, 1, base.origin + '/'); else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`); @@ -503,27 +519,69 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { // With --serve the scenarios run from the same clean checkout the app was // built from, so other work in this tree cannot change them. const scripts = served?.worktree ?? root; - const run = spawnSync(process.execPath, [join(scripts, script), ...args], { - encoding: 'utf8', - timeout: seconds * 1000, - maxBuffer: 1 << 26, - env: scrubbed(), - }); - const status = - run.status === 0 - ? 'pass' - : run.error?.code === 'ETIMEDOUT' - ? 'timeout' - : 'fail'; - const log = machinePaths( - (run.stdout || '') + (run.stderr || '') + (run.error?.message || ''), - ); - writeFileSync(join(output, label + '.log'), log); - const manifest = existsSync(evidence) - ? JSON.parse(machinePaths(readFileSync(evidence, 'utf8'))) + const attempts = []; + for (let attempt = 1; attempt <= (flag('no-retry') ? 1 : 2); attempt++) { + rmSync(evidence, { force: true }); + const attemptStart = Date.now(); + const run = await runScenario( + process.execPath, + [join(scripts, script), ...args], + { + timeout: seconds * 1000, + env: scrubbed(), + }, + ); + const log = machinePaths( + (run.stdout || '') + (run.stderr || '') + (run.error?.message || ''), + ); + let manifest = null; + let evidenceError = null; + if (existsSync(evidence)) { + try { + manifest = JSON.parse(machinePaths(readFileSync(evidence, 'utf8'))); + } catch (error) { + evidenceError = 'Invalid evidence: ' + error.message; + } + } + const status = + evidenceError && !run.error + ? 'fail' + : classifyAttempt(run, manifest, { external }); + const logFile = `${label}.attempt-${attempt}.log`; + const evidenceFile = manifest + ? `evidence/${label}.attempt-${attempt}.json` + : null; + writeFileSync(join(output, logFile), log); + if (manifest) + writeFileSync( + join(output, evidenceFile), + JSON.stringify(manifest, null, 2) + '\n', + ); + attempts.push({ + status, + exitCode: run.status, + seconds: Math.round((Date.now() - attemptStart) / 100) / 10, + log: logFile, + logSha256: createHash('sha256').update(log).digest('hex'), + evidence: evidenceFile, + error: evidenceError ?? manifest?.error ?? run.error?.message ?? null, + }); + writeFileSync(join(output, label + '.log'), log); + if (manifest) + writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n'); + console.log( + status.toUpperCase(), + label, + attempts.at(-1).seconds, + `attempt ${attempt}`, + ); + if (status === 'pass' || attempt === 2 || flag('no-retry')) break; + } + const last = attempts.at(-1); + const status = retryStatus(attempts[0].status, attempts[1]?.status); + const manifest = last.evidence + ? JSON.parse(readFileSync(join(output, last.evidence), 'utf8')) : null; - if (manifest) - writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n'); scenarios.push({ label, script, @@ -533,10 +591,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { layers, external: Boolean(external), status, - exitCode: run.status, + exitCode: last.exitCode, seconds: Math.round((Date.now() - start) / 100) / 10, log: label + '.log', - logSha256: createHash('sha256').update(log).digest('hex'), + logSha256: last.logSha256, + attempts, evidence: manifest ? 'evidence/' + label + '.json' : null, assertions: manifest?.assertions.length ?? 0, coverage: manifest?.coverage ?? null, @@ -554,10 +613,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { } : null, contracts: manifest?.contracts ?? null, - error: manifest?.error ?? null, + error: last.error, }); if (manifest?.soak) scenarios.at(-1).soak = manifest.soak; - report.passed = scenarios.filter((row) => row.status === 'pass').length; + report.counts = statusCounts(scenarios); + report.passed = report.counts.pass + report.counts.flaky; report.failed = scenarios.length - report.passed; report.networkViolations = scenarios.reduce( @@ -599,9 +659,18 @@ report.catalogMatches = report.finishedAt = new Date().toISOString(); report.dirtyFilesAtEnd = dirty(); save(); +if (flag('history')) + appendFileSync( + resolve(String(flag('history'))), + JSON.stringify(historyLine(report)) + '\n', + ); console.log( `${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` + `${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` + + Object.entries(report.counts) + .map(([status, count]) => `${count} ${status}`) + .join(', ') + + ', ' + `paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`, ); if ( @@ -615,3 +684,4 @@ if ( // The preview child would keep this process alive: stop it and drop the // checkout now. cleanup(); +await exitAfterFlush(); diff --git a/scripts/qa-app-once.mjs b/scripts/qa-app-once.mjs index 8e39f7fe3..6c850a5ef 100644 --- a/scripts/qa-app-once.mjs +++ b/scripts/qa-app-once.mjs @@ -1,3 +1,4 @@ +import { closeBrowser, exitAfterFlush } from './qa/browser.mjs'; import { localPreview, waitForEngine } from './qa-strict-harness.mjs'; // The application is created once per page: switching engines detaches and // attaches only the engine-bound parts. This gate runs three Cesium→Navara→ @@ -413,6 +414,10 @@ try { console.log(JSON.stringify(report, null, 2)); if (!reportOnly) console.log('PASS application created once; session kept'); +} catch (error) { + console.error(error); + process.exitCode = 1; } finally { - await browser.close(); + await closeBrowser(browser); } +await exitAfterFlush(); diff --git a/scripts/qa-imagery-contract.mjs b/scripts/qa-imagery-contract.mjs index c0a4cd4dd..1a7a34c50 100644 --- a/scripts/qa-imagery-contract.mjs +++ b/scripts/qa-imagery-contract.mjs @@ -6,6 +6,7 @@ // // node scripts/qa-imagery-contract.mjs http://127.0.0.1: // [--layers=recent-imagery] [--evidence=FILE] +import { closeBrowser } from './qa/browser.mjs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -470,9 +471,8 @@ try { report.errors.push(e.stack); check('driver completes', false); } finally { - await browser.close(); + await closeBrowser(browser); await fs.writeFile(output, JSON.stringify(report, null, 2)); } const failed = report.checks.some((c) => !c.pass); -await strict.finish({ failed: failed ? 'imagery checks failed' : null }); -if (failed) process.exitCode = 1; +await strict.finishAndExit({ failed: failed ? 'imagery checks failed' : null }); diff --git a/scripts/qa-navara-bloom.mjs b/scripts/qa-navara-bloom.mjs index ae9ad105c..2029ddb16 100644 --- a/scripts/qa-navara-bloom.mjs +++ b/scripts/qa-navara-bloom.mjs @@ -21,7 +21,6 @@ import { diff, freeze, pickStyle, - pinDaytime, saveCapture, stats, visualPageUrl, @@ -59,7 +58,6 @@ await strict.run({}, async (browser) => { page.on('pageerror', (error) => errors.push(error.message)); // The same sun on every run: the frame's brightness decides what blooms // and what each style's signature reads. - await pinDaytime(page); await page.evaluateOnNewDocument(() => { localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); }); diff --git a/scripts/qa-navara-cockpit-clouds.mjs b/scripts/qa-navara-cockpit-clouds.mjs index 5dffd909f..78899958a 100644 --- a/scripts/qa-navara-cockpit-clouds.mjs +++ b/scripts/qa-navara-cockpit-clouds.mjs @@ -50,6 +50,7 @@ const asked = []; const strict = createStrict({ origin: base.origin, layers: [], + time: MIDDAY_UTC, routes: [ { pattern: `${base.origin}/api/weather-effects*`, @@ -105,21 +106,9 @@ await strict.run({}, async (browser) => { const page = await browser.newPage(); const errors = []; page.on('pageerror', (error) => errors.push(error.message)); - await page.evaluateOnNewDocument((utc) => { + await page.evaluateOnNewDocument(() => { localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); - const offset = Date.parse(utc) - Date.now(); - const RealDate = Date; - class ShiftedDate extends RealDate { - constructor(...args) { - if (args.length === 0) super(RealDate.now() + offset); - else super(...args); - } - static now() { - return RealDate.now() + offset; - } - } - globalThis.Date = ShiftedDate; - }, MIDDAY_UTC); + }); await page.setViewport({ width: 1280, height: 800 }); await page.goto(visualPageUrl(base.origin, 'navara', '', LOW_VIEW), { waitUntil: 'domcontentloaded', diff --git a/scripts/qa-navara-visual-styles.mjs b/scripts/qa-navara-visual-styles.mjs index 2b6931755..9a5668e57 100644 --- a/scripts/qa-navara-visual-styles.mjs +++ b/scripts/qa-navara-visual-styles.mjs @@ -24,7 +24,6 @@ import { freeze, histogramShift, pickStyle, - pinDaytime, saveCapture, stats, visualPageUrl, @@ -51,7 +50,6 @@ await strict.run({}, async (browser) => { page.on('pageerror', (error) => errors.push(error.message)); // The same sun on every run: the frame's brightness decides what blooms // and what each style's signature reads. - await pinDaytime(page); await page.evaluateOnNewDocument(() => { localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); }); diff --git a/scripts/qa-strict-acceptance.test.mjs b/scripts/qa-strict-acceptance.test.mjs new file mode 100644 index 000000000..37aea42c6 --- /dev/null +++ b/scripts/qa-strict-acceptance.test.mjs @@ -0,0 +1,191 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { + classifyAttempt, + flakeReport, + formatFlakeReport, + historyLine, + retryStatus, +} from './qa/acceptance.mjs'; +import { processTree, readProcesses, runScenario } from './qa/process.mjs'; + +test('a timeout after passing evidence is hung, even after a zero exit', () => { + const timeout = { status: 0, error: { code: 'ETIMEDOUT' } }; + assert.equal(classifyAttempt(timeout, { status: 'pass' }), 'hung'); + assert.equal(classifyAttempt(timeout, { status: 'fail' }), 'timeout'); + assert.equal(classifyAttempt(timeout, null), 'timeout'); +}); + +test('a pass requires a successful exit and passing evidence', () => { + const pass = { status: 'pass' }; + assert.equal(classifyAttempt({ status: 0 }, pass), 'pass'); + assert.equal(classifyAttempt({ status: 1 }, pass), 'fail'); + assert.equal(classifyAttempt({ status: 0, signal: 'SIGTERM' }, pass), 'fail'); + assert.equal( + classifyAttempt({ status: 0, error: { code: 'ENOBUFS' } }, pass), + 'fail', + ); + assert.equal(classifyAttempt({ status: 0 }, { status: 'fail' }), 'fail'); + assert.equal(classifyAttempt({ status: 0 }, null), 'fail'); + assert.equal( + classifyAttempt({ status: 0 }, null, { external: true }), + 'pass', + ); +}); + +test('a passing retry is flaky; no retry preserves the first status', () => { + for (const status of ['fail', 'timeout', 'hung']) { + assert.equal(retryStatus(status, 'pass'), 'flaky'); + assert.equal(retryStatus(status), status); + assert.equal(retryStatus(status, 'fail'), 'fail'); + } + assert.equal(retryStatus('pass'), 'pass'); +}); + +const report = (scenarios) => ({ + commit: 'abc123', + dirty: true, + startedAt: '2026-11-10T12:00:00Z', + finishedAt: '2026-11-10T12:00:12.500Z', + scenarios: Object.entries(scenarios).map(([label, status]) => ({ + label, + status, + })), +}); + +test('history records provenance, duration, every status, and selected scenarios', () => { + assert.deepEqual( + historyLine( + report({ a: 'pass', b: 'flaky', c: 'hung', d: 'timeout', e: 'fail' }), + ), + { + commit: 'abc123', + dirty: true, + startedAt: '2026-11-10T12:00:00Z', + finishedAt: '2026-11-10T12:00:12.500Z', + seconds: 12.5, + counts: { pass: 1, flaky: 1, hung: 1, timeout: 1, fail: 1 }, + scenarios: ['a', 'b', 'c', 'd', 'e'], + nonPass: [ + { label: 'b', status: 'flaky' }, + { label: 'c', status: 'hung' }, + { label: 'd', status: 'timeout' }, + { label: 'e', status: 'fail' }, + ], + }, + ); +}); + +test('flake shares include flaky runs and exclude scenarios absent from partial runs', () => { + const history = [ + historyLine(report({ a: 'pass', b: 'flaky', c: 'fail' })), + historyLine(report({ a: 'pass', b: 'pass' })), + historyLine(report({ a: 'hung' })), + ]; + assert.deepEqual(flakeReport(history), { + scenarios: [ + { label: 'c', runs: 1, passes: 0, nonPassShare: 1 }, + { label: 'b', runs: 2, passes: 1, nonPassShare: 0.5 }, + { label: 'a', runs: 3, passes: 2, nonPassShare: 1 / 3 }, + ], + overall: { runs: 6, passes: 3, nonPassShare: 0.5 }, + }); + assert.match(formatFlakeReport(history), /overall\t6\t3\t50.0%$/); + assert.deepEqual(flakeReport([]), { + scenarios: [], + overall: { runs: 0, passes: 0, nonPassShare: 0 }, + }); +}); + +test('the report CLI reads JSON lines and rejects malformed input', async (t) => { + const dir = mkdtempSync('tmp_acceptance_history_'); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const file = `${dir}/history.jsonl`; + writeFileSync( + file, + JSON.stringify(historyLine(report({ a: 'flaky' }))) + '\n', + ); + const run = promisify(execFile); + const script = new URL('./report-acceptance-flake.mjs', import.meta.url); + const result = await run(process.execPath, [script.pathname, file]); + assert.match(result.stdout, /a\t1\t0\t100.0%/); + writeFileSync(file, '{broken}\n'); + await assert.rejects(run(process.execPath, [script.pathname, file]), { + code: 1, + }); +}); + +test('process trees include detached descendants and orphaned group members only', () => { + const rows = [ + { pid: 10, ppid: 1, pgid: 10 }, + { pid: 20, ppid: 10, pgid: 20 }, + { pid: 21, ppid: 20, pgid: 20 }, + { pid: 22, ppid: 1, pgid: 20 }, + { pid: 99, ppid: 1, pgid: 99 }, + ]; + assert.deepEqual( + processTree(10, rows).map((row) => row.pid), + [10, 20, 21], + ); + assert.deepEqual( + processTree(20, rows).map((row) => row.pid), + [20, 21, 22], + ); +}); + +test('scenario spawning captures both streams and reports spawn failures', async () => { + const run = await runScenario( + process.execPath, + ['-e', "console.log('out'); console.error('err'); process.exitCode = 3"], + { timeout: 5000 }, + ); + assert.equal(run.status, 3); + assert.equal(run.stdout, 'out\n'); + assert.equal(run.stderr, 'err\n'); + assert.equal(run.error, null); + const missing = await runScenario('./tmp_acceptance_missing_executable', [], { + timeout: 5000, + }); + assert.equal(missing.error.code, 'ENOENT'); +}); + +test('timeout kills the scenario and a detached group after its leader exits', async (t) => { + // These Node children stand in for Chrome and a renderer; no browser is run. + const renderer = 'setInterval(() => {}, 1000)'; + const browser = ` + const { spawn } = require('node:child_process'); + const child = spawn(process.execPath, ['-e', ${JSON.stringify(renderer)}], { stdio: 'inherit' }); + console.log('renderer', child.pid); + setTimeout(() => process.exit(0), 1500); + `; + const scenario = ` + const { spawn } = require('node:child_process'); + const child = spawn(process.execPath, ['-e', ${JSON.stringify(browser)}], { detached: true, stdio: 'inherit' }); + console.log('browser', child.pid); + console.log('scenario', process.pid); + setInterval(() => {}, 1000); + `; + const run = await runScenario(process.execPath, ['-e', scenario], { + timeout: 2600, + }); + const pids = [ + ...run.stdout.matchAll(/(?:browser|renderer|scenario) (\d+)/g), + ].map((match) => Number(match[1])); + t.after(() => { + for (const pid of pids) { + try { + process.kill(-pid, 'SIGKILL'); + } catch {} + } + }); + assert.equal(run.error.code, 'ETIMEDOUT'); + assert.equal(pids.length, 3); + await new Promise((resolve) => setTimeout(resolve, 100)); + const survivors = readProcesses().filter( + (row) => pids.includes(row.pid) && !/defunct/.test(row.command), + ); + assert.deepEqual(survivors, []); +}); diff --git a/scripts/qa-strict-harness.mjs b/scripts/qa-strict-harness.mjs index 3a6d6100c..de9d9821f 100644 --- a/scripts/qa-strict-harness.mjs +++ b/scripts/qa-strict-harness.mjs @@ -27,6 +27,8 @@ import { STRICT_TILE_HOSTS, } from './qa-map-policy.mjs'; import { createMissLoader, tileCacheFile } from './qa/tileCache.mjs'; +import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs'; +import { DAYTIME, shiftTime } from './qa/time.mjs'; export const ENGINES = Object.freeze(['cesium', 'navara']); /** What the Globe Engine picker offers: the engines and Navara HD. */ @@ -92,7 +94,14 @@ const live = (response) => Boolean(response.remoteIPAddress); */ export async function installStrictNetwork( browser, - { origin, fixtures = [], routes = [], tileHosts = STRICT_TILE_HOSTS, ledger }, + { + origin, + fixtures = [], + routes = [], + tileHosts = STRICT_TILE_HOSTS, + ledger, + time = DAYTIME, + }, ) { const preview = new URL(origin); fixtures = [...fixtures, ...routes.map((route) => route.pattern)]; @@ -298,7 +307,7 @@ export async function installStrictNetwork( // requests, and a live answer would still show up as a violation. if (type === 'page' || type === 'iframe') steps.push(intercept(session, type)); - if (type === 'page') steps.push(trapCesiumGlobal(session, ledger)); + if (type === 'page') steps.push(trapCesiumGlobal(session, ledger, time)); await Promise.all(steps); }; const attach = async (sessionId, info, waiting) => { @@ -344,7 +353,7 @@ export async function installStrictNetwork( * has assigned it are logged with their stack in `ledger.cesiumBeforeLoad` * and answer `undefined`, as production would. */ -async function trapCesiumGlobal(session, ledger) { +async function trapCesiumGlobal(session, ledger, time) { const binding = '__gevStrictCesiumRead'; session.on('Runtime.bindingCalled', (event) => { if (event.name !== binding) return; @@ -358,7 +367,8 @@ async function trapCesiumGlobal(session, ledger) { await session.send('Runtime.addBinding', { name: binding }); await session.send('Page.enable'); await session.send('Page.addScriptToEvaluateOnNewDocument', { - source: `(() => { + source: `(${shiftTime})(${JSON.stringify(time)}); + (() => { if (window.top !== window) return; const seen = new Set(); let sdk; @@ -630,6 +640,7 @@ export async function openPage(browser, { shaderErrors = false } = {}) { * @param {readonly string[]} [options.tileHosts] Hosts the disk cache * answers (default STRICT_TILE_HOSTS). * @param {string} [options.name] Scenario name for the manifest. + * @param {string} [options.time] ISO instant for the page clock, or 'real'. */ export function createStrict({ origin, @@ -638,7 +649,14 @@ export function createStrict({ routes = [], tileHosts = STRICT_TILE_HOSTS, name = path.basename(process.argv[1] ?? 'scenario', '.mjs'), + time = DAYTIME, }) { + assert.ok( + typeof time === 'string' && + (time === 'real' || Number.isFinite(Date.parse(time))), + 'valid page time', + ); + const browserPids = []; const args = strictArguments(); if (args.layers) assert.deepEqual( @@ -685,28 +703,45 @@ export function createStrict({ ...options, args: [...CHROME_ARGS, ...args], }); - await installStrictNetwork(browser, { - origin, - fixtures, - routes, - tileHosts, - ledger, - }); + const pid = browser.process()?.pid; + if (pid) browserPids.push(pid); + try { + await installStrictNetwork(browser, { + origin, + fixtures, + routes, + tileHosts, + ledger, + time, + }); + } catch (error) { + await closeBrowser(browser); + throw error; + } return browser; }, /** Launch with `options`, run `body(browser)`, close, then finish(). */ async run(options, body) { - const browser = await strict.launch(options); + let browser; let failed = null; try { + browser = await strict.launch(options); await body(browser); } catch (error) { failed = error; } finally { - await browser.close().catch(() => {}); + await closeBrowser(browser); } - await strict.finish({ failed }); - if (failed) throw failed; + await strict.finishAndExit({ failed }); + }, + async finishAndExit({ failed = null } = {}) { + try { + await strict.finish({ failed }); + } catch (error) { + failed = error; + } + if (failed) console.error(failed); + await exitAfterFlush(failed ? 1 : process.exitCode ?? 0); }, /** Record an assertion that already passed. */ record, @@ -788,6 +823,30 @@ export function createStrict({ fixtures, tileHosts: [...tileHosts], }; + let validationError = null; + if (!failed) { + try { + assert.equal( + ledger.violations.length, + 0, + 'network policy violations: ' + + JSON.stringify(ledger.violations.slice(0, 5)), + ); + assert.deepEqual( + ledger.cesiumBeforeLoad, + [], + 'the page read the Cesium global before Cesium.js loaded', + ); + assert.deepEqual(missing, [], 'layers without an assertion'); + assert.ok( + picks.some((pick) => pick.via === 'pointer+keyboard'), + 'switch at least once with real input on the picker', + ); + } catch (error) { + validationError = error; + failed = error; + } + } const manifest = { scenario: name, argv: process.argv.slice(2), @@ -825,23 +884,8 @@ export function createStrict({ console.log('NETWORK VIOLATION', JSON.stringify(v)); for (const read of ledger.cesiumBeforeLoad.slice(0, 10)) console.log('CESIUM BEFORE LOAD', JSON.stringify(read)); - if (failed) return; - assert.equal( - ledger.violations.length, - 0, - 'network policy violations: ' + - JSON.stringify(ledger.violations.slice(0, 5)), - ); - assert.deepEqual( - ledger.cesiumBeforeLoad, - [], - 'the page read the Cesium global before Cesium.js loaded', - ); - assert.deepEqual(missing, [], 'layers without an assertion'); - assert.ok( - picks.some((pick) => pick.via === 'pointer+keyboard'), - 'switch at least once with real input on the picker', - ); + diagnoseExit(browserPids); + if (validationError) throw validationError; }, }; return strict; diff --git a/scripts/qa-strict-harness.test.mjs b/scripts/qa-strict-harness.test.mjs new file mode 100644 index 000000000..cd2b5b7df --- /dev/null +++ b/scripts/qa-strict-harness.test.mjs @@ -0,0 +1,305 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { EventEmitter } from 'node:events'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import vm from 'node:vm'; +import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs'; +import { DAYTIME, daytimeNear, shiftTime } from './qa/time.mjs'; +import { + createStrict, + installStrictNetwork, + networkLedger, +} from './qa-strict-harness.mjs'; + +function fakeBrowser(t, close = async () => {}) { + t.mock.method(process, 'kill', () => true); + const stdout = { destroy: t.mock.fn() }; + const stderr = { destroy: t.mock.fn() }; + return { + close, + process: () => ({ pid: 12345, stdout, stderr }), + stdout, + stderr, + }; +} + +test('closeBrowser kills the remaining group and drops both pipes after closing', async (t) => { + const browser = fakeBrowser(t); + const result = await closeBrowser(browser); + assert.equal(result.closed, true); + assert.equal(result.killed, true); + assert.ok(result.ms >= 0); + assert.deepEqual( + process.kill.mock.calls.map((call) => call.arguments), + [ + [-12345, 0], + [-12345, 'SIGKILL'], + ], + ); + assert.equal(browser.stdout.destroy.mock.callCount(), 1); + assert.equal(browser.stderr.destroy.mock.callCount(), 1); +}); + +test('closeBrowser bounds a stuck close and tolerates a late rejection', async (t) => { + let reject; + const browser = fakeBrowser( + t, + () => + new Promise((resolve, failure) => { + reject = failure; + }), + ); + const result = await closeBrowser(browser, { deadlineMs: 10 }); + assert.equal(result.closed, false); + assert.equal(result.killed, true); + assert.ok(result.ms < 1000); + reject(new Error('closed later')); +}); + +test('missing processes, dead groups and broken streams never prevent cleanup', async (t) => { + const browser = fakeBrowser(t, () => { + throw new Error('close failed'); + }); + t.mock.method(process, 'kill', () => { + throw new Error('ESRCH'); + }); + browser.stdout.destroy = () => { + throw new Error('pipe failed'); + }; + const result = await closeBrowser(browser); + assert.equal(result.closed, false); + assert.equal(result.killed, false); + assert.equal(browser.stderr.destroy.mock.callCount(), 1); + for (const process of [ + () => null, + () => ({}), + () => { + throw new Error('no process'); + }, + ]) { + const missing = await closeBrowser({ process, close: async () => {} }); + assert.equal(missing.closed, true); + assert.equal(missing.killed, false); + } + assert.equal((await closeBrowser(undefined)).closed, false); +}); + +test('diagnostic shutdown preserves the process group and pipes', async (t) => { + const old = process.env.QA_EXIT_DIAGNOSTICS; + process.env.QA_EXIT_DIAGNOSTICS = '1'; + t.after(() => { + if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS; + else process.env.QA_EXIT_DIAGNOSTICS = old; + }); + const browser = fakeBrowser(t); + const result = await closeBrowser(browser); + assert.equal(result.closed, true); + assert.equal(result.killed, false); + assert.equal(process.kill.mock.callCount(), 0); + assert.equal(browser.stdout.destroy.mock.callCount(), 0); + assert.equal(browser.stderr.destroy.mock.callCount(), 0); + let callback; + const unref = t.mock.fn(); + t.mock.method(globalThis, 'setTimeout', (fn, delay) => { + callback = fn; + assert.equal(delay, 5000); + return { unref }; + }); + t.mock.method(console, 'error', () => {}); + diagnoseExit([]); + assert.equal(unref.mock.callCount(), 1); + assert.equal(console.error.mock.callCount(), 0); + callback(); + assert.equal(console.error.mock.calls[0].arguments[0], 'QA EXIT resources'); + assert.match( + console.error.mock.calls[1].arguments[0], + /pid ppid pgid elapsed command/, + ); +}); + +test('the page clock moves to the nearest 18:30 UTC, at most 12 hours away', () => { + const at = (iso) => daytimeNear(Date.parse(iso)); + assert.equal(at('2026-11-10T10:00:00Z'), '2026-11-10T18:30:00.000Z'); + assert.equal(at('2026-11-10T03:00:00Z'), '2026-11-09T18:30:00.000Z'); + assert.equal(at('2026-12-31T23:00:00Z'), '2026-12-31T18:30:00.000Z'); + assert.equal(at('2027-01-01T07:00:00Z'), '2027-01-01T18:30:00.000Z'); + assert.ok(Math.abs(Date.parse(DAYTIME) - Date.now()) <= 12 * 3_600_000); +}); + +test('the shifted clock ticks, preserves explicit dates, and supports the real clock', () => { + const context = vm.createContext({}); + vm.runInContext( + ` + globalThis.tick = 1000; + const NativeDate = Date; + globalThis.Date = class extends NativeDate { + static now() { return tick; } + }; + globalThis.original = Date; + (${shiftTime})(${JSON.stringify(DAYTIME)}); + `, + context, + ); + assert.equal(vm.runInContext('Date.now()', context), Date.parse(DAYTIME)); + assert.equal(vm.runInContext('new Date().toISOString()', context), DAYTIME); + assert.equal( + vm.runInContext('tick += 250; Date.now()', context), + Date.parse(DAYTIME) + 250, + ); + assert.equal(vm.runInContext('new Date(0).getTime()', context), 0); + assert.equal( + vm.runInContext('Date.UTC(2026, 0, 1)', context), + Date.UTC(2026, 0, 1), + ); + vm.runInContext( + `globalThis.Date = original; (${shiftTime})('real');`, + context, + ); + assert.equal(vm.runInContext('Date === original', context), true); +}); + +test('diagnostics keeps natural exit and the requested failure code', async (t) => { + const old = process.env.QA_EXIT_DIAGNOSTICS; + const code = process.exitCode; + process.env.QA_EXIT_DIAGNOSTICS = '1'; + t.after(() => { + if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS; + else process.env.QA_EXIT_DIAGNOSTICS = old; + process.exitCode = code; + }); + t.mock.method(process, 'exit', () => {}); + await exitAfterFlush(1); + assert.equal(process.exit.mock.callCount(), 0); + assert.equal(process.exitCode, 1); +}); + +test('each attached page gets its clock script before it is resumed', async (t) => { + // Network installation creates a tile-cache directory; this test makes no requests. + t.mock.method(fs, 'mkdirSync', () => {}); + for (const time of [undefined, '2026-07-15T18:00:00Z', 'real']) { + const sessions = new Map(); + const root = new EventEmitter(); + root.send = async () => {}; + root.connection = () => ({ session: (id) => sessions.get(id) }); + await installStrictNetwork( + { target: () => ({ createCDPSession: async () => root }) }, + { + origin: 'http://localhost:4242', + ledger: networkLedger(), + time, + }, + ); + for (let id = 1; id <= 2; id++) { + const session = new EventEmitter(); + const calls = []; + const ready = Promise.withResolvers(); + session.send = async (method, params) => { + calls.push({ method, params }); + if (method === 'Runtime.runIfWaitingForDebugger') ready.resolve(); + }; + sessions.set(id, session); + root.emit('Target.attachedToTarget', { + sessionId: id, + targetInfo: { type: 'page' }, + waitingForDebugger: true, + }); + await ready.promise; + const script = calls.find( + (call) => call.method === 'Page.addScriptToEvaluateOnNewDocument', + ); + assert.ok(script); + assert.equal(calls.at(-1).method, 'Runtime.runIfWaitingForDebugger'); + const context = vm.createContext({}); + vm.runInContext( + 'globalThis.window = globalThis; window.top = window; globalThis.original = Date;', + context, + ); + vm.runInContext(script.params.source, context); + if (time === 'real') + assert.equal(vm.runInContext('Date === original', context), true); + else + assert.ok( + Math.abs( + vm.runInContext('Date.now()', context) - + Date.parse(time ?? DAYTIME), + ) < 1000, + ); + assert.equal( + vm.runInContext( + 'Object.getOwnPropertyDescriptor(window, "Cesium").get instanceof Function', + context, + ), + true, + ); + } + } +}); + +test('finish writes failing evidence for final validation and body failures', async (t) => { + const dir = fs.mkdtempSync('tmp_acceptance_evidence_'); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const file = `${dir}/evidence.json`; + const old = process.env.QA_STRICT_EVIDENCE; + process.env.QA_STRICT_EVIDENCE = file; + t.after(() => { + if (old === undefined) delete process.env.QA_STRICT_EVIDENCE; + else process.env.QA_STRICT_EVIDENCE = old; + }); + t.mock.method(console, 'log', () => {}); + const strict = createStrict({ + origin: 'http://localhost:4242', + layers: ['test'], + }); + await assert.rejects(strict.finish(), /layers without an assertion/); + let manifest = JSON.parse(fs.readFileSync(file, 'utf8')); + assert.equal(manifest.status, 'fail'); + assert.match(manifest.error, /layers without an assertion/); + await createStrict({ origin: 'http://localhost:4242', layers: [] }).finish({ + failed: new Error('body failed'), + }); + manifest = JSON.parse(fs.readFileSync(file, 'utf8')); + assert.equal(manifest.status, 'fail'); + assert.match(manifest.error, /body failed/); + assert.throws( + () => + createStrict({ + origin: 'http://localhost:4242', + layers: [], + time: 'invalid', + }), + /valid page time/, + ); +}); + +test('strict.run flushes large output and exits despite active resources on success or failure', async () => { + const harness = new URL('./qa-strict-harness.mjs', import.meta.url).href; + for (const fail of [false, true]) { + const source = ` + import { createStrict } from ${JSON.stringify(harness)}; + const strict = createStrict({ origin: 'http://localhost:4242', layers: [] }); + strict.launch = async () => ({ close: async () => console.log('CLOSED'), process: () => null }); + strict.finish = async () => console.log('STRICT'); + setInterval(() => {}, 1000); + await strict.run({}, async () => { + process.stdout.write('x'.repeat(300000) + 'OUT_END\\n'); + process.stderr.write('y'.repeat(300000) + 'ERR_END\\n'); + if (${fail}) throw new Error('body failed'); + }); + `; + const result = await promisify(execFile)( + process.execPath, + ['--input-type=module', '-e', source], + { + timeout: 10000, + maxBuffer: 2000000, + env: { ...process.env, QA_EXIT_DIAGNOSTICS: '' }, + }, + ).catch((error) => error); + assert.equal(result.code ?? 0, fail ? 1 : 0); + assert.match(result.stdout, /OUT_END\nCLOSED\nSTRICT\n$/); + assert.match(result.stderr, /ERR_END/); + if (fail) assert.match(result.stderr, /Error: body failed/); + } +}); diff --git a/scripts/qa-strict-soak.mjs b/scripts/qa-strict-soak.mjs index 747896d8e..710430dbf 100644 --- a/scripts/qa-strict-soak.mjs +++ b/scripts/qa-strict-soak.mjs @@ -28,6 +28,7 @@ // node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50 // node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert +import { closeBrowser } from './qa/browser.mjs'; import assert from 'node:assert/strict'; import { createStrict, @@ -311,7 +312,7 @@ for (let run = 1; run <= runs; run++) { } } } finally { - await browser.close().catch(() => {}); + await closeBrowser(browser); } const warm = (to) => row.swaps @@ -433,8 +434,7 @@ if (strictMode) { report.checks = checks; for (const check of checks) console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check)); - await strict.finish({ + await strict.finishAndExit({ failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null, }); - if (checks.some((c) => !c.pass)) process.exitCode = 1; -} else await strict.finish(); +} else await strict.finishAndExit(); diff --git a/scripts/qa-voice-engine-switch.mjs b/scripts/qa-voice-engine-switch.mjs index c93f550d4..15dfd38dd 100644 --- a/scripts/qa-voice-engine-switch.mjs +++ b/scripts/qa-voice-engine-switch.mjs @@ -1,3 +1,4 @@ +import { closeBrowser, exitAfterFlush } from './qa/browser.mjs'; import { localPreview, waitForEngine } from './qa-strict-harness.mjs'; // An active voice session survives engine switches. With a fake microphone // and a fake Realtime transport (no model is reached: /api/openai, /api/realtime @@ -253,6 +254,10 @@ try { ); results.push('PASS stopping voice closes the transport'); console.log(results.join('\n')); +} catch (error) { + console.error(error); + process.exitCode = 1; } finally { - await browser.close(); + await closeBrowser(browser); } +await exitAfterFlush(); diff --git a/scripts/qa-wildlife-switch.mjs b/scripts/qa-wildlife-switch.mjs index f3e7fa1e3..c76143641 100644 --- a/scripts/qa-wildlife-switch.mjs +++ b/scripts/qa-wildlife-switch.mjs @@ -56,6 +56,8 @@ const fixture = wildlifeRoutes(base.origin); const strict = createStrict({ origin: base.origin, layers: [LAYER], + // Wildlife freshness is checked against fixtures timestamped in Node. + time: 'real', routes: fixture.routes, }); await strict.run({}, async (b) => { diff --git a/scripts/qa/acceptance.mjs b/scripts/qa/acceptance.mjs new file mode 100644 index 000000000..52c3aa1c0 --- /dev/null +++ b/scripts/qa/acceptance.mjs @@ -0,0 +1,84 @@ +export const STATUSES = ['pass', 'flaky', 'hung', 'timeout', 'fail']; + +export function classifyAttempt(run, manifest, { external = false } = {}) { + if (run.error?.code === 'ETIMEDOUT') + return manifest?.status === 'pass' ? 'hung' : 'timeout'; + if (run.error || run.signal || run.status !== 0) return 'fail'; + return manifest?.status === 'pass' || (external && !manifest) + ? 'pass' + : 'fail'; +} + +export function retryStatus(first, second) { + return first !== 'pass' && second === 'pass' ? 'flaky' : (second ?? first); +} + +export function statusCounts(scenarios) { + return Object.fromEntries( + STATUSES.map((status) => [ + status, + scenarios.filter((row) => row.status === status).length, + ]), + ); +} + +export function historyLine(report) { + return { + commit: report.commit, + dirty: report.dirty, + startedAt: report.startedAt, + finishedAt: report.finishedAt, + seconds: + (Date.parse(report.finishedAt) - Date.parse(report.startedAt)) / 1000, + counts: statusCounts(report.scenarios), + // Keep the denominator correct for --only runs too. + scenarios: report.scenarios.map((row) => row.label), + nonPass: report.scenarios + .filter((row) => row.status !== 'pass') + .map(({ label, status }) => ({ label, status })), + }; +} + +export function flakeReport(history) { + const rows = new Map(); + for (const run of history) { + const nonPass = new Set(run.nonPass.map((row) => row.label)); + for (const label of run.scenarios) { + const row = rows.get(label) ?? { label, runs: 0, passes: 0 }; + row.runs++; + if (!nonPass.has(label)) row.passes++; + rows.set(label, row); + } + } + const share = (row) => ({ + ...row, + nonPassShare: row.runs ? (row.runs - row.passes) / row.runs : 0, + }); + const scenarios = [...rows.values()] + .map(share) + .sort( + (a, b) => + b.nonPassShare - a.nonPassShare || a.label.localeCompare(b.label), + ); + const overall = share( + scenarios.reduce( + (total, row) => ({ + runs: total.runs + row.runs, + passes: total.passes + row.passes, + }), + { runs: 0, passes: 0 }, + ), + ); + return { scenarios, overall }; +} + +export function formatFlakeReport(history) { + const { scenarios, overall } = flakeReport(history); + return [ + 'scenario\truns\tpasses\tnon-pass', + ...[...scenarios, { label: 'overall', ...overall }].map( + (row) => + `${row.label}\t${row.runs}\t${row.passes}\t${(row.nonPassShare * 100).toFixed(1)}%`, + ), + ].join('\n'); +} diff --git a/scripts/qa/browser.mjs b/scripts/qa/browser.mjs new file mode 100644 index 000000000..f91cf9695 --- /dev/null +++ b/scripts/qa/browser.mjs @@ -0,0 +1,73 @@ +import { processTree, readProcesses } from './process.mjs'; + +/** Bound shutdown even when Chrome leaves descendants holding its pipes. */ +export async function closeBrowser(browser, { deadlineMs = 5000 } = {}) { + const start = performance.now(); + let child; + let closed = false; + let killed = false; + let timer; + try { + child = browser.process(); + } catch {} + try { + closed = await Promise.race([ + Promise.resolve() + .then(() => browser.close()) + .then( + () => true, + () => false, + ), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), deadlineMs); + }), + ]); + } catch {} + clearTimeout(timer); + if (process.env.QA_EXIT_DIAGNOSTICS !== '1') { + try { + if (Number.isInteger(child?.pid) && child.pid > 0) { + // The group can outlive its leader, so probe the group itself. + process.kill(-child.pid, 0); + process.kill(-child.pid, 'SIGKILL'); + killed = true; + } + } catch {} + for (const name of ['stdout', 'stderr']) { + try { + child?.[name]?.destroy(); + } catch {} + } + } + return { closed, killed, ms: performance.now() - start }; +} + +export async function exitAfterFlush(code = process.exitCode ?? 0) { + process.exitCode = code; + if (process.env.QA_EXIT_DIAGNOSTICS === '1') return; + await Promise.all( + [process.stdout, process.stderr].map( + (stream) => + new Promise((resolve) => { + if (stream.destroyed) return resolve(); + stream.write('', resolve); + }), + ), + ); + process.exit(code); +} + +export function diagnoseExit(pids) { + if (process.env.QA_EXIT_DIAGNOSTICS !== '1') return; + setTimeout(() => { + console.error('QA EXIT resources', process.getActiveResourcesInfo()); + const processes = readProcesses(); + console.error('QA EXIT Chrome: pid ppid pgid elapsed command'); + for (const pid of pids) { + const rows = processTree(pid, processes); + console.error(`Chrome ${pid}:`); + for (const row of rows) console.error(row.line); + if (!rows.length) console.error('(no remaining processes)'); + } + }, 5000).unref(); +} diff --git a/scripts/qa/navaraVisual.mjs b/scripts/qa/navaraVisual.mjs index 852a0d52a..4b9457008 100644 --- a/scripts/qa/navaraVisual.mjs +++ b/scripts/qa/navaraVisual.mjs @@ -7,6 +7,7 @@ import { assertKeylessPreview } from '../qa-strict-harness.mjs'; import fs from 'node:fs'; import path from 'node:path'; import sharp from 'sharp'; +import { DAYTIME, shiftTime } from './time.mjs'; /** A still view over Austin, no layers, OSM imagery. */ export const STILL_VIEW = @@ -18,21 +19,8 @@ export const STILL_VIEW = * the styles' signatures read) does not follow the time of day the * scenario happens to run at. Default: Austin early afternoon. */ -export function pinDaytime(page, utc = '2026-11-03T18:30:00Z') { - return page.evaluateOnNewDocument((utc) => { - const offset = Date.parse(utc) - Date.now(); - const RealDate = Date; - class ShiftedDate extends RealDate { - constructor(...args) { - if (args.length === 0) super(RealDate.now() + offset); - else super(...args); - } - static now() { - return RealDate.now() + offset; - } - } - globalThis.Date = ShiftedDate; - }, utc); +export function pinDaytime(page, utc = DAYTIME) { + return page.evaluateOnNewDocument(shiftTime, utc); } /** The keyless page on `engine` with the still view (and `extra` hash). */ diff --git a/scripts/qa/process.mjs b/scripts/qa/process.mjs new file mode 100644 index 000000000..7a3e0cfca --- /dev/null +++ b/scripts/qa/process.mjs @@ -0,0 +1,119 @@ +import { execFileSync, spawn } from 'node:child_process'; + +export function readProcesses() { + try { + return execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,etime=,command='], { + encoding: 'utf8', + timeout: 2000, + maxBuffer: 1 << 24, + }) + .trim() + .split('\n') + .flatMap((line) => { + const match = line + .trim() + .match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/); + return match + ? [ + { + pid: Number(match[1]), + ppid: Number(match[2]), + pgid: Number(match[3]), + elapsed: match[4], + command: match[5], + line: line.trim(), + }, + ] + : []; + }); + } catch { + return []; + } +} + +/** Include the group even when its leader has already exited. */ +export function processTree(pid, processes) { + const ids = new Set([pid]); + let changed; + do { + changed = false; + for (const row of processes) { + if (!ids.has(row.pid) && (ids.has(row.ppid) || row.pgid === pid)) { + ids.add(row.pid); + changed = true; + } + } + } while (changed); + return processes.filter((row) => ids.has(row.pid)); +} + +/** Sequential callers get captured output and a deadline for the whole tree. */ +export function runScenario(command, args, { timeout, env }) { + return new Promise((resolve) => { + const child = spawn(command, args, { + env, + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const groups = new Set(child.pid ? [child.pid] : []); + const stdout = []; + const stderr = []; + let bytes = 0; + let error = null; + let done = false; + const rememberGroups = () => { + const processes = readProcesses(); + for (const pid of [...groups]) { + const rows = processTree(pid, processes); + if (processes.length && !rows.length && pid !== child.pid) + groups.delete(pid); + for (const row of rows) groups.add(row.pgid); + } + }; + const finish = (status, signal) => { + if (done) return; + done = true; + clearTimeout(deadline); + clearInterval(watch); + resolve({ + status, + signal, + error, + stdout: Buffer.concat(stdout).toString(), + stderr: Buffer.concat(stderr).toString(), + }); + }; + const stop = (code, message) => { + if (done) return; + error = Object.assign(new Error(message), { code }); + rememberGroups(); + // Chrome detaches into another group; retain it before Node can exit. + for (const pid of [...groups].reverse()) { + try { + if (pid > 0) process.kill(-pid, 'SIGKILL'); + } catch {} + } + child.stdout.destroy(); + child.stderr.destroy(); + child.unref(); + finish(null, 'SIGKILL'); + }; + const capture = (chunks) => (chunk) => { + bytes += chunk.length; + if (bytes > 1 << 26) stop('ENOBUFS', 'Scenario output exceeded 64 MiB'); + else chunks.push(chunk); + }; + child.stdout.on('data', capture(stdout)); + child.stderr.on('data', capture(stderr)); + child.on('error', (failure) => { + error = failure; + }); + child.on('close', finish); + // Keep detached descendants identifiable if Chrome's leader exits first. + const watch = setInterval(rememberGroups, 1000); + const deadline = setTimeout( + () => stop('ETIMEDOUT', `Scenario timed out after ${timeout} ms`), + timeout, + ); + }); +} diff --git a/scripts/qa/time.mjs b/scripts/qa/time.mjs new file mode 100644 index 000000000..84b7c6086 --- /dev/null +++ b/scripts/qa/time.mjs @@ -0,0 +1,40 @@ +const DAY_MS = 86_400_000; + +/** + * Austin's early afternoon (18:30 UTC) nearest to `now`. The page clock then + * moves by at most 12 hours: the sun is up over the test cities whenever a + * scenario runs, and data stamped with the real time stays current. + */ +export function daytimeNear(now = Date.now()) { + const day = new Date(now); + const at = Date.UTC( + day.getUTCFullYear(), + day.getUTCMonth(), + day.getUTCDate(), + 18, + 30, + ); + const nearest = [at - DAY_MS, at, at + DAY_MS].reduce((best, next) => + Math.abs(next - now) < Math.abs(best - now) ? next : best, + ); + return new Date(nearest).toISOString(); +} + +export const DAYTIME = daytimeNear(); + +/** Keep elapsed time moving while giving the sun a repeatable position. */ +export function shiftTime(utc) { + if (utc === 'real') return; + const offset = Date.parse(utc) - Date.now(); + const RealDate = Date; + class ShiftedDate extends RealDate { + constructor(...args) { + if (args.length === 0) super(RealDate.now() + offset); + else super(...args); + } + static now() { + return RealDate.now() + offset; + } + } + globalThis.Date = ShiftedDate; +} diff --git a/scripts/report-acceptance-flake.mjs b/scripts/report-acceptance-flake.mjs new file mode 100644 index 000000000..6377f33cc --- /dev/null +++ b/scripts/report-acceptance-flake.mjs @@ -0,0 +1,15 @@ +import { readFileSync } from 'node:fs'; +import { formatFlakeReport } from './qa/acceptance.mjs'; + +try { + if (process.argv.length !== 3) + throw new Error('Usage: node scripts/report-acceptance-flake.mjs FILE'); + const history = readFileSync(process.argv[2], 'utf8') + .split('\n') + .filter((line) => line.trim()) + .map((line) => JSON.parse(line)); + console.log(formatFlakeReport(history)); +} catch (error) { + console.error(error.message); + process.exitCode = 1; +}