mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-11 13:52:19 +00:00
Stop the strict acceptance from waiting for itself
A scenario whose body ran past about 19 s idled until 45 s after its start, and twice one never exited and hit the runner's 300 s timeout after it had passed: something of Chrome outlived Browser.close and kept its stdout and stderr open. - scripts/qa/browser.mjs closes Chrome within 5 s, kills its process group and closes its pipes; the harness then finishes and exits with the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and prints the open handles and Chrome's process tree 5 s after finish. - Every strict page starts with its clock at the 18:30 UTC nearest to the real time, so Navara's sun is up over the test cities whenever the acceptance runs; a scenario can ask for its own instant or the real clock (Wildlife does, for its freshness fixtures). - The runner records a scenario that passed but did not exit as hung, kills the whole process group on a timeout, runs a failed scenario once more and reports one that passes then as flaky, and appends each run to --history=FILE; scripts/report-acceptance-flake.mjs reads it.
This commit is contained in:
1 parent
a913e5466e
commit
ed6ae54ef6
19 files changed
+1039
-104
No files matched your search
+10
-1
@@ -416,7 +416,16 @@ one application, and only the globe-bound parts move to the next engine.
|
||||
--serve` builds the commit keyless in a clean worktree and runs every layer
|
||||
and switch scenario under a deny-by-default network policy, with free map
|
||||
tiles from a disk cache and a trap that fails any read of Cesium before it
|
||||
has loaded. Every catalog layer must be asserted on both engines. Point
|
||||
has loaded. Every catalog layer must be asserted on both engines.
|
||||
Page clocks start at `2026-11-03T18:30:00Z` unless a scenario supplies
|
||||
`time` (an ISO instant or `'real'`); `QA_EXIT_DIAGNOSTICS=1` preserves
|
||||
shutdown resources and prints them five seconds after completion.
|
||||
A timed-out scenario with passing evidence is `hung`; non-passing scenarios
|
||||
retry once unless `--no-retry` is set, and a passing retry is reported as
|
||||
`flaky` without failing the run.
|
||||
`--history=FILE` appends run summaries, and
|
||||
`node scripts/report-acceptance-flake.mjs FILE` reports each scenario's
|
||||
share of non-pass runs, including flaky runs. Point
|
||||
`PUPPETEER_EXECUTABLE_PATH` at an installed Chrome if Puppeteer's pinned
|
||||
build is missing. Navara lights the globe by the real sun; every base map,
|
||||
street maps included, takes the night fill after dusk, so a scenario over
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { execFileSync, spawn, spawnSync } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
appendFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
@@ -24,17 +25,31 @@ import {
|
||||
localPreview,
|
||||
waitForEngine,
|
||||
} from './qa-strict-harness.mjs';
|
||||
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { runScenario } from './qa/process.mjs';
|
||||
import {
|
||||
classifyAttempt,
|
||||
historyLine,
|
||||
retryStatus,
|
||||
statusCounts,
|
||||
} from './qa/acceptance.mjs';
|
||||
|
||||
// The strict acceptance: every browser scenario against one keyless local
|
||||
// preview, under the strict network policy (scripts/qa-strict-harness.mjs),
|
||||
// writing <output>/acceptance.json, one log and one assertion manifest per
|
||||
// scenario.
|
||||
// scenario. A timed-out pass is hung; a retry that passes is flaky and does
|
||||
// not fail the run. QA_EXIT_DIAGNOSTICS=1 preserves Chrome resources and
|
||||
// reports them five seconds after strict completion instead of forcing exit.
|
||||
// Pages start at 2026-11-03T18:30:00Z unless createStrict's time overrides it.
|
||||
//
|
||||
// node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 <output>
|
||||
// [--serve[=<commit>]] build <commit> (default HEAD) keyless in a clean
|
||||
// worktree and serve it with every .env key blank
|
||||
// [--scratch=DIR] where --serve puts its checkout and build
|
||||
// [--only=a,b] run only these labels
|
||||
// [--no-retry] disable the one retry of a non-passing scenario
|
||||
// [--history=FILE] append a JSONL run summary; inspect it with
|
||||
// node scripts/report-acceptance-flake.mjs FILE
|
||||
// [--quick] the soak with 2 runs of 10 switches and 100
|
||||
// switches for the long run
|
||||
//
|
||||
@@ -42,6 +57,7 @@ import {
|
||||
// the live page and cross-checked with src/app/layers/directory.js) has no
|
||||
// assertion on either engine, or when any network policy violation shows.
|
||||
|
||||
const startedAt = new Date().toISOString();
|
||||
const base = localPreview(process.argv[2]);
|
||||
const output = process.argv[3]?.startsWith('--')
|
||||
? mkdtempSync(join(tmpdir(), 'gev-acceptance-'))
|
||||
@@ -67,7 +83,7 @@ let chrome;
|
||||
try {
|
||||
const browser = await puppeteer.launch({ headless: true });
|
||||
chrome = await browser.version();
|
||||
await browser.close();
|
||||
await closeBrowser(browser);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
'Puppeteer cannot start Chrome: ' +
|
||||
@@ -429,7 +445,7 @@ const liveCatalog = {};
|
||||
await page.close();
|
||||
}
|
||||
} finally {
|
||||
await browser.close();
|
||||
await closeBrowser(browser);
|
||||
}
|
||||
liveCatalog.violations = strict.ledger.violations;
|
||||
}
|
||||
@@ -475,10 +491,11 @@ const report = {
|
||||
live: liveCatalog,
|
||||
scenarios: scenarioIds,
|
||||
},
|
||||
startedAt: new Date().toISOString(),
|
||||
startedAt,
|
||||
finishedAt: null,
|
||||
passed: 0,
|
||||
failed: 0,
|
||||
counts: statusCounts([]),
|
||||
networkViolations: 0,
|
||||
coverage: null,
|
||||
uncovered: null,
|
||||
@@ -495,7 +512,6 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
|
||||
if (only && !only.includes(label)) continue;
|
||||
const script = 'scripts/qa-' + name + '.mjs';
|
||||
const evidence = join(output, 'evidence', label + '.json');
|
||||
rmSync(evidence, { force: true });
|
||||
const args = [base.origin, ...extra];
|
||||
if (external) args.splice(0, 1, base.origin + '/');
|
||||
else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`);
|
||||
@@ -503,27 +519,69 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
|
||||
// With --serve the scenarios run from the same clean checkout the app was
|
||||
// built from, so other work in this tree cannot change them.
|
||||
const scripts = served?.worktree ?? root;
|
||||
const run = spawnSync(process.execPath, [join(scripts, script), ...args], {
|
||||
encoding: 'utf8',
|
||||
timeout: seconds * 1000,
|
||||
maxBuffer: 1 << 26,
|
||||
env: scrubbed(),
|
||||
});
|
||||
const status =
|
||||
run.status === 0
|
||||
? 'pass'
|
||||
: run.error?.code === 'ETIMEDOUT'
|
||||
? 'timeout'
|
||||
: 'fail';
|
||||
const log = machinePaths(
|
||||
(run.stdout || '') + (run.stderr || '') + (run.error?.message || ''),
|
||||
);
|
||||
writeFileSync(join(output, label + '.log'), log);
|
||||
const manifest = existsSync(evidence)
|
||||
? JSON.parse(machinePaths(readFileSync(evidence, 'utf8')))
|
||||
const attempts = [];
|
||||
for (let attempt = 1; attempt <= (flag('no-retry') ? 1 : 2); attempt++) {
|
||||
rmSync(evidence, { force: true });
|
||||
const attemptStart = Date.now();
|
||||
const run = await runScenario(
|
||||
process.execPath,
|
||||
[join(scripts, script), ...args],
|
||||
{
|
||||
timeout: seconds * 1000,
|
||||
env: scrubbed(),
|
||||
},
|
||||
);
|
||||
const log = machinePaths(
|
||||
(run.stdout || '') + (run.stderr || '') + (run.error?.message || ''),
|
||||
);
|
||||
let manifest = null;
|
||||
let evidenceError = null;
|
||||
if (existsSync(evidence)) {
|
||||
try {
|
||||
manifest = JSON.parse(machinePaths(readFileSync(evidence, 'utf8')));
|
||||
} catch (error) {
|
||||
evidenceError = 'Invalid evidence: ' + error.message;
|
||||
}
|
||||
}
|
||||
const status =
|
||||
evidenceError && !run.error
|
||||
? 'fail'
|
||||
: classifyAttempt(run, manifest, { external });
|
||||
const logFile = `${label}.attempt-${attempt}.log`;
|
||||
const evidenceFile = manifest
|
||||
? `evidence/${label}.attempt-${attempt}.json`
|
||||
: null;
|
||||
writeFileSync(join(output, logFile), log);
|
||||
if (manifest)
|
||||
writeFileSync(
|
||||
join(output, evidenceFile),
|
||||
JSON.stringify(manifest, null, 2) + '\n',
|
||||
);
|
||||
attempts.push({
|
||||
status,
|
||||
exitCode: run.status,
|
||||
seconds: Math.round((Date.now() - attemptStart) / 100) / 10,
|
||||
log: logFile,
|
||||
logSha256: createHash('sha256').update(log).digest('hex'),
|
||||
evidence: evidenceFile,
|
||||
error: evidenceError ?? manifest?.error ?? run.error?.message ?? null,
|
||||
});
|
||||
writeFileSync(join(output, label + '.log'), log);
|
||||
if (manifest)
|
||||
writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n');
|
||||
console.log(
|
||||
status.toUpperCase(),
|
||||
label,
|
||||
attempts.at(-1).seconds,
|
||||
`attempt ${attempt}`,
|
||||
);
|
||||
if (status === 'pass' || attempt === 2 || flag('no-retry')) break;
|
||||
}
|
||||
const last = attempts.at(-1);
|
||||
const status = retryStatus(attempts[0].status, attempts[1]?.status);
|
||||
const manifest = last.evidence
|
||||
? JSON.parse(readFileSync(join(output, last.evidence), 'utf8'))
|
||||
: null;
|
||||
if (manifest)
|
||||
writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n');
|
||||
scenarios.push({
|
||||
label,
|
||||
script,
|
||||
@@ -533,10 +591,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
|
||||
layers,
|
||||
external: Boolean(external),
|
||||
status,
|
||||
exitCode: run.status,
|
||||
exitCode: last.exitCode,
|
||||
seconds: Math.round((Date.now() - start) / 100) / 10,
|
||||
log: label + '.log',
|
||||
logSha256: createHash('sha256').update(log).digest('hex'),
|
||||
logSha256: last.logSha256,
|
||||
attempts,
|
||||
evidence: manifest ? 'evidence/' + label + '.json' : null,
|
||||
assertions: manifest?.assertions.length ?? 0,
|
||||
coverage: manifest?.coverage ?? null,
|
||||
@@ -554,10 +613,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
|
||||
}
|
||||
: null,
|
||||
contracts: manifest?.contracts ?? null,
|
||||
error: manifest?.error ?? null,
|
||||
error: last.error,
|
||||
});
|
||||
if (manifest?.soak) scenarios.at(-1).soak = manifest.soak;
|
||||
report.passed = scenarios.filter((row) => row.status === 'pass').length;
|
||||
report.counts = statusCounts(scenarios);
|
||||
report.passed = report.counts.pass + report.counts.flaky;
|
||||
report.failed = scenarios.length - report.passed;
|
||||
report.networkViolations =
|
||||
scenarios.reduce(
|
||||
@@ -599,9 +659,18 @@ report.catalogMatches =
|
||||
report.finishedAt = new Date().toISOString();
|
||||
report.dirtyFilesAtEnd = dirty();
|
||||
save();
|
||||
if (flag('history'))
|
||||
appendFileSync(
|
||||
resolve(String(flag('history'))),
|
||||
JSON.stringify(historyLine(report)) + '\n',
|
||||
);
|
||||
console.log(
|
||||
`${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` +
|
||||
`${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` +
|
||||
Object.entries(report.counts)
|
||||
.map(([status, count]) => `${count} ${status}`)
|
||||
.join(', ') +
|
||||
', ' +
|
||||
`paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`,
|
||||
);
|
||||
if (
|
||||
@@ -615,3 +684,4 @@ if (
|
||||
// The preview child would keep this process alive: stop it and drop the
|
||||
// checkout now.
|
||||
cleanup();
|
||||
await exitAfterFlush();
|
||||
@@ -1,3 +1,4 @@
|
||||
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { localPreview, waitForEngine } from './qa-strict-harness.mjs';
|
||||
// The application is created once per page: switching engines detaches and
|
||||
// attaches only the engine-bound parts. This gate runs three Cesium→Navara→
|
||||
@@ -413,6 +414,10 @@ try {
|
||||
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (!reportOnly) console.log('PASS application created once; session kept');
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await browser.close();
|
||||
await closeBrowser(browser);
|
||||
}
|
||||
await exitAfterFlush();
|
||||
@@ -6,6 +6,7 @@
|
||||
//
|
||||
// node scripts/qa-imagery-contract.mjs http://127.0.0.1:<PORT>
|
||||
// [--layers=recent-imagery] [--evidence=FILE]
|
||||
import { closeBrowser } from './qa/browser.mjs';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
@@ -470,9 +471,8 @@ try {
|
||||
report.errors.push(e.stack);
|
||||
check('driver completes', false);
|
||||
} finally {
|
||||
await browser.close();
|
||||
await closeBrowser(browser);
|
||||
await fs.writeFile(output, JSON.stringify(report, null, 2));
|
||||
}
|
||||
const failed = report.checks.some((c) => !c.pass);
|
||||
await strict.finish({ failed: failed ? 'imagery checks failed' : null });
|
||||
if (failed) process.exitCode = 1;
|
||||
await strict.finishAndExit({ failed: failed ? 'imagery checks failed' : null });
|
||||
@@ -21,7 +21,6 @@ import {
|
||||
diff,
|
||||
freeze,
|
||||
pickStyle,
|
||||
pinDaytime,
|
||||
saveCapture,
|
||||
stats,
|
||||
visualPageUrl,
|
||||
@@ -59,7 +58,6 @@ await strict.run({}, async (browser) => {
|
||||
page.on('pageerror', (error) => errors.push(error.message));
|
||||
// The same sun on every run: the frame's brightness decides what blooms
|
||||
// and what each style's signature reads.
|
||||
await pinDaytime(page);
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
|
||||
});
|
||||
|
||||
@@ -50,6 +50,7 @@ const asked = [];
|
||||
const strict = createStrict({
|
||||
origin: base.origin,
|
||||
layers: [],
|
||||
time: MIDDAY_UTC,
|
||||
routes: [
|
||||
{
|
||||
pattern: `${base.origin}/api/weather-effects*`,
|
||||
@@ -105,21 +106,9 @@ await strict.run({}, async (browser) => {
|
||||
const page = await browser.newPage();
|
||||
const errors = [];
|
||||
page.on('pageerror', (error) => errors.push(error.message));
|
||||
await page.evaluateOnNewDocument((utc) => {
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
|
||||
const offset = Date.parse(utc) - Date.now();
|
||||
const RealDate = Date;
|
||||
class ShiftedDate extends RealDate {
|
||||
constructor(...args) {
|
||||
if (args.length === 0) super(RealDate.now() + offset);
|
||||
else super(...args);
|
||||
}
|
||||
static now() {
|
||||
return RealDate.now() + offset;
|
||||
}
|
||||
}
|
||||
globalThis.Date = ShiftedDate;
|
||||
}, MIDDAY_UTC);
|
||||
});
|
||||
await page.setViewport({ width: 1280, height: 800 });
|
||||
await page.goto(visualPageUrl(base.origin, 'navara', '', LOW_VIEW), {
|
||||
waitUntil: 'domcontentloaded',
|
||||
|
||||
@@ -24,7 +24,6 @@ import {
|
||||
freeze,
|
||||
histogramShift,
|
||||
pickStyle,
|
||||
pinDaytime,
|
||||
saveCapture,
|
||||
stats,
|
||||
visualPageUrl,
|
||||
@@ -51,7 +50,6 @@ await strict.run({}, async (browser) => {
|
||||
page.on('pageerror', (error) => errors.push(error.message));
|
||||
// The same sun on every run: the frame's brightness decides what blooms
|
||||
// and what each style's signature reads.
|
||||
await pinDaytime(page);
|
||||
await page.evaluateOnNewDocument(() => {
|
||||
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import {
|
||||
classifyAttempt,
|
||||
flakeReport,
|
||||
formatFlakeReport,
|
||||
historyLine,
|
||||
retryStatus,
|
||||
} from './qa/acceptance.mjs';
|
||||
import { processTree, readProcesses, runScenario } from './qa/process.mjs';
|
||||
|
||||
test('a timeout after passing evidence is hung, even after a zero exit', () => {
|
||||
const timeout = { status: 0, error: { code: 'ETIMEDOUT' } };
|
||||
assert.equal(classifyAttempt(timeout, { status: 'pass' }), 'hung');
|
||||
assert.equal(classifyAttempt(timeout, { status: 'fail' }), 'timeout');
|
||||
assert.equal(classifyAttempt(timeout, null), 'timeout');
|
||||
});
|
||||
|
||||
test('a pass requires a successful exit and passing evidence', () => {
|
||||
const pass = { status: 'pass' };
|
||||
assert.equal(classifyAttempt({ status: 0 }, pass), 'pass');
|
||||
assert.equal(classifyAttempt({ status: 1 }, pass), 'fail');
|
||||
assert.equal(classifyAttempt({ status: 0, signal: 'SIGTERM' }, pass), 'fail');
|
||||
assert.equal(
|
||||
classifyAttempt({ status: 0, error: { code: 'ENOBUFS' } }, pass),
|
||||
'fail',
|
||||
);
|
||||
assert.equal(classifyAttempt({ status: 0 }, { status: 'fail' }), 'fail');
|
||||
assert.equal(classifyAttempt({ status: 0 }, null), 'fail');
|
||||
assert.equal(
|
||||
classifyAttempt({ status: 0 }, null, { external: true }),
|
||||
'pass',
|
||||
);
|
||||
});
|
||||
|
||||
test('a passing retry is flaky; no retry preserves the first status', () => {
|
||||
for (const status of ['fail', 'timeout', 'hung']) {
|
||||
assert.equal(retryStatus(status, 'pass'), 'flaky');
|
||||
assert.equal(retryStatus(status), status);
|
||||
assert.equal(retryStatus(status, 'fail'), 'fail');
|
||||
}
|
||||
assert.equal(retryStatus('pass'), 'pass');
|
||||
});
|
||||
|
||||
const report = (scenarios) => ({
|
||||
commit: 'abc123',
|
||||
dirty: true,
|
||||
startedAt: '2026-11-10T12:00:00Z',
|
||||
finishedAt: '2026-11-10T12:00:12.500Z',
|
||||
scenarios: Object.entries(scenarios).map(([label, status]) => ({
|
||||
label,
|
||||
status,
|
||||
})),
|
||||
});
|
||||
|
||||
test('history records provenance, duration, every status, and selected scenarios', () => {
|
||||
assert.deepEqual(
|
||||
historyLine(
|
||||
report({ a: 'pass', b: 'flaky', c: 'hung', d: 'timeout', e: 'fail' }),
|
||||
),
|
||||
{
|
||||
commit: 'abc123',
|
||||
dirty: true,
|
||||
startedAt: '2026-11-10T12:00:00Z',
|
||||
finishedAt: '2026-11-10T12:00:12.500Z',
|
||||
seconds: 12.5,
|
||||
counts: { pass: 1, flaky: 1, hung: 1, timeout: 1, fail: 1 },
|
||||
scenarios: ['a', 'b', 'c', 'd', 'e'],
|
||||
nonPass: [
|
||||
{ label: 'b', status: 'flaky' },
|
||||
{ label: 'c', status: 'hung' },
|
||||
{ label: 'd', status: 'timeout' },
|
||||
{ label: 'e', status: 'fail' },
|
||||
],
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('flake shares include flaky runs and exclude scenarios absent from partial runs', () => {
|
||||
const history = [
|
||||
historyLine(report({ a: 'pass', b: 'flaky', c: 'fail' })),
|
||||
historyLine(report({ a: 'pass', b: 'pass' })),
|
||||
historyLine(report({ a: 'hung' })),
|
||||
];
|
||||
assert.deepEqual(flakeReport(history), {
|
||||
scenarios: [
|
||||
{ label: 'c', runs: 1, passes: 0, nonPassShare: 1 },
|
||||
{ label: 'b', runs: 2, passes: 1, nonPassShare: 0.5 },
|
||||
{ label: 'a', runs: 3, passes: 2, nonPassShare: 1 / 3 },
|
||||
],
|
||||
overall: { runs: 6, passes: 3, nonPassShare: 0.5 },
|
||||
});
|
||||
assert.match(formatFlakeReport(history), /overall\t6\t3\t50.0%$/);
|
||||
assert.deepEqual(flakeReport([]), {
|
||||
scenarios: [],
|
||||
overall: { runs: 0, passes: 0, nonPassShare: 0 },
|
||||
});
|
||||
});
|
||||
|
||||
test('the report CLI reads JSON lines and rejects malformed input', async (t) => {
|
||||
const dir = mkdtempSync('tmp_acceptance_history_');
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const file = `${dir}/history.jsonl`;
|
||||
writeFileSync(
|
||||
file,
|
||||
JSON.stringify(historyLine(report({ a: 'flaky' }))) + '\n',
|
||||
);
|
||||
const run = promisify(execFile);
|
||||
const script = new URL('./report-acceptance-flake.mjs', import.meta.url);
|
||||
const result = await run(process.execPath, [script.pathname, file]);
|
||||
assert.match(result.stdout, /a\t1\t0\t100.0%/);
|
||||
writeFileSync(file, '{broken}\n');
|
||||
await assert.rejects(run(process.execPath, [script.pathname, file]), {
|
||||
code: 1,
|
||||
});
|
||||
});
|
||||
|
||||
test('process trees include detached descendants and orphaned group members only', () => {
|
||||
const rows = [
|
||||
{ pid: 10, ppid: 1, pgid: 10 },
|
||||
{ pid: 20, ppid: 10, pgid: 20 },
|
||||
{ pid: 21, ppid: 20, pgid: 20 },
|
||||
{ pid: 22, ppid: 1, pgid: 20 },
|
||||
{ pid: 99, ppid: 1, pgid: 99 },
|
||||
];
|
||||
assert.deepEqual(
|
||||
processTree(10, rows).map((row) => row.pid),
|
||||
[10, 20, 21],
|
||||
);
|
||||
assert.deepEqual(
|
||||
processTree(20, rows).map((row) => row.pid),
|
||||
[20, 21, 22],
|
||||
);
|
||||
});
|
||||
|
||||
test('scenario spawning captures both streams and reports spawn failures', async () => {
|
||||
const run = await runScenario(
|
||||
process.execPath,
|
||||
['-e', "console.log('out'); console.error('err'); process.exitCode = 3"],
|
||||
{ timeout: 5000 },
|
||||
);
|
||||
assert.equal(run.status, 3);
|
||||
assert.equal(run.stdout, 'out\n');
|
||||
assert.equal(run.stderr, 'err\n');
|
||||
assert.equal(run.error, null);
|
||||
const missing = await runScenario('./tmp_acceptance_missing_executable', [], {
|
||||
timeout: 5000,
|
||||
});
|
||||
assert.equal(missing.error.code, 'ENOENT');
|
||||
});
|
||||
|
||||
test('timeout kills the scenario and a detached group after its leader exits', async (t) => {
|
||||
// These Node children stand in for Chrome and a renderer; no browser is run.
|
||||
const renderer = 'setInterval(() => {}, 1000)';
|
||||
const browser = `
|
||||
const { spawn } = require('node:child_process');
|
||||
const child = spawn(process.execPath, ['-e', ${JSON.stringify(renderer)}], { stdio: 'inherit' });
|
||||
console.log('renderer', child.pid);
|
||||
setTimeout(() => process.exit(0), 1500);
|
||||
`;
|
||||
const scenario = `
|
||||
const { spawn } = require('node:child_process');
|
||||
const child = spawn(process.execPath, ['-e', ${JSON.stringify(browser)}], { detached: true, stdio: 'inherit' });
|
||||
console.log('browser', child.pid);
|
||||
console.log('scenario', process.pid);
|
||||
setInterval(() => {}, 1000);
|
||||
`;
|
||||
const run = await runScenario(process.execPath, ['-e', scenario], {
|
||||
timeout: 2600,
|
||||
});
|
||||
const pids = [
|
||||
...run.stdout.matchAll(/(?:browser|renderer|scenario) (\d+)/g),
|
||||
].map((match) => Number(match[1]));
|
||||
t.after(() => {
|
||||
for (const pid of pids) {
|
||||
try {
|
||||
process.kill(-pid, 'SIGKILL');
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
assert.equal(run.error.code, 'ETIMEDOUT');
|
||||
assert.equal(pids.length, 3);
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
const survivors = readProcesses().filter(
|
||||
(row) => pids.includes(row.pid) && !/defunct/.test(row.command),
|
||||
);
|
||||
assert.deepEqual(survivors, []);
|
||||
});
|
||||
@@ -27,6 +27,8 @@ import {
|
||||
STRICT_TILE_HOSTS,
|
||||
} from './qa-map-policy.mjs';
|
||||
import { createMissLoader, tileCacheFile } from './qa/tileCache.mjs';
|
||||
import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { DAYTIME, shiftTime } from './qa/time.mjs';
|
||||
|
||||
export const ENGINES = Object.freeze(['cesium', 'navara']);
|
||||
/** What the Globe Engine picker offers: the engines and Navara HD. */
|
||||
@@ -92,7 +94,14 @@ const live = (response) => Boolean(response.remoteIPAddress);
|
||||
*/
|
||||
export async function installStrictNetwork(
|
||||
browser,
|
||||
{ origin, fixtures = [], routes = [], tileHosts = STRICT_TILE_HOSTS, ledger },
|
||||
{
|
||||
origin,
|
||||
fixtures = [],
|
||||
routes = [],
|
||||
tileHosts = STRICT_TILE_HOSTS,
|
||||
ledger,
|
||||
time = DAYTIME,
|
||||
},
|
||||
) {
|
||||
const preview = new URL(origin);
|
||||
fixtures = [...fixtures, ...routes.map((route) => route.pattern)];
|
||||
@@ -298,7 +307,7 @@ export async function installStrictNetwork(
|
||||
// requests, and a live answer would still show up as a violation.
|
||||
if (type === 'page' || type === 'iframe')
|
||||
steps.push(intercept(session, type));
|
||||
if (type === 'page') steps.push(trapCesiumGlobal(session, ledger));
|
||||
if (type === 'page') steps.push(trapCesiumGlobal(session, ledger, time));
|
||||
await Promise.all(steps);
|
||||
};
|
||||
const attach = async (sessionId, info, waiting) => {
|
||||
@@ -344,7 +353,7 @@ export async function installStrictNetwork(
|
||||
* has assigned it are logged with their stack in `ledger.cesiumBeforeLoad`
|
||||
* and answer `undefined`, as production would.
|
||||
*/
|
||||
async function trapCesiumGlobal(session, ledger) {
|
||||
async function trapCesiumGlobal(session, ledger, time) {
|
||||
const binding = '__gevStrictCesiumRead';
|
||||
session.on('Runtime.bindingCalled', (event) => {
|
||||
if (event.name !== binding) return;
|
||||
@@ -358,7 +367,8 @@ async function trapCesiumGlobal(session, ledger) {
|
||||
await session.send('Runtime.addBinding', { name: binding });
|
||||
await session.send('Page.enable');
|
||||
await session.send('Page.addScriptToEvaluateOnNewDocument', {
|
||||
source: `(() => {
|
||||
source: `(${shiftTime})(${JSON.stringify(time)});
|
||||
(() => {
|
||||
if (window.top !== window) return;
|
||||
const seen = new Set();
|
||||
let sdk;
|
||||
@@ -630,6 +640,7 @@ export async function openPage(browser, { shaderErrors = false } = {}) {
|
||||
* @param {readonly string[]} [options.tileHosts] Hosts the disk cache
|
||||
* answers (default STRICT_TILE_HOSTS).
|
||||
* @param {string} [options.name] Scenario name for the manifest.
|
||||
* @param {string} [options.time] ISO instant for the page clock, or 'real'.
|
||||
*/
|
||||
export function createStrict({
|
||||
origin,
|
||||
@@ -638,7 +649,14 @@ export function createStrict({
|
||||
routes = [],
|
||||
tileHosts = STRICT_TILE_HOSTS,
|
||||
name = path.basename(process.argv[1] ?? 'scenario', '.mjs'),
|
||||
time = DAYTIME,
|
||||
}) {
|
||||
assert.ok(
|
||||
typeof time === 'string' &&
|
||||
(time === 'real' || Number.isFinite(Date.parse(time))),
|
||||
'valid page time',
|
||||
);
|
||||
const browserPids = [];
|
||||
const args = strictArguments();
|
||||
if (args.layers)
|
||||
assert.deepEqual(
|
||||
@@ -685,28 +703,45 @@ export function createStrict({
|
||||
...options,
|
||||
args: [...CHROME_ARGS, ...args],
|
||||
});
|
||||
await installStrictNetwork(browser, {
|
||||
origin,
|
||||
fixtures,
|
||||
routes,
|
||||
tileHosts,
|
||||
ledger,
|
||||
});
|
||||
const pid = browser.process()?.pid;
|
||||
if (pid) browserPids.push(pid);
|
||||
try {
|
||||
await installStrictNetwork(browser, {
|
||||
origin,
|
||||
fixtures,
|
||||
routes,
|
||||
tileHosts,
|
||||
ledger,
|
||||
time,
|
||||
});
|
||||
} catch (error) {
|
||||
await closeBrowser(browser);
|
||||
throw error;
|
||||
}
|
||||
return browser;
|
||||
},
|
||||
/** Launch with `options`, run `body(browser)`, close, then finish(). */
|
||||
async run(options, body) {
|
||||
const browser = await strict.launch(options);
|
||||
let browser;
|
||||
let failed = null;
|
||||
try {
|
||||
browser = await strict.launch(options);
|
||||
await body(browser);
|
||||
} catch (error) {
|
||||
failed = error;
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
await closeBrowser(browser);
|
||||
}
|
||||
await strict.finish({ failed });
|
||||
if (failed) throw failed;
|
||||
await strict.finishAndExit({ failed });
|
||||
},
|
||||
async finishAndExit({ failed = null } = {}) {
|
||||
try {
|
||||
await strict.finish({ failed });
|
||||
} catch (error) {
|
||||
failed = error;
|
||||
}
|
||||
if (failed) console.error(failed);
|
||||
await exitAfterFlush(failed ? 1 : process.exitCode ?? 0);
|
||||
},
|
||||
/** Record an assertion that already passed. */
|
||||
record,
|
||||
@@ -788,6 +823,30 @@ export function createStrict({
|
||||
fixtures,
|
||||
tileHosts: [...tileHosts],
|
||||
};
|
||||
let validationError = null;
|
||||
if (!failed) {
|
||||
try {
|
||||
assert.equal(
|
||||
ledger.violations.length,
|
||||
0,
|
||||
'network policy violations: ' +
|
||||
JSON.stringify(ledger.violations.slice(0, 5)),
|
||||
);
|
||||
assert.deepEqual(
|
||||
ledger.cesiumBeforeLoad,
|
||||
[],
|
||||
'the page read the Cesium global before Cesium.js loaded',
|
||||
);
|
||||
assert.deepEqual(missing, [], 'layers without an assertion');
|
||||
assert.ok(
|
||||
picks.some((pick) => pick.via === 'pointer+keyboard'),
|
||||
'switch at least once with real input on the picker',
|
||||
);
|
||||
} catch (error) {
|
||||
validationError = error;
|
||||
failed = error;
|
||||
}
|
||||
}
|
||||
const manifest = {
|
||||
scenario: name,
|
||||
argv: process.argv.slice(2),
|
||||
@@ -825,23 +884,8 @@ export function createStrict({
|
||||
console.log('NETWORK VIOLATION', JSON.stringify(v));
|
||||
for (const read of ledger.cesiumBeforeLoad.slice(0, 10))
|
||||
console.log('CESIUM BEFORE LOAD', JSON.stringify(read));
|
||||
if (failed) return;
|
||||
assert.equal(
|
||||
ledger.violations.length,
|
||||
0,
|
||||
'network policy violations: ' +
|
||||
JSON.stringify(ledger.violations.slice(0, 5)),
|
||||
);
|
||||
assert.deepEqual(
|
||||
ledger.cesiumBeforeLoad,
|
||||
[],
|
||||
'the page read the Cesium global before Cesium.js loaded',
|
||||
);
|
||||
assert.deepEqual(missing, [], 'layers without an assertion');
|
||||
assert.ok(
|
||||
picks.some((pick) => pick.via === 'pointer+keyboard'),
|
||||
'switch at least once with real input on the picker',
|
||||
);
|
||||
diagnoseExit(browserPids);
|
||||
if (validationError) throw validationError;
|
||||
},
|
||||
};
|
||||
return strict;
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import vm from 'node:vm';
|
||||
import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { DAYTIME, daytimeNear, shiftTime } from './qa/time.mjs';
|
||||
import {
|
||||
createStrict,
|
||||
installStrictNetwork,
|
||||
networkLedger,
|
||||
} from './qa-strict-harness.mjs';
|
||||
|
||||
function fakeBrowser(t, close = async () => {}) {
|
||||
t.mock.method(process, 'kill', () => true);
|
||||
const stdout = { destroy: t.mock.fn() };
|
||||
const stderr = { destroy: t.mock.fn() };
|
||||
return {
|
||||
close,
|
||||
process: () => ({ pid: 12345, stdout, stderr }),
|
||||
stdout,
|
||||
stderr,
|
||||
};
|
||||
}
|
||||
|
||||
test('closeBrowser kills the remaining group and drops both pipes after closing', async (t) => {
|
||||
const browser = fakeBrowser(t);
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.killed, true);
|
||||
assert.ok(result.ms >= 0);
|
||||
assert.deepEqual(
|
||||
process.kill.mock.calls.map((call) => call.arguments),
|
||||
[
|
||||
[-12345, 0],
|
||||
[-12345, 'SIGKILL'],
|
||||
],
|
||||
);
|
||||
assert.equal(browser.stdout.destroy.mock.callCount(), 1);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
|
||||
});
|
||||
|
||||
test('closeBrowser bounds a stuck close and tolerates a late rejection', async (t) => {
|
||||
let reject;
|
||||
const browser = fakeBrowser(
|
||||
t,
|
||||
() =>
|
||||
new Promise((resolve, failure) => {
|
||||
reject = failure;
|
||||
}),
|
||||
);
|
||||
const result = await closeBrowser(browser, { deadlineMs: 10 });
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.killed, true);
|
||||
assert.ok(result.ms < 1000);
|
||||
reject(new Error('closed later'));
|
||||
});
|
||||
|
||||
test('missing processes, dead groups and broken streams never prevent cleanup', async (t) => {
|
||||
const browser = fakeBrowser(t, () => {
|
||||
throw new Error('close failed');
|
||||
});
|
||||
t.mock.method(process, 'kill', () => {
|
||||
throw new Error('ESRCH');
|
||||
});
|
||||
browser.stdout.destroy = () => {
|
||||
throw new Error('pipe failed');
|
||||
};
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.killed, false);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
|
||||
for (const process of [
|
||||
() => null,
|
||||
() => ({}),
|
||||
() => {
|
||||
throw new Error('no process');
|
||||
},
|
||||
]) {
|
||||
const missing = await closeBrowser({ process, close: async () => {} });
|
||||
assert.equal(missing.closed, true);
|
||||
assert.equal(missing.killed, false);
|
||||
}
|
||||
assert.equal((await closeBrowser(undefined)).closed, false);
|
||||
});
|
||||
|
||||
test('diagnostic shutdown preserves the process group and pipes', async (t) => {
|
||||
const old = process.env.QA_EXIT_DIAGNOSTICS;
|
||||
process.env.QA_EXIT_DIAGNOSTICS = '1';
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
|
||||
else process.env.QA_EXIT_DIAGNOSTICS = old;
|
||||
});
|
||||
const browser = fakeBrowser(t);
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.killed, false);
|
||||
assert.equal(process.kill.mock.callCount(), 0);
|
||||
assert.equal(browser.stdout.destroy.mock.callCount(), 0);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 0);
|
||||
let callback;
|
||||
const unref = t.mock.fn();
|
||||
t.mock.method(globalThis, 'setTimeout', (fn, delay) => {
|
||||
callback = fn;
|
||||
assert.equal(delay, 5000);
|
||||
return { unref };
|
||||
});
|
||||
t.mock.method(console, 'error', () => {});
|
||||
diagnoseExit([]);
|
||||
assert.equal(unref.mock.callCount(), 1);
|
||||
assert.equal(console.error.mock.callCount(), 0);
|
||||
callback();
|
||||
assert.equal(console.error.mock.calls[0].arguments[0], 'QA EXIT resources');
|
||||
assert.match(
|
||||
console.error.mock.calls[1].arguments[0],
|
||||
/pid ppid pgid elapsed command/,
|
||||
);
|
||||
});
|
||||
|
||||
test('the page clock moves to the nearest 18:30 UTC, at most 12 hours away', () => {
|
||||
const at = (iso) => daytimeNear(Date.parse(iso));
|
||||
assert.equal(at('2026-11-10T10:00:00Z'), '2026-11-10T18:30:00.000Z');
|
||||
assert.equal(at('2026-11-10T03:00:00Z'), '2026-11-09T18:30:00.000Z');
|
||||
assert.equal(at('2026-12-31T23:00:00Z'), '2026-12-31T18:30:00.000Z');
|
||||
assert.equal(at('2027-01-01T07:00:00Z'), '2027-01-01T18:30:00.000Z');
|
||||
assert.ok(Math.abs(Date.parse(DAYTIME) - Date.now()) <= 12 * 3_600_000);
|
||||
});
|
||||
|
||||
test('the shifted clock ticks, preserves explicit dates, and supports the real clock', () => {
|
||||
const context = vm.createContext({});
|
||||
vm.runInContext(
|
||||
`
|
||||
globalThis.tick = 1000;
|
||||
const NativeDate = Date;
|
||||
globalThis.Date = class extends NativeDate {
|
||||
static now() { return tick; }
|
||||
};
|
||||
globalThis.original = Date;
|
||||
(${shiftTime})(${JSON.stringify(DAYTIME)});
|
||||
`,
|
||||
context,
|
||||
);
|
||||
assert.equal(vm.runInContext('Date.now()', context), Date.parse(DAYTIME));
|
||||
assert.equal(vm.runInContext('new Date().toISOString()', context), DAYTIME);
|
||||
assert.equal(
|
||||
vm.runInContext('tick += 250; Date.now()', context),
|
||||
Date.parse(DAYTIME) + 250,
|
||||
);
|
||||
assert.equal(vm.runInContext('new Date(0).getTime()', context), 0);
|
||||
assert.equal(
|
||||
vm.runInContext('Date.UTC(2026, 0, 1)', context),
|
||||
Date.UTC(2026, 0, 1),
|
||||
);
|
||||
vm.runInContext(
|
||||
`globalThis.Date = original; (${shiftTime})('real');`,
|
||||
context,
|
||||
);
|
||||
assert.equal(vm.runInContext('Date === original', context), true);
|
||||
});
|
||||
|
||||
test('diagnostics keeps natural exit and the requested failure code', async (t) => {
|
||||
const old = process.env.QA_EXIT_DIAGNOSTICS;
|
||||
const code = process.exitCode;
|
||||
process.env.QA_EXIT_DIAGNOSTICS = '1';
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
|
||||
else process.env.QA_EXIT_DIAGNOSTICS = old;
|
||||
process.exitCode = code;
|
||||
});
|
||||
t.mock.method(process, 'exit', () => {});
|
||||
await exitAfterFlush(1);
|
||||
assert.equal(process.exit.mock.callCount(), 0);
|
||||
assert.equal(process.exitCode, 1);
|
||||
});
|
||||
|
||||
test('each attached page gets its clock script before it is resumed', async (t) => {
|
||||
// Network installation creates a tile-cache directory; this test makes no requests.
|
||||
t.mock.method(fs, 'mkdirSync', () => {});
|
||||
for (const time of [undefined, '2026-07-15T18:00:00Z', 'real']) {
|
||||
const sessions = new Map();
|
||||
const root = new EventEmitter();
|
||||
root.send = async () => {};
|
||||
root.connection = () => ({ session: (id) => sessions.get(id) });
|
||||
await installStrictNetwork(
|
||||
{ target: () => ({ createCDPSession: async () => root }) },
|
||||
{
|
||||
origin: 'http://localhost:4242',
|
||||
ledger: networkLedger(),
|
||||
time,
|
||||
},
|
||||
);
|
||||
for (let id = 1; id <= 2; id++) {
|
||||
const session = new EventEmitter();
|
||||
const calls = [];
|
||||
const ready = Promise.withResolvers();
|
||||
session.send = async (method, params) => {
|
||||
calls.push({ method, params });
|
||||
if (method === 'Runtime.runIfWaitingForDebugger') ready.resolve();
|
||||
};
|
||||
sessions.set(id, session);
|
||||
root.emit('Target.attachedToTarget', {
|
||||
sessionId: id,
|
||||
targetInfo: { type: 'page' },
|
||||
waitingForDebugger: true,
|
||||
});
|
||||
await ready.promise;
|
||||
const script = calls.find(
|
||||
(call) => call.method === 'Page.addScriptToEvaluateOnNewDocument',
|
||||
);
|
||||
assert.ok(script);
|
||||
assert.equal(calls.at(-1).method, 'Runtime.runIfWaitingForDebugger');
|
||||
const context = vm.createContext({});
|
||||
vm.runInContext(
|
||||
'globalThis.window = globalThis; window.top = window; globalThis.original = Date;',
|
||||
context,
|
||||
);
|
||||
vm.runInContext(script.params.source, context);
|
||||
if (time === 'real')
|
||||
assert.equal(vm.runInContext('Date === original', context), true);
|
||||
else
|
||||
assert.ok(
|
||||
Math.abs(
|
||||
vm.runInContext('Date.now()', context) -
|
||||
Date.parse(time ?? DAYTIME),
|
||||
) < 1000,
|
||||
);
|
||||
assert.equal(
|
||||
vm.runInContext(
|
||||
'Object.getOwnPropertyDescriptor(window, "Cesium").get instanceof Function',
|
||||
context,
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('finish writes failing evidence for final validation and body failures', async (t) => {
|
||||
const dir = fs.mkdtempSync('tmp_acceptance_evidence_');
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const file = `${dir}/evidence.json`;
|
||||
const old = process.env.QA_STRICT_EVIDENCE;
|
||||
process.env.QA_STRICT_EVIDENCE = file;
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_STRICT_EVIDENCE;
|
||||
else process.env.QA_STRICT_EVIDENCE = old;
|
||||
});
|
||||
t.mock.method(console, 'log', () => {});
|
||||
const strict = createStrict({
|
||||
origin: 'http://localhost:4242',
|
||||
layers: ['test'],
|
||||
});
|
||||
await assert.rejects(strict.finish(), /layers without an assertion/);
|
||||
let manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
assert.equal(manifest.status, 'fail');
|
||||
assert.match(manifest.error, /layers without an assertion/);
|
||||
await createStrict({ origin: 'http://localhost:4242', layers: [] }).finish({
|
||||
failed: new Error('body failed'),
|
||||
});
|
||||
manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
assert.equal(manifest.status, 'fail');
|
||||
assert.match(manifest.error, /body failed/);
|
||||
assert.throws(
|
||||
() =>
|
||||
createStrict({
|
||||
origin: 'http://localhost:4242',
|
||||
layers: [],
|
||||
time: 'invalid',
|
||||
}),
|
||||
/valid page time/,
|
||||
);
|
||||
});
|
||||
|
||||
test('strict.run flushes large output and exits despite active resources on success or failure', async () => {
|
||||
const harness = new URL('./qa-strict-harness.mjs', import.meta.url).href;
|
||||
for (const fail of [false, true]) {
|
||||
const source = `
|
||||
import { createStrict } from ${JSON.stringify(harness)};
|
||||
const strict = createStrict({ origin: 'http://localhost:4242', layers: [] });
|
||||
strict.launch = async () => ({ close: async () => console.log('CLOSED'), process: () => null });
|
||||
strict.finish = async () => console.log('STRICT');
|
||||
setInterval(() => {}, 1000);
|
||||
await strict.run({}, async () => {
|
||||
process.stdout.write('x'.repeat(300000) + 'OUT_END\\n');
|
||||
process.stderr.write('y'.repeat(300000) + 'ERR_END\\n');
|
||||
if (${fail}) throw new Error('body failed');
|
||||
});
|
||||
`;
|
||||
const result = await promisify(execFile)(
|
||||
process.execPath,
|
||||
['--input-type=module', '-e', source],
|
||||
{
|
||||
timeout: 10000,
|
||||
maxBuffer: 2000000,
|
||||
env: { ...process.env, QA_EXIT_DIAGNOSTICS: '' },
|
||||
},
|
||||
).catch((error) => error);
|
||||
assert.equal(result.code ?? 0, fail ? 1 : 0);
|
||||
assert.match(result.stdout, /OUT_END\nCLOSED\nSTRICT\n$/);
|
||||
assert.match(result.stderr, /ERR_END/);
|
||||
if (fail) assert.match(result.stderr, /Error: body failed/);
|
||||
}
|
||||
});
|
||||
@@ -28,6 +28,7 @@
|
||||
// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50
|
||||
// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert
|
||||
|
||||
import { closeBrowser } from './qa/browser.mjs';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
createStrict,
|
||||
@@ -311,7 +312,7 @@ for (let run = 1; run <= runs; run++) {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
await closeBrowser(browser);
|
||||
}
|
||||
const warm = (to) =>
|
||||
row.swaps
|
||||
@@ -433,8 +434,7 @@ if (strictMode) {
|
||||
report.checks = checks;
|
||||
for (const check of checks)
|
||||
console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check));
|
||||
await strict.finish({
|
||||
await strict.finishAndExit({
|
||||
failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null,
|
||||
});
|
||||
if (checks.some((c) => !c.pass)) process.exitCode = 1;
|
||||
} else await strict.finish();
|
||||
} else await strict.finishAndExit();
|
||||
@@ -1,3 +1,4 @@
|
||||
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { localPreview, waitForEngine } from './qa-strict-harness.mjs';
|
||||
// An active voice session survives engine switches. With a fake microphone
|
||||
// and a fake Realtime transport (no model is reached: /api/openai, /api/realtime
|
||||
@@ -253,6 +254,10 @@ try {
|
||||
);
|
||||
results.push('PASS stopping voice closes the transport');
|
||||
console.log(results.join('\n'));
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
await browser.close();
|
||||
await closeBrowser(browser);
|
||||
}
|
||||
await exitAfterFlush();
|
||||
@@ -56,6 +56,8 @@ const fixture = wildlifeRoutes(base.origin);
|
||||
const strict = createStrict({
|
||||
origin: base.origin,
|
||||
layers: [LAYER],
|
||||
// Wildlife freshness is checked against fixtures timestamped in Node.
|
||||
time: 'real',
|
||||
routes: fixture.routes,
|
||||
});
|
||||
await strict.run({}, async (b) => {
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
export const STATUSES = ['pass', 'flaky', 'hung', 'timeout', 'fail'];
|
||||
|
||||
export function classifyAttempt(run, manifest, { external = false } = {}) {
|
||||
if (run.error?.code === 'ETIMEDOUT')
|
||||
return manifest?.status === 'pass' ? 'hung' : 'timeout';
|
||||
if (run.error || run.signal || run.status !== 0) return 'fail';
|
||||
return manifest?.status === 'pass' || (external && !manifest)
|
||||
? 'pass'
|
||||
: 'fail';
|
||||
}
|
||||
|
||||
export function retryStatus(first, second) {
|
||||
return first !== 'pass' && second === 'pass' ? 'flaky' : (second ?? first);
|
||||
}
|
||||
|
||||
export function statusCounts(scenarios) {
|
||||
return Object.fromEntries(
|
||||
STATUSES.map((status) => [
|
||||
status,
|
||||
scenarios.filter((row) => row.status === status).length,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export function historyLine(report) {
|
||||
return {
|
||||
commit: report.commit,
|
||||
dirty: report.dirty,
|
||||
startedAt: report.startedAt,
|
||||
finishedAt: report.finishedAt,
|
||||
seconds:
|
||||
(Date.parse(report.finishedAt) - Date.parse(report.startedAt)) / 1000,
|
||||
counts: statusCounts(report.scenarios),
|
||||
// Keep the denominator correct for --only runs too.
|
||||
scenarios: report.scenarios.map((row) => row.label),
|
||||
nonPass: report.scenarios
|
||||
.filter((row) => row.status !== 'pass')
|
||||
.map(({ label, status }) => ({ label, status })),
|
||||
};
|
||||
}
|
||||
|
||||
export function flakeReport(history) {
|
||||
const rows = new Map();
|
||||
for (const run of history) {
|
||||
const nonPass = new Set(run.nonPass.map((row) => row.label));
|
||||
for (const label of run.scenarios) {
|
||||
const row = rows.get(label) ?? { label, runs: 0, passes: 0 };
|
||||
row.runs++;
|
||||
if (!nonPass.has(label)) row.passes++;
|
||||
rows.set(label, row);
|
||||
}
|
||||
}
|
||||
const share = (row) => ({
|
||||
...row,
|
||||
nonPassShare: row.runs ? (row.runs - row.passes) / row.runs : 0,
|
||||
});
|
||||
const scenarios = [...rows.values()]
|
||||
.map(share)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
b.nonPassShare - a.nonPassShare || a.label.localeCompare(b.label),
|
||||
);
|
||||
const overall = share(
|
||||
scenarios.reduce(
|
||||
(total, row) => ({
|
||||
runs: total.runs + row.runs,
|
||||
passes: total.passes + row.passes,
|
||||
}),
|
||||
{ runs: 0, passes: 0 },
|
||||
),
|
||||
);
|
||||
return { scenarios, overall };
|
||||
}
|
||||
|
||||
export function formatFlakeReport(history) {
|
||||
const { scenarios, overall } = flakeReport(history);
|
||||
return [
|
||||
'scenario\truns\tpasses\tnon-pass',
|
||||
...[...scenarios, { label: 'overall', ...overall }].map(
|
||||
(row) =>
|
||||
`${row.label}\t${row.runs}\t${row.passes}\t${(row.nonPassShare * 100).toFixed(1)}%`,
|
||||
),
|
||||
].join('\n');
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { processTree, readProcesses } from './process.mjs';
|
||||
|
||||
/** Bound shutdown even when Chrome leaves descendants holding its pipes. */
|
||||
export async function closeBrowser(browser, { deadlineMs = 5000 } = {}) {
|
||||
const start = performance.now();
|
||||
let child;
|
||||
let closed = false;
|
||||
let killed = false;
|
||||
let timer;
|
||||
try {
|
||||
child = browser.process();
|
||||
} catch {}
|
||||
try {
|
||||
closed = await Promise.race([
|
||||
Promise.resolve()
|
||||
.then(() => browser.close())
|
||||
.then(
|
||||
() => true,
|
||||
() => false,
|
||||
),
|
||||
new Promise((resolve) => {
|
||||
timer = setTimeout(() => resolve(false), deadlineMs);
|
||||
}),
|
||||
]);
|
||||
} catch {}
|
||||
clearTimeout(timer);
|
||||
if (process.env.QA_EXIT_DIAGNOSTICS !== '1') {
|
||||
try {
|
||||
if (Number.isInteger(child?.pid) && child.pid > 0) {
|
||||
// The group can outlive its leader, so probe the group itself.
|
||||
process.kill(-child.pid, 0);
|
||||
process.kill(-child.pid, 'SIGKILL');
|
||||
killed = true;
|
||||
}
|
||||
} catch {}
|
||||
for (const name of ['stdout', 'stderr']) {
|
||||
try {
|
||||
child?.[name]?.destroy();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
return { closed, killed, ms: performance.now() - start };
|
||||
}
|
||||
|
||||
export async function exitAfterFlush(code = process.exitCode ?? 0) {
|
||||
process.exitCode = code;
|
||||
if (process.env.QA_EXIT_DIAGNOSTICS === '1') return;
|
||||
await Promise.all(
|
||||
[process.stdout, process.stderr].map(
|
||||
(stream) =>
|
||||
new Promise((resolve) => {
|
||||
if (stream.destroyed) return resolve();
|
||||
stream.write('', resolve);
|
||||
}),
|
||||
),
|
||||
);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
export function diagnoseExit(pids) {
|
||||
if (process.env.QA_EXIT_DIAGNOSTICS !== '1') return;
|
||||
setTimeout(() => {
|
||||
console.error('QA EXIT resources', process.getActiveResourcesInfo());
|
||||
const processes = readProcesses();
|
||||
console.error('QA EXIT Chrome: pid ppid pgid elapsed command');
|
||||
for (const pid of pids) {
|
||||
const rows = processTree(pid, processes);
|
||||
console.error(`Chrome ${pid}:`);
|
||||
for (const row of rows) console.error(row.line);
|
||||
if (!rows.length) console.error('(no remaining processes)');
|
||||
}
|
||||
}, 5000).unref();
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { assertKeylessPreview } from '../qa-strict-harness.mjs';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import sharp from 'sharp';
|
||||
import { DAYTIME, shiftTime } from './time.mjs';
|
||||
|
||||
/** A still view over Austin, no layers, OSM imagery. */
|
||||
export const STILL_VIEW =
|
||||
@@ -18,21 +19,8 @@ export const STILL_VIEW =
|
||||
* the styles' signatures read) does not follow the time of day the
|
||||
* scenario happens to run at. Default: Austin early afternoon.
|
||||
*/
|
||||
export function pinDaytime(page, utc = '2026-11-03T18:30:00Z') {
|
||||
return page.evaluateOnNewDocument((utc) => {
|
||||
const offset = Date.parse(utc) - Date.now();
|
||||
const RealDate = Date;
|
||||
class ShiftedDate extends RealDate {
|
||||
constructor(...args) {
|
||||
if (args.length === 0) super(RealDate.now() + offset);
|
||||
else super(...args);
|
||||
}
|
||||
static now() {
|
||||
return RealDate.now() + offset;
|
||||
}
|
||||
}
|
||||
globalThis.Date = ShiftedDate;
|
||||
}, utc);
|
||||
export function pinDaytime(page, utc = DAYTIME) {
|
||||
return page.evaluateOnNewDocument(shiftTime, utc);
|
||||
}
|
||||
|
||||
/** The keyless page on `engine` with the still view (and `extra` hash). */
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { execFileSync, spawn } from 'node:child_process';
|
||||
|
||||
export function readProcesses() {
|
||||
try {
|
||||
return execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,etime=,command='], {
|
||||
encoding: 'utf8',
|
||||
timeout: 2000,
|
||||
maxBuffer: 1 << 24,
|
||||
})
|
||||
.trim()
|
||||
.split('\n')
|
||||
.flatMap((line) => {
|
||||
const match = line
|
||||
.trim()
|
||||
.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/);
|
||||
return match
|
||||
? [
|
||||
{
|
||||
pid: Number(match[1]),
|
||||
ppid: Number(match[2]),
|
||||
pgid: Number(match[3]),
|
||||
elapsed: match[4],
|
||||
command: match[5],
|
||||
line: line.trim(),
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Include the group even when its leader has already exited. */
|
||||
export function processTree(pid, processes) {
|
||||
const ids = new Set([pid]);
|
||||
let changed;
|
||||
do {
|
||||
changed = false;
|
||||
for (const row of processes) {
|
||||
if (!ids.has(row.pid) && (ids.has(row.ppid) || row.pgid === pid)) {
|
||||
ids.add(row.pid);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
} while (changed);
|
||||
return processes.filter((row) => ids.has(row.pid));
|
||||
}
|
||||
|
||||
/** Sequential callers get captured output and a deadline for the whole tree. */
|
||||
export function runScenario(command, args, { timeout, env }) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
env,
|
||||
detached: true,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const groups = new Set(child.pid ? [child.pid] : []);
|
||||
const stdout = [];
|
||||
const stderr = [];
|
||||
let bytes = 0;
|
||||
let error = null;
|
||||
let done = false;
|
||||
const rememberGroups = () => {
|
||||
const processes = readProcesses();
|
||||
for (const pid of [...groups]) {
|
||||
const rows = processTree(pid, processes);
|
||||
if (processes.length && !rows.length && pid !== child.pid)
|
||||
groups.delete(pid);
|
||||
for (const row of rows) groups.add(row.pgid);
|
||||
}
|
||||
};
|
||||
const finish = (status, signal) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(deadline);
|
||||
clearInterval(watch);
|
||||
resolve({
|
||||
status,
|
||||
signal,
|
||||
error,
|
||||
stdout: Buffer.concat(stdout).toString(),
|
||||
stderr: Buffer.concat(stderr).toString(),
|
||||
});
|
||||
};
|
||||
const stop = (code, message) => {
|
||||
if (done) return;
|
||||
error = Object.assign(new Error(message), { code });
|
||||
rememberGroups();
|
||||
// Chrome detaches into another group; retain it before Node can exit.
|
||||
for (const pid of [...groups].reverse()) {
|
||||
try {
|
||||
if (pid > 0) process.kill(-pid, 'SIGKILL');
|
||||
} catch {}
|
||||
}
|
||||
child.stdout.destroy();
|
||||
child.stderr.destroy();
|
||||
child.unref();
|
||||
finish(null, 'SIGKILL');
|
||||
};
|
||||
const capture = (chunks) => (chunk) => {
|
||||
bytes += chunk.length;
|
||||
if (bytes > 1 << 26) stop('ENOBUFS', 'Scenario output exceeded 64 MiB');
|
||||
else chunks.push(chunk);
|
||||
};
|
||||
child.stdout.on('data', capture(stdout));
|
||||
child.stderr.on('data', capture(stderr));
|
||||
child.on('error', (failure) => {
|
||||
error = failure;
|
||||
});
|
||||
child.on('close', finish);
|
||||
// Keep detached descendants identifiable if Chrome's leader exits first.
|
||||
const watch = setInterval(rememberGroups, 1000);
|
||||
const deadline = setTimeout(
|
||||
() => stop('ETIMEDOUT', `Scenario timed out after ${timeout} ms`),
|
||||
timeout,
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/**
|
||||
* Austin's early afternoon (18:30 UTC) nearest to `now`. The page clock then
|
||||
* moves by at most 12 hours: the sun is up over the test cities whenever a
|
||||
* scenario runs, and data stamped with the real time stays current.
|
||||
*/
|
||||
export function daytimeNear(now = Date.now()) {
|
||||
const day = new Date(now);
|
||||
const at = Date.UTC(
|
||||
day.getUTCFullYear(),
|
||||
day.getUTCMonth(),
|
||||
day.getUTCDate(),
|
||||
18,
|
||||
30,
|
||||
);
|
||||
const nearest = [at - DAY_MS, at, at + DAY_MS].reduce((best, next) =>
|
||||
Math.abs(next - now) < Math.abs(best - now) ? next : best,
|
||||
);
|
||||
return new Date(nearest).toISOString();
|
||||
}
|
||||
|
||||
export const DAYTIME = daytimeNear();
|
||||
|
||||
/** Keep elapsed time moving while giving the sun a repeatable position. */
|
||||
export function shiftTime(utc) {
|
||||
if (utc === 'real') return;
|
||||
const offset = Date.parse(utc) - Date.now();
|
||||
const RealDate = Date;
|
||||
class ShiftedDate extends RealDate {
|
||||
constructor(...args) {
|
||||
if (args.length === 0) super(RealDate.now() + offset);
|
||||
else super(...args);
|
||||
}
|
||||
static now() {
|
||||
return RealDate.now() + offset;
|
||||
}
|
||||
}
|
||||
globalThis.Date = ShiftedDate;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { formatFlakeReport } from './qa/acceptance.mjs';
|
||||
|
||||
try {
|
||||
if (process.argv.length !== 3)
|
||||
throw new Error('Usage: node scripts/report-acceptance-flake.mjs FILE');
|
||||
const history = readFileSync(process.argv[2], 'utf8')
|
||||
.split('\n')
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line));
|
||||
console.log(formatFlakeReport(history));
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in new issue
Block a user