Stop the strict acceptance from waiting for itself

A scenario whose body ran past about 19 s idled until 45 s after its
start, and twice one never exited and hit the runner's 300 s timeout
after it had passed: something of Chrome outlived Browser.close and kept
its stdout and stderr open.

- scripts/qa/browser.mjs closes Chrome within 5 s, kills its process
  group and closes its pipes; the harness then finishes and exits with
  the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and
  prints the open handles and Chrome's process tree 5 s after finish.
- Every strict page starts with its clock at the 18:30 UTC nearest to
  the real time, so Navara's sun is up over the test cities whenever the
  acceptance runs; a scenario can ask for its own instant or the real
  clock (Wildlife does, for its freshness fixtures).
- The runner records a scenario that passed but did not exit as hung,
  kills the whole process group on a timeout, runs a failed scenario
  once more and reports one that passes then as flaky, and appends each
  run to --history=FILE; scripts/report-acceptance-flake.mjs reads it.
This commit is contained in:
German Khokhlov committed 2026-11-10 14:10:46 +04:00
1 parent a913e5466e
commit ed6ae54ef6
19 files changed
+1039 -104

No files matched your search

+10 -1
View File
@@ -416,7 +416,16 @@ one application, and only the globe-bound parts move to the next engine.
--serve` builds the commit keyless in a clean worktree and runs every layer
and switch scenario under a deny-by-default network policy, with free map
tiles from a disk cache and a trap that fails any read of Cesium before it
has loaded. Every catalog layer must be asserted on both engines. Point
has loaded. Every catalog layer must be asserted on both engines.
Page clocks start at `2026-11-03T18:30:00Z` unless a scenario supplies
`time` (an ISO instant or `'real'`); `QA_EXIT_DIAGNOSTICS=1` preserves
shutdown resources and prints them five seconds after completion.
A timed-out scenario with passing evidence is `hung`; non-passing scenarios
retry once unless `--no-retry` is set, and a passing retry is reported as
`flaky` without failing the run.
`--history=FILE` appends run summaries, and
`node scripts/report-acceptance-flake.mjs FILE` reports each scenario's
share of non-pass runs, including flaky runs. Point
`PUPPETEER_EXECUTABLE_PATH` at an installed Chrome if Puppeteer's pinned
build is missing. Navara lights the globe by the real sun; every base map,
street maps included, takes the night fill after dusk, so a scenario over
+99 -29
View File
@@ -1,6 +1,7 @@
import { execFileSync, spawn, spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
appendFileSync,
existsSync,
mkdirSync,
mkdtempSync,
@@ -24,17 +25,31 @@ import {
localPreview,
waitForEngine,
} from './qa-strict-harness.mjs';
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
import { runScenario } from './qa/process.mjs';
import {
classifyAttempt,
historyLine,
retryStatus,
statusCounts,
} from './qa/acceptance.mjs';
// The strict acceptance: every browser scenario against one keyless local
// preview, under the strict network policy (scripts/qa-strict-harness.mjs),
// writing <output>/acceptance.json, one log and one assertion manifest per
// scenario.
// scenario. A timed-out pass is hung; a retry that passes is flaky and does
// not fail the run. QA_EXIT_DIAGNOSTICS=1 preserves Chrome resources and
// reports them five seconds after strict completion instead of forcing exit.
// Pages start at 2026-11-03T18:30:00Z unless createStrict's time overrides it.
//
// node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 <output>
// [--serve[=<commit>]] build <commit> (default HEAD) keyless in a clean
// worktree and serve it with every .env key blank
// [--scratch=DIR] where --serve puts its checkout and build
// [--only=a,b] run only these labels
// [--no-retry] disable the one retry of a non-passing scenario
// [--history=FILE] append a JSONL run summary; inspect it with
// node scripts/report-acceptance-flake.mjs FILE
// [--quick] the soak with 2 runs of 10 switches and 100
// switches for the long run
//
@@ -42,6 +57,7 @@ import {
// the live page and cross-checked with src/app/layers/directory.js) has no
// assertion on either engine, or when any network policy violation shows.
const startedAt = new Date().toISOString();
const base = localPreview(process.argv[2]);
const output = process.argv[3]?.startsWith('--')
? mkdtempSync(join(tmpdir(), 'gev-acceptance-'))
@@ -67,7 +83,7 @@ let chrome;
try {
const browser = await puppeteer.launch({ headless: true });
chrome = await browser.version();
await browser.close();
await closeBrowser(browser);
} catch (error) {
console.error(
'Puppeteer cannot start Chrome: ' +
@@ -429,7 +445,7 @@ const liveCatalog = {};
await page.close();
}
} finally {
await browser.close();
await closeBrowser(browser);
}
liveCatalog.violations = strict.ledger.violations;
}
@@ -475,10 +491,11 @@ const report = {
live: liveCatalog,
scenarios: scenarioIds,
},
startedAt: new Date().toISOString(),
startedAt,
finishedAt: null,
passed: 0,
failed: 0,
counts: statusCounts([]),
networkViolations: 0,
coverage: null,
uncovered: null,
@@ -495,7 +512,6 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
if (only && !only.includes(label)) continue;
const script = 'scripts/qa-' + name + '.mjs';
const evidence = join(output, 'evidence', label + '.json');
rmSync(evidence, { force: true });
const args = [base.origin, ...extra];
if (external) args.splice(0, 1, base.origin + '/');
else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`);
@@ -503,27 +519,69 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
// With --serve the scenarios run from the same clean checkout the app was
// built from, so other work in this tree cannot change them.
const scripts = served?.worktree ?? root;
const run = spawnSync(process.execPath, [join(scripts, script), ...args], {
encoding: 'utf8',
timeout: seconds * 1000,
maxBuffer: 1 << 26,
env: scrubbed(),
});
const status =
run.status === 0
? 'pass'
: run.error?.code === 'ETIMEDOUT'
? 'timeout'
: 'fail';
const log = machinePaths(
(run.stdout || '') + (run.stderr || '') + (run.error?.message || ''),
);
writeFileSync(join(output, label + '.log'), log);
const manifest = existsSync(evidence)
? JSON.parse(machinePaths(readFileSync(evidence, 'utf8')))
const attempts = [];
for (let attempt = 1; attempt <= (flag('no-retry') ? 1 : 2); attempt++) {
rmSync(evidence, { force: true });
const attemptStart = Date.now();
const run = await runScenario(
process.execPath,
[join(scripts, script), ...args],
{
timeout: seconds * 1000,
env: scrubbed(),
},
);
const log = machinePaths(
(run.stdout || '') + (run.stderr || '') + (run.error?.message || ''),
);
let manifest = null;
let evidenceError = null;
if (existsSync(evidence)) {
try {
manifest = JSON.parse(machinePaths(readFileSync(evidence, 'utf8')));
} catch (error) {
evidenceError = 'Invalid evidence: ' + error.message;
}
}
const status =
evidenceError && !run.error
? 'fail'
: classifyAttempt(run, manifest, { external });
const logFile = `${label}.attempt-${attempt}.log`;
const evidenceFile = manifest
? `evidence/${label}.attempt-${attempt}.json`
: null;
writeFileSync(join(output, logFile), log);
if (manifest)
writeFileSync(
join(output, evidenceFile),
JSON.stringify(manifest, null, 2) + '\n',
);
attempts.push({
status,
exitCode: run.status,
seconds: Math.round((Date.now() - attemptStart) / 100) / 10,
log: logFile,
logSha256: createHash('sha256').update(log).digest('hex'),
evidence: evidenceFile,
error: evidenceError ?? manifest?.error ?? run.error?.message ?? null,
});
writeFileSync(join(output, label + '.log'), log);
if (manifest)
writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n');
console.log(
status.toUpperCase(),
label,
attempts.at(-1).seconds,
`attempt ${attempt}`,
);
if (status === 'pass' || attempt === 2 || flag('no-retry')) break;
}
const last = attempts.at(-1);
const status = retryStatus(attempts[0].status, attempts[1]?.status);
const manifest = last.evidence
? JSON.parse(readFileSync(join(output, last.evidence), 'utf8'))
: null;
if (manifest)
writeFileSync(evidence, JSON.stringify(manifest, null, 2) + '\n');
scenarios.push({
label,
script,
@@ -533,10 +591,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
layers,
external: Boolean(external),
status,
exitCode: run.status,
exitCode: last.exitCode,
seconds: Math.round((Date.now() - start) / 100) / 10,
log: label + '.log',
logSha256: createHash('sha256').update(log).digest('hex'),
logSha256: last.logSha256,
attempts,
evidence: manifest ? 'evidence/' + label + '.json' : null,
assertions: manifest?.assertions.length ?? 0,
coverage: manifest?.coverage ?? null,
@@ -554,10 +613,11 @@ for (const [label, name, layers, extra = [], seconds = 300, external] of runs) {
}
: null,
contracts: manifest?.contracts ?? null,
error: manifest?.error ?? null,
error: last.error,
});
if (manifest?.soak) scenarios.at(-1).soak = manifest.soak;
report.passed = scenarios.filter((row) => row.status === 'pass').length;
report.counts = statusCounts(scenarios);
report.passed = report.counts.pass + report.counts.flaky;
report.failed = scenarios.length - report.passed;
report.networkViolations =
scenarios.reduce(
@@ -599,9 +659,18 @@ report.catalogMatches =
report.finishedAt = new Date().toISOString();
report.dirtyFilesAtEnd = dirty();
save();
if (flag('history'))
appendFileSync(
resolve(String(flag('history'))),
JSON.stringify(historyLine(report)) + '\n',
);
console.log(
`${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` +
`${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` +
Object.entries(report.counts)
.map(([status, count]) => `${count} ${status}`)
.join(', ') +
', ' +
`paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`,
);
if (
@@ -615,3 +684,4 @@ if (
// The preview child would keep this process alive: stop it and drop the
// checkout now.
cleanup();
await exitAfterFlush();
+6 -1
View File
@@ -1,3 +1,4 @@
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
import { localPreview, waitForEngine } from './qa-strict-harness.mjs';
// The application is created once per page: switching engines detaches and
// attaches only the engine-bound parts. This gate runs three Cesium→Navara→
@@ -413,6 +414,10 @@ try {
console.log(JSON.stringify(report, null, 2));
if (!reportOnly) console.log('PASS application created once; session kept');
} catch (error) {
console.error(error);
process.exitCode = 1;
} finally {
await browser.close();
await closeBrowser(browser);
}
await exitAfterFlush();
+3 -3
View File
@@ -6,6 +6,7 @@
//
// node scripts/qa-imagery-contract.mjs http://127.0.0.1:<PORT>
// [--layers=recent-imagery] [--evidence=FILE]
import { closeBrowser } from './qa/browser.mjs';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
@@ -470,9 +471,8 @@ try {
report.errors.push(e.stack);
check('driver completes', false);
} finally {
await browser.close();
await closeBrowser(browser);
await fs.writeFile(output, JSON.stringify(report, null, 2));
}
const failed = report.checks.some((c) => !c.pass);
await strict.finish({ failed: failed ? 'imagery checks failed' : null });
if (failed) process.exitCode = 1;
await strict.finishAndExit({ failed: failed ? 'imagery checks failed' : null });
-2
View File
@@ -21,7 +21,6 @@ import {
diff,
freeze,
pickStyle,
pinDaytime,
saveCapture,
stats,
visualPageUrl,
@@ -59,7 +58,6 @@ await strict.run({}, async (browser) => {
page.on('pageerror', (error) => errors.push(error.message));
// The same sun on every run: the frame's brightness decides what blooms
// and what each style's signature reads.
await pinDaytime(page);
await page.evaluateOnNewDocument(() => {
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
});
+3 -14
View File
@@ -50,6 +50,7 @@ const asked = [];
const strict = createStrict({
origin: base.origin,
layers: [],
time: MIDDAY_UTC,
routes: [
{
pattern: `${base.origin}/api/weather-effects*`,
@@ -105,21 +106,9 @@ await strict.run({}, async (browser) => {
const page = await browser.newPage();
const errors = [];
page.on('pageerror', (error) => errors.push(error.message));
await page.evaluateOnNewDocument((utc) => {
await page.evaluateOnNewDocument(() => {
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
const offset = Date.parse(utc) - Date.now();
const RealDate = Date;
class ShiftedDate extends RealDate {
constructor(...args) {
if (args.length === 0) super(RealDate.now() + offset);
else super(...args);
}
static now() {
return RealDate.now() + offset;
}
}
globalThis.Date = ShiftedDate;
}, MIDDAY_UTC);
});
await page.setViewport({ width: 1280, height: 800 });
await page.goto(visualPageUrl(base.origin, 'navara', '', LOW_VIEW), {
waitUntil: 'domcontentloaded',
-2
View File
@@ -24,7 +24,6 @@ import {
freeze,
histogramShift,
pickStyle,
pinDaytime,
saveCapture,
stats,
visualPageUrl,
@@ -51,7 +50,6 @@ await strict.run({}, async (browser) => {
page.on('pageerror', (error) => errors.push(error.message));
// The same sun on every run: the frame's brightness decides what blooms
// and what each style's signature reads.
await pinDaytime(page);
await page.evaluateOnNewDocument(() => {
localStorage.setItem('gev:first-run-mission:v1', 'suppressed');
});
+191
View File
@@ -0,0 +1,191 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import {
classifyAttempt,
flakeReport,
formatFlakeReport,
historyLine,
retryStatus,
} from './qa/acceptance.mjs';
import { processTree, readProcesses, runScenario } from './qa/process.mjs';
test('a timeout after passing evidence is hung, even after a zero exit', () => {
const timeout = { status: 0, error: { code: 'ETIMEDOUT' } };
assert.equal(classifyAttempt(timeout, { status: 'pass' }), 'hung');
assert.equal(classifyAttempt(timeout, { status: 'fail' }), 'timeout');
assert.equal(classifyAttempt(timeout, null), 'timeout');
});
test('a pass requires a successful exit and passing evidence', () => {
const pass = { status: 'pass' };
assert.equal(classifyAttempt({ status: 0 }, pass), 'pass');
assert.equal(classifyAttempt({ status: 1 }, pass), 'fail');
assert.equal(classifyAttempt({ status: 0, signal: 'SIGTERM' }, pass), 'fail');
assert.equal(
classifyAttempt({ status: 0, error: { code: 'ENOBUFS' } }, pass),
'fail',
);
assert.equal(classifyAttempt({ status: 0 }, { status: 'fail' }), 'fail');
assert.equal(classifyAttempt({ status: 0 }, null), 'fail');
assert.equal(
classifyAttempt({ status: 0 }, null, { external: true }),
'pass',
);
});
test('a passing retry is flaky; no retry preserves the first status', () => {
for (const status of ['fail', 'timeout', 'hung']) {
assert.equal(retryStatus(status, 'pass'), 'flaky');
assert.equal(retryStatus(status), status);
assert.equal(retryStatus(status, 'fail'), 'fail');
}
assert.equal(retryStatus('pass'), 'pass');
});
const report = (scenarios) => ({
commit: 'abc123',
dirty: true,
startedAt: '2026-11-10T12:00:00Z',
finishedAt: '2026-11-10T12:00:12.500Z',
scenarios: Object.entries(scenarios).map(([label, status]) => ({
label,
status,
})),
});
test('history records provenance, duration, every status, and selected scenarios', () => {
assert.deepEqual(
historyLine(
report({ a: 'pass', b: 'flaky', c: 'hung', d: 'timeout', e: 'fail' }),
),
{
commit: 'abc123',
dirty: true,
startedAt: '2026-11-10T12:00:00Z',
finishedAt: '2026-11-10T12:00:12.500Z',
seconds: 12.5,
counts: { pass: 1, flaky: 1, hung: 1, timeout: 1, fail: 1 },
scenarios: ['a', 'b', 'c', 'd', 'e'],
nonPass: [
{ label: 'b', status: 'flaky' },
{ label: 'c', status: 'hung' },
{ label: 'd', status: 'timeout' },
{ label: 'e', status: 'fail' },
],
},
);
});
test('flake shares include flaky runs and exclude scenarios absent from partial runs', () => {
const history = [
historyLine(report({ a: 'pass', b: 'flaky', c: 'fail' })),
historyLine(report({ a: 'pass', b: 'pass' })),
historyLine(report({ a: 'hung' })),
];
assert.deepEqual(flakeReport(history), {
scenarios: [
{ label: 'c', runs: 1, passes: 0, nonPassShare: 1 },
{ label: 'b', runs: 2, passes: 1, nonPassShare: 0.5 },
{ label: 'a', runs: 3, passes: 2, nonPassShare: 1 / 3 },
],
overall: { runs: 6, passes: 3, nonPassShare: 0.5 },
});
assert.match(formatFlakeReport(history), /overall\t6\t3\t50.0%$/);
assert.deepEqual(flakeReport([]), {
scenarios: [],
overall: { runs: 0, passes: 0, nonPassShare: 0 },
});
});
test('the report CLI reads JSON lines and rejects malformed input', async (t) => {
const dir = mkdtempSync('tmp_acceptance_history_');
t.after(() => rmSync(dir, { recursive: true, force: true }));
const file = `${dir}/history.jsonl`;
writeFileSync(
file,
JSON.stringify(historyLine(report({ a: 'flaky' }))) + '\n',
);
const run = promisify(execFile);
const script = new URL('./report-acceptance-flake.mjs', import.meta.url);
const result = await run(process.execPath, [script.pathname, file]);
assert.match(result.stdout, /a\t1\t0\t100.0%/);
writeFileSync(file, '{broken}\n');
await assert.rejects(run(process.execPath, [script.pathname, file]), {
code: 1,
});
});
test('process trees include detached descendants and orphaned group members only', () => {
const rows = [
{ pid: 10, ppid: 1, pgid: 10 },
{ pid: 20, ppid: 10, pgid: 20 },
{ pid: 21, ppid: 20, pgid: 20 },
{ pid: 22, ppid: 1, pgid: 20 },
{ pid: 99, ppid: 1, pgid: 99 },
];
assert.deepEqual(
processTree(10, rows).map((row) => row.pid),
[10, 20, 21],
);
assert.deepEqual(
processTree(20, rows).map((row) => row.pid),
[20, 21, 22],
);
});
test('scenario spawning captures both streams and reports spawn failures', async () => {
const run = await runScenario(
process.execPath,
['-e', "console.log('out'); console.error('err'); process.exitCode = 3"],
{ timeout: 5000 },
);
assert.equal(run.status, 3);
assert.equal(run.stdout, 'out\n');
assert.equal(run.stderr, 'err\n');
assert.equal(run.error, null);
const missing = await runScenario('./tmp_acceptance_missing_executable', [], {
timeout: 5000,
});
assert.equal(missing.error.code, 'ENOENT');
});
test('timeout kills the scenario and a detached group after its leader exits', async (t) => {
// These Node children stand in for Chrome and a renderer; no browser is run.
const renderer = 'setInterval(() => {}, 1000)';
const browser = `
const { spawn } = require('node:child_process');
const child = spawn(process.execPath, ['-e', ${JSON.stringify(renderer)}], { stdio: 'inherit' });
console.log('renderer', child.pid);
setTimeout(() => process.exit(0), 1500);
`;
const scenario = `
const { spawn } = require('node:child_process');
const child = spawn(process.execPath, ['-e', ${JSON.stringify(browser)}], { detached: true, stdio: 'inherit' });
console.log('browser', child.pid);
console.log('scenario', process.pid);
setInterval(() => {}, 1000);
`;
const run = await runScenario(process.execPath, ['-e', scenario], {
timeout: 2600,
});
const pids = [
...run.stdout.matchAll(/(?:browser|renderer|scenario) (\d+)/g),
].map((match) => Number(match[1]));
t.after(() => {
for (const pid of pids) {
try {
process.kill(-pid, 'SIGKILL');
} catch {}
}
});
assert.equal(run.error.code, 'ETIMEDOUT');
assert.equal(pids.length, 3);
await new Promise((resolve) => setTimeout(resolve, 100));
const survivors = readProcesses().filter(
(row) => pids.includes(row.pid) && !/defunct/.test(row.command),
);
assert.deepEqual(survivors, []);
});
+76 -32
View File
@@ -27,6 +27,8 @@ import {
STRICT_TILE_HOSTS,
} from './qa-map-policy.mjs';
import { createMissLoader, tileCacheFile } from './qa/tileCache.mjs';
import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs';
import { DAYTIME, shiftTime } from './qa/time.mjs';
export const ENGINES = Object.freeze(['cesium', 'navara']);
/** What the Globe Engine picker offers: the engines and Navara HD. */
@@ -92,7 +94,14 @@ const live = (response) => Boolean(response.remoteIPAddress);
*/
export async function installStrictNetwork(
browser,
{ origin, fixtures = [], routes = [], tileHosts = STRICT_TILE_HOSTS, ledger },
{
origin,
fixtures = [],
routes = [],
tileHosts = STRICT_TILE_HOSTS,
ledger,
time = DAYTIME,
},
) {
const preview = new URL(origin);
fixtures = [...fixtures, ...routes.map((route) => route.pattern)];
@@ -298,7 +307,7 @@ export async function installStrictNetwork(
// requests, and a live answer would still show up as a violation.
if (type === 'page' || type === 'iframe')
steps.push(intercept(session, type));
if (type === 'page') steps.push(trapCesiumGlobal(session, ledger));
if (type === 'page') steps.push(trapCesiumGlobal(session, ledger, time));
await Promise.all(steps);
};
const attach = async (sessionId, info, waiting) => {
@@ -344,7 +353,7 @@ export async function installStrictNetwork(
* has assigned it are logged with their stack in `ledger.cesiumBeforeLoad`
* and answer `undefined`, as production would.
*/
async function trapCesiumGlobal(session, ledger) {
async function trapCesiumGlobal(session, ledger, time) {
const binding = '__gevStrictCesiumRead';
session.on('Runtime.bindingCalled', (event) => {
if (event.name !== binding) return;
@@ -358,7 +367,8 @@ async function trapCesiumGlobal(session, ledger) {
await session.send('Runtime.addBinding', { name: binding });
await session.send('Page.enable');
await session.send('Page.addScriptToEvaluateOnNewDocument', {
source: `(() => {
source: `(${shiftTime})(${JSON.stringify(time)});
(() => {
if (window.top !== window) return;
const seen = new Set();
let sdk;
@@ -630,6 +640,7 @@ export async function openPage(browser, { shaderErrors = false } = {}) {
* @param {readonly string[]} [options.tileHosts] Hosts the disk cache
* answers (default STRICT_TILE_HOSTS).
* @param {string} [options.name] Scenario name for the manifest.
* @param {string} [options.time] ISO instant for the page clock, or 'real'.
*/
export function createStrict({
origin,
@@ -638,7 +649,14 @@ export function createStrict({
routes = [],
tileHosts = STRICT_TILE_HOSTS,
name = path.basename(process.argv[1] ?? 'scenario', '.mjs'),
time = DAYTIME,
}) {
assert.ok(
typeof time === 'string' &&
(time === 'real' || Number.isFinite(Date.parse(time))),
'valid page time',
);
const browserPids = [];
const args = strictArguments();
if (args.layers)
assert.deepEqual(
@@ -685,28 +703,45 @@ export function createStrict({
...options,
args: [...CHROME_ARGS, ...args],
});
await installStrictNetwork(browser, {
origin,
fixtures,
routes,
tileHosts,
ledger,
});
const pid = browser.process()?.pid;
if (pid) browserPids.push(pid);
try {
await installStrictNetwork(browser, {
origin,
fixtures,
routes,
tileHosts,
ledger,
time,
});
} catch (error) {
await closeBrowser(browser);
throw error;
}
return browser;
},
/** Launch with `options`, run `body(browser)`, close, then finish(). */
async run(options, body) {
const browser = await strict.launch(options);
let browser;
let failed = null;
try {
browser = await strict.launch(options);
await body(browser);
} catch (error) {
failed = error;
} finally {
await browser.close().catch(() => {});
await closeBrowser(browser);
}
await strict.finish({ failed });
if (failed) throw failed;
await strict.finishAndExit({ failed });
},
async finishAndExit({ failed = null } = {}) {
try {
await strict.finish({ failed });
} catch (error) {
failed = error;
}
if (failed) console.error(failed);
await exitAfterFlush(failed ? 1 : process.exitCode ?? 0);
},
/** Record an assertion that already passed. */
record,
@@ -788,6 +823,30 @@ export function createStrict({
fixtures,
tileHosts: [...tileHosts],
};
let validationError = null;
if (!failed) {
try {
assert.equal(
ledger.violations.length,
0,
'network policy violations: ' +
JSON.stringify(ledger.violations.slice(0, 5)),
);
assert.deepEqual(
ledger.cesiumBeforeLoad,
[],
'the page read the Cesium global before Cesium.js loaded',
);
assert.deepEqual(missing, [], 'layers without an assertion');
assert.ok(
picks.some((pick) => pick.via === 'pointer+keyboard'),
'switch at least once with real input on the picker',
);
} catch (error) {
validationError = error;
failed = error;
}
}
const manifest = {
scenario: name,
argv: process.argv.slice(2),
@@ -825,23 +884,8 @@ export function createStrict({
console.log('NETWORK VIOLATION', JSON.stringify(v));
for (const read of ledger.cesiumBeforeLoad.slice(0, 10))
console.log('CESIUM BEFORE LOAD', JSON.stringify(read));
if (failed) return;
assert.equal(
ledger.violations.length,
0,
'network policy violations: ' +
JSON.stringify(ledger.violations.slice(0, 5)),
);
assert.deepEqual(
ledger.cesiumBeforeLoad,
[],
'the page read the Cesium global before Cesium.js loaded',
);
assert.deepEqual(missing, [], 'layers without an assertion');
assert.ok(
picks.some((pick) => pick.via === 'pointer+keyboard'),
'switch at least once with real input on the picker',
);
diagnoseExit(browserPids);
if (validationError) throw validationError;
},
};
return strict;
+305
View File
@@ -0,0 +1,305 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { EventEmitter } from 'node:events';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import vm from 'node:vm';
import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs';
import { DAYTIME, daytimeNear, shiftTime } from './qa/time.mjs';
import {
createStrict,
installStrictNetwork,
networkLedger,
} from './qa-strict-harness.mjs';
function fakeBrowser(t, close = async () => {}) {
t.mock.method(process, 'kill', () => true);
const stdout = { destroy: t.mock.fn() };
const stderr = { destroy: t.mock.fn() };
return {
close,
process: () => ({ pid: 12345, stdout, stderr }),
stdout,
stderr,
};
}
test('closeBrowser kills the remaining group and drops both pipes after closing', async (t) => {
const browser = fakeBrowser(t);
const result = await closeBrowser(browser);
assert.equal(result.closed, true);
assert.equal(result.killed, true);
assert.ok(result.ms >= 0);
assert.deepEqual(
process.kill.mock.calls.map((call) => call.arguments),
[
[-12345, 0],
[-12345, 'SIGKILL'],
],
);
assert.equal(browser.stdout.destroy.mock.callCount(), 1);
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
});
test('closeBrowser bounds a stuck close and tolerates a late rejection', async (t) => {
let reject;
const browser = fakeBrowser(
t,
() =>
new Promise((resolve, failure) => {
reject = failure;
}),
);
const result = await closeBrowser(browser, { deadlineMs: 10 });
assert.equal(result.closed, false);
assert.equal(result.killed, true);
assert.ok(result.ms < 1000);
reject(new Error('closed later'));
});
test('missing processes, dead groups and broken streams never prevent cleanup', async (t) => {
const browser = fakeBrowser(t, () => {
throw new Error('close failed');
});
t.mock.method(process, 'kill', () => {
throw new Error('ESRCH');
});
browser.stdout.destroy = () => {
throw new Error('pipe failed');
};
const result = await closeBrowser(browser);
assert.equal(result.closed, false);
assert.equal(result.killed, false);
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
for (const process of [
() => null,
() => ({}),
() => {
throw new Error('no process');
},
]) {
const missing = await closeBrowser({ process, close: async () => {} });
assert.equal(missing.closed, true);
assert.equal(missing.killed, false);
}
assert.equal((await closeBrowser(undefined)).closed, false);
});
test('diagnostic shutdown preserves the process group and pipes', async (t) => {
const old = process.env.QA_EXIT_DIAGNOSTICS;
process.env.QA_EXIT_DIAGNOSTICS = '1';
t.after(() => {
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
else process.env.QA_EXIT_DIAGNOSTICS = old;
});
const browser = fakeBrowser(t);
const result = await closeBrowser(browser);
assert.equal(result.closed, true);
assert.equal(result.killed, false);
assert.equal(process.kill.mock.callCount(), 0);
assert.equal(browser.stdout.destroy.mock.callCount(), 0);
assert.equal(browser.stderr.destroy.mock.callCount(), 0);
let callback;
const unref = t.mock.fn();
t.mock.method(globalThis, 'setTimeout', (fn, delay) => {
callback = fn;
assert.equal(delay, 5000);
return { unref };
});
t.mock.method(console, 'error', () => {});
diagnoseExit([]);
assert.equal(unref.mock.callCount(), 1);
assert.equal(console.error.mock.callCount(), 0);
callback();
assert.equal(console.error.mock.calls[0].arguments[0], 'QA EXIT resources');
assert.match(
console.error.mock.calls[1].arguments[0],
/pid ppid pgid elapsed command/,
);
});
test('the page clock moves to the nearest 18:30 UTC, at most 12 hours away', () => {
const at = (iso) => daytimeNear(Date.parse(iso));
assert.equal(at('2026-11-10T10:00:00Z'), '2026-11-10T18:30:00.000Z');
assert.equal(at('2026-11-10T03:00:00Z'), '2026-11-09T18:30:00.000Z');
assert.equal(at('2026-12-31T23:00:00Z'), '2026-12-31T18:30:00.000Z');
assert.equal(at('2027-01-01T07:00:00Z'), '2027-01-01T18:30:00.000Z');
assert.ok(Math.abs(Date.parse(DAYTIME) - Date.now()) <= 12 * 3_600_000);
});
test('the shifted clock ticks, preserves explicit dates, and supports the real clock', () => {
const context = vm.createContext({});
vm.runInContext(
`
globalThis.tick = 1000;
const NativeDate = Date;
globalThis.Date = class extends NativeDate {
static now() { return tick; }
};
globalThis.original = Date;
(${shiftTime})(${JSON.stringify(DAYTIME)});
`,
context,
);
assert.equal(vm.runInContext('Date.now()', context), Date.parse(DAYTIME));
assert.equal(vm.runInContext('new Date().toISOString()', context), DAYTIME);
assert.equal(
vm.runInContext('tick += 250; Date.now()', context),
Date.parse(DAYTIME) + 250,
);
assert.equal(vm.runInContext('new Date(0).getTime()', context), 0);
assert.equal(
vm.runInContext('Date.UTC(2026, 0, 1)', context),
Date.UTC(2026, 0, 1),
);
vm.runInContext(
`globalThis.Date = original; (${shiftTime})('real');`,
context,
);
assert.equal(vm.runInContext('Date === original', context), true);
});
test('diagnostics keeps natural exit and the requested failure code', async (t) => {
const old = process.env.QA_EXIT_DIAGNOSTICS;
const code = process.exitCode;
process.env.QA_EXIT_DIAGNOSTICS = '1';
t.after(() => {
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
else process.env.QA_EXIT_DIAGNOSTICS = old;
process.exitCode = code;
});
t.mock.method(process, 'exit', () => {});
await exitAfterFlush(1);
assert.equal(process.exit.mock.callCount(), 0);
assert.equal(process.exitCode, 1);
});
test('each attached page gets its clock script before it is resumed', async (t) => {
// Network installation creates a tile-cache directory; this test makes no requests.
t.mock.method(fs, 'mkdirSync', () => {});
for (const time of [undefined, '2026-07-15T18:00:00Z', 'real']) {
const sessions = new Map();
const root = new EventEmitter();
root.send = async () => {};
root.connection = () => ({ session: (id) => sessions.get(id) });
await installStrictNetwork(
{ target: () => ({ createCDPSession: async () => root }) },
{
origin: 'http://localhost:4242',
ledger: networkLedger(),
time,
},
);
for (let id = 1; id <= 2; id++) {
const session = new EventEmitter();
const calls = [];
const ready = Promise.withResolvers();
session.send = async (method, params) => {
calls.push({ method, params });
if (method === 'Runtime.runIfWaitingForDebugger') ready.resolve();
};
sessions.set(id, session);
root.emit('Target.attachedToTarget', {
sessionId: id,
targetInfo: { type: 'page' },
waitingForDebugger: true,
});
await ready.promise;
const script = calls.find(
(call) => call.method === 'Page.addScriptToEvaluateOnNewDocument',
);
assert.ok(script);
assert.equal(calls.at(-1).method, 'Runtime.runIfWaitingForDebugger');
const context = vm.createContext({});
vm.runInContext(
'globalThis.window = globalThis; window.top = window; globalThis.original = Date;',
context,
);
vm.runInContext(script.params.source, context);
if (time === 'real')
assert.equal(vm.runInContext('Date === original', context), true);
else
assert.ok(
Math.abs(
vm.runInContext('Date.now()', context) -
Date.parse(time ?? DAYTIME),
) < 1000,
);
assert.equal(
vm.runInContext(
'Object.getOwnPropertyDescriptor(window, "Cesium").get instanceof Function',
context,
),
true,
);
}
}
});
test('finish writes failing evidence for final validation and body failures', async (t) => {
const dir = fs.mkdtempSync('tmp_acceptance_evidence_');
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
const file = `${dir}/evidence.json`;
const old = process.env.QA_STRICT_EVIDENCE;
process.env.QA_STRICT_EVIDENCE = file;
t.after(() => {
if (old === undefined) delete process.env.QA_STRICT_EVIDENCE;
else process.env.QA_STRICT_EVIDENCE = old;
});
t.mock.method(console, 'log', () => {});
const strict = createStrict({
origin: 'http://localhost:4242',
layers: ['test'],
});
await assert.rejects(strict.finish(), /layers without an assertion/);
let manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
assert.equal(manifest.status, 'fail');
assert.match(manifest.error, /layers without an assertion/);
await createStrict({ origin: 'http://localhost:4242', layers: [] }).finish({
failed: new Error('body failed'),
});
manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
assert.equal(manifest.status, 'fail');
assert.match(manifest.error, /body failed/);
assert.throws(
() =>
createStrict({
origin: 'http://localhost:4242',
layers: [],
time: 'invalid',
}),
/valid page time/,
);
});
test('strict.run flushes large output and exits despite active resources on success or failure', async () => {
const harness = new URL('./qa-strict-harness.mjs', import.meta.url).href;
for (const fail of [false, true]) {
const source = `
import { createStrict } from ${JSON.stringify(harness)};
const strict = createStrict({ origin: 'http://localhost:4242', layers: [] });
strict.launch = async () => ({ close: async () => console.log('CLOSED'), process: () => null });
strict.finish = async () => console.log('STRICT');
setInterval(() => {}, 1000);
await strict.run({}, async () => {
process.stdout.write('x'.repeat(300000) + 'OUT_END\\n');
process.stderr.write('y'.repeat(300000) + 'ERR_END\\n');
if (${fail}) throw new Error('body failed');
});
`;
const result = await promisify(execFile)(
process.execPath,
['--input-type=module', '-e', source],
{
timeout: 10000,
maxBuffer: 2000000,
env: { ...process.env, QA_EXIT_DIAGNOSTICS: '' },
},
).catch((error) => error);
assert.equal(result.code ?? 0, fail ? 1 : 0);
assert.match(result.stdout, /OUT_END\nCLOSED\nSTRICT\n$/);
assert.match(result.stderr, /ERR_END/);
if (fail) assert.match(result.stderr, /Error: body failed/);
}
});
+4 -4
View File
@@ -28,6 +28,7 @@
// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50
// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert
import { closeBrowser } from './qa/browser.mjs';
import assert from 'node:assert/strict';
import {
createStrict,
@@ -311,7 +312,7 @@ for (let run = 1; run <= runs; run++) {
}
}
} finally {
await browser.close().catch(() => {});
await closeBrowser(browser);
}
const warm = (to) =>
row.swaps
@@ -433,8 +434,7 @@ if (strictMode) {
report.checks = checks;
for (const check of checks)
console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check));
await strict.finish({
await strict.finishAndExit({
failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null,
});
if (checks.some((c) => !c.pass)) process.exitCode = 1;
} else await strict.finish();
} else await strict.finishAndExit();
+6 -1
View File
@@ -1,3 +1,4 @@
import { closeBrowser, exitAfterFlush } from './qa/browser.mjs';
import { localPreview, waitForEngine } from './qa-strict-harness.mjs';
// An active voice session survives engine switches. With a fake microphone
// and a fake Realtime transport (no model is reached: /api/openai, /api/realtime
@@ -253,6 +254,10 @@ try {
);
results.push('PASS stopping voice closes the transport');
console.log(results.join('\n'));
} catch (error) {
console.error(error);
process.exitCode = 1;
} finally {
await browser.close();
await closeBrowser(browser);
}
await exitAfterFlush();
+2
View File
@@ -56,6 +56,8 @@ const fixture = wildlifeRoutes(base.origin);
const strict = createStrict({
origin: base.origin,
layers: [LAYER],
// Wildlife freshness is checked against fixtures timestamped in Node.
time: 'real',
routes: fixture.routes,
});
await strict.run({}, async (b) => {
+84
View File
@@ -0,0 +1,84 @@
export const STATUSES = ['pass', 'flaky', 'hung', 'timeout', 'fail'];
export function classifyAttempt(run, manifest, { external = false } = {}) {
if (run.error?.code === 'ETIMEDOUT')
return manifest?.status === 'pass' ? 'hung' : 'timeout';
if (run.error || run.signal || run.status !== 0) return 'fail';
return manifest?.status === 'pass' || (external && !manifest)
? 'pass'
: 'fail';
}
export function retryStatus(first, second) {
return first !== 'pass' && second === 'pass' ? 'flaky' : (second ?? first);
}
export function statusCounts(scenarios) {
return Object.fromEntries(
STATUSES.map((status) => [
status,
scenarios.filter((row) => row.status === status).length,
]),
);
}
export function historyLine(report) {
return {
commit: report.commit,
dirty: report.dirty,
startedAt: report.startedAt,
finishedAt: report.finishedAt,
seconds:
(Date.parse(report.finishedAt) - Date.parse(report.startedAt)) / 1000,
counts: statusCounts(report.scenarios),
// Keep the denominator correct for --only runs too.
scenarios: report.scenarios.map((row) => row.label),
nonPass: report.scenarios
.filter((row) => row.status !== 'pass')
.map(({ label, status }) => ({ label, status })),
};
}
export function flakeReport(history) {
const rows = new Map();
for (const run of history) {
const nonPass = new Set(run.nonPass.map((row) => row.label));
for (const label of run.scenarios) {
const row = rows.get(label) ?? { label, runs: 0, passes: 0 };
row.runs++;
if (!nonPass.has(label)) row.passes++;
rows.set(label, row);
}
}
const share = (row) => ({
...row,
nonPassShare: row.runs ? (row.runs - row.passes) / row.runs : 0,
});
const scenarios = [...rows.values()]
.map(share)
.sort(
(a, b) =>
b.nonPassShare - a.nonPassShare || a.label.localeCompare(b.label),
);
const overall = share(
scenarios.reduce(
(total, row) => ({
runs: total.runs + row.runs,
passes: total.passes + row.passes,
}),
{ runs: 0, passes: 0 },
),
);
return { scenarios, overall };
}
export function formatFlakeReport(history) {
const { scenarios, overall } = flakeReport(history);
return [
'scenario\truns\tpasses\tnon-pass',
...[...scenarios, { label: 'overall', ...overall }].map(
(row) =>
`${row.label}\t${row.runs}\t${row.passes}\t${(row.nonPassShare * 100).toFixed(1)}%`,
),
].join('\n');
}
+73
View File
@@ -0,0 +1,73 @@
import { processTree, readProcesses } from './process.mjs';
/** Bound shutdown even when Chrome leaves descendants holding its pipes. */
export async function closeBrowser(browser, { deadlineMs = 5000 } = {}) {
const start = performance.now();
let child;
let closed = false;
let killed = false;
let timer;
try {
child = browser.process();
} catch {}
try {
closed = await Promise.race([
Promise.resolve()
.then(() => browser.close())
.then(
() => true,
() => false,
),
new Promise((resolve) => {
timer = setTimeout(() => resolve(false), deadlineMs);
}),
]);
} catch {}
clearTimeout(timer);
if (process.env.QA_EXIT_DIAGNOSTICS !== '1') {
try {
if (Number.isInteger(child?.pid) && child.pid > 0) {
// The group can outlive its leader, so probe the group itself.
process.kill(-child.pid, 0);
process.kill(-child.pid, 'SIGKILL');
killed = true;
}
} catch {}
for (const name of ['stdout', 'stderr']) {
try {
child?.[name]?.destroy();
} catch {}
}
}
return { closed, killed, ms: performance.now() - start };
}
export async function exitAfterFlush(code = process.exitCode ?? 0) {
process.exitCode = code;
if (process.env.QA_EXIT_DIAGNOSTICS === '1') return;
await Promise.all(
[process.stdout, process.stderr].map(
(stream) =>
new Promise((resolve) => {
if (stream.destroyed) return resolve();
stream.write('', resolve);
}),
),
);
process.exit(code);
}
export function diagnoseExit(pids) {
if (process.env.QA_EXIT_DIAGNOSTICS !== '1') return;
setTimeout(() => {
console.error('QA EXIT resources', process.getActiveResourcesInfo());
const processes = readProcesses();
console.error('QA EXIT Chrome: pid ppid pgid elapsed command');
for (const pid of pids) {
const rows = processTree(pid, processes);
console.error(`Chrome ${pid}:`);
for (const row of rows) console.error(row.line);
if (!rows.length) console.error('(no remaining processes)');
}
}, 5000).unref();
}
+3 -15
View File
@@ -7,6 +7,7 @@ import { assertKeylessPreview } from '../qa-strict-harness.mjs';
import fs from 'node:fs';
import path from 'node:path';
import sharp from 'sharp';
import { DAYTIME, shiftTime } from './time.mjs';
/** A still view over Austin, no layers, OSM imagery. */
export const STILL_VIEW =
@@ -18,21 +19,8 @@ export const STILL_VIEW =
* the styles' signatures read) does not follow the time of day the
* scenario happens to run at. Default: Austin early afternoon.
*/
export function pinDaytime(page, utc = '2026-11-03T18:30:00Z') {
return page.evaluateOnNewDocument((utc) => {
const offset = Date.parse(utc) - Date.now();
const RealDate = Date;
class ShiftedDate extends RealDate {
constructor(...args) {
if (args.length === 0) super(RealDate.now() + offset);
else super(...args);
}
static now() {
return RealDate.now() + offset;
}
}
globalThis.Date = ShiftedDate;
}, utc);
export function pinDaytime(page, utc = DAYTIME) {
return page.evaluateOnNewDocument(shiftTime, utc);
}
/** The keyless page on `engine` with the still view (and `extra` hash). */
+119
View File
@@ -0,0 +1,119 @@
import { execFileSync, spawn } from 'node:child_process';
export function readProcesses() {
try {
return execFileSync('ps', ['-axo', 'pid=,ppid=,pgid=,etime=,command='], {
encoding: 'utf8',
timeout: 2000,
maxBuffer: 1 << 24,
})
.trim()
.split('\n')
.flatMap((line) => {
const match = line
.trim()
.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/);
return match
? [
{
pid: Number(match[1]),
ppid: Number(match[2]),
pgid: Number(match[3]),
elapsed: match[4],
command: match[5],
line: line.trim(),
},
]
: [];
});
} catch {
return [];
}
}
/** Include the group even when its leader has already exited. */
export function processTree(pid, processes) {
const ids = new Set([pid]);
let changed;
do {
changed = false;
for (const row of processes) {
if (!ids.has(row.pid) && (ids.has(row.ppid) || row.pgid === pid)) {
ids.add(row.pid);
changed = true;
}
}
} while (changed);
return processes.filter((row) => ids.has(row.pid));
}
/** Sequential callers get captured output and a deadline for the whole tree. */
export function runScenario(command, args, { timeout, env }) {
return new Promise((resolve) => {
const child = spawn(command, args, {
env,
detached: true,
stdio: ['ignore', 'pipe', 'pipe'],
});
const groups = new Set(child.pid ? [child.pid] : []);
const stdout = [];
const stderr = [];
let bytes = 0;
let error = null;
let done = false;
const rememberGroups = () => {
const processes = readProcesses();
for (const pid of [...groups]) {
const rows = processTree(pid, processes);
if (processes.length && !rows.length && pid !== child.pid)
groups.delete(pid);
for (const row of rows) groups.add(row.pgid);
}
};
const finish = (status, signal) => {
if (done) return;
done = true;
clearTimeout(deadline);
clearInterval(watch);
resolve({
status,
signal,
error,
stdout: Buffer.concat(stdout).toString(),
stderr: Buffer.concat(stderr).toString(),
});
};
const stop = (code, message) => {
if (done) return;
error = Object.assign(new Error(message), { code });
rememberGroups();
// Chrome detaches into another group; retain it before Node can exit.
for (const pid of [...groups].reverse()) {
try {
if (pid > 0) process.kill(-pid, 'SIGKILL');
} catch {}
}
child.stdout.destroy();
child.stderr.destroy();
child.unref();
finish(null, 'SIGKILL');
};
const capture = (chunks) => (chunk) => {
bytes += chunk.length;
if (bytes > 1 << 26) stop('ENOBUFS', 'Scenario output exceeded 64 MiB');
else chunks.push(chunk);
};
child.stdout.on('data', capture(stdout));
child.stderr.on('data', capture(stderr));
child.on('error', (failure) => {
error = failure;
});
child.on('close', finish);
// Keep detached descendants identifiable if Chrome's leader exits first.
const watch = setInterval(rememberGroups, 1000);
const deadline = setTimeout(
() => stop('ETIMEDOUT', `Scenario timed out after ${timeout} ms`),
timeout,
);
});
}
+40
View File
@@ -0,0 +1,40 @@
const DAY_MS = 86_400_000;
/**
* Austin's early afternoon (18:30 UTC) nearest to `now`. The page clock then
* moves by at most 12 hours: the sun is up over the test cities whenever a
* scenario runs, and data stamped with the real time stays current.
*/
export function daytimeNear(now = Date.now()) {
const day = new Date(now);
const at = Date.UTC(
day.getUTCFullYear(),
day.getUTCMonth(),
day.getUTCDate(),
18,
30,
);
const nearest = [at - DAY_MS, at, at + DAY_MS].reduce((best, next) =>
Math.abs(next - now) < Math.abs(best - now) ? next : best,
);
return new Date(nearest).toISOString();
}
export const DAYTIME = daytimeNear();
/** Keep elapsed time moving while giving the sun a repeatable position. */
export function shiftTime(utc) {
if (utc === 'real') return;
const offset = Date.parse(utc) - Date.now();
const RealDate = Date;
class ShiftedDate extends RealDate {
constructor(...args) {
if (args.length === 0) super(RealDate.now() + offset);
else super(...args);
}
static now() {
return RealDate.now() + offset;
}
}
globalThis.Date = ShiftedDate;
}
+15
View File
@@ -0,0 +1,15 @@
import { readFileSync } from 'node:fs';
import { formatFlakeReport } from './qa/acceptance.mjs';
try {
if (process.argv.length !== 3)
throw new Error('Usage: node scripts/report-acceptance-flake.mjs FILE');
const history = readFileSync(process.argv[2], 'utf8')
.split('\n')
.filter((line) => line.trim())
.map((line) => JSON.parse(line));
console.log(formatFlakeReport(history));
} catch (error) {
console.error(error.message);
process.exitCode = 1;
}