mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
Stop the strict acceptance from waiting for itself
A scenario whose body ran past about 19 s idled until 45 s after its start, and twice one never exited and hit the runner's 300 s timeout after it had passed: something of Chrome outlived Browser.close and kept its stdout and stderr open. - scripts/qa/browser.mjs closes Chrome within 5 s, kills its process group and closes its pipes; the harness then finishes and exits with the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and prints the open handles and Chrome's process tree 5 s after finish. - Every strict page starts with its clock at the 18:30 UTC nearest to the real time, so Navara's sun is up over the test cities whenever the acceptance runs; a scenario can ask for its own instant or the real clock (Wildlife does, for its freshness fixtures). - The runner records a scenario that passed but did not exit as hung, kills the whole process group on a timeout, runs a failed scenario once more and reports one that passes then as flaky, and appends each run to --history=FILE; scripts/report-acceptance-flake.mjs reads it.
This commit is contained in:
1 parent
a913e5466e
commit
ed6ae54ef6
19 files changed
+1039
-104
No files matched your search
@@ -0,0 +1,305 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import vm from 'node:vm';
|
||||
import { closeBrowser, diagnoseExit, exitAfterFlush } from './qa/browser.mjs';
|
||||
import { DAYTIME, daytimeNear, shiftTime } from './qa/time.mjs';
|
||||
import {
|
||||
createStrict,
|
||||
installStrictNetwork,
|
||||
networkLedger,
|
||||
} from './qa-strict-harness.mjs';
|
||||
|
||||
function fakeBrowser(t, close = async () => {}) {
|
||||
t.mock.method(process, 'kill', () => true);
|
||||
const stdout = { destroy: t.mock.fn() };
|
||||
const stderr = { destroy: t.mock.fn() };
|
||||
return {
|
||||
close,
|
||||
process: () => ({ pid: 12345, stdout, stderr }),
|
||||
stdout,
|
||||
stderr,
|
||||
};
|
||||
}
|
||||
|
||||
test('closeBrowser kills the remaining group and drops both pipes after closing', async (t) => {
|
||||
const browser = fakeBrowser(t);
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.killed, true);
|
||||
assert.ok(result.ms >= 0);
|
||||
assert.deepEqual(
|
||||
process.kill.mock.calls.map((call) => call.arguments),
|
||||
[
|
||||
[-12345, 0],
|
||||
[-12345, 'SIGKILL'],
|
||||
],
|
||||
);
|
||||
assert.equal(browser.stdout.destroy.mock.callCount(), 1);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
|
||||
});
|
||||
|
||||
test('closeBrowser bounds a stuck close and tolerates a late rejection', async (t) => {
|
||||
let reject;
|
||||
const browser = fakeBrowser(
|
||||
t,
|
||||
() =>
|
||||
new Promise((resolve, failure) => {
|
||||
reject = failure;
|
||||
}),
|
||||
);
|
||||
const result = await closeBrowser(browser, { deadlineMs: 10 });
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.killed, true);
|
||||
assert.ok(result.ms < 1000);
|
||||
reject(new Error('closed later'));
|
||||
});
|
||||
|
||||
test('missing processes, dead groups and broken streams never prevent cleanup', async (t) => {
|
||||
const browser = fakeBrowser(t, () => {
|
||||
throw new Error('close failed');
|
||||
});
|
||||
t.mock.method(process, 'kill', () => {
|
||||
throw new Error('ESRCH');
|
||||
});
|
||||
browser.stdout.destroy = () => {
|
||||
throw new Error('pipe failed');
|
||||
};
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, false);
|
||||
assert.equal(result.killed, false);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 1);
|
||||
for (const process of [
|
||||
() => null,
|
||||
() => ({}),
|
||||
() => {
|
||||
throw new Error('no process');
|
||||
},
|
||||
]) {
|
||||
const missing = await closeBrowser({ process, close: async () => {} });
|
||||
assert.equal(missing.closed, true);
|
||||
assert.equal(missing.killed, false);
|
||||
}
|
||||
assert.equal((await closeBrowser(undefined)).closed, false);
|
||||
});
|
||||
|
||||
test('diagnostic shutdown preserves the process group and pipes', async (t) => {
|
||||
const old = process.env.QA_EXIT_DIAGNOSTICS;
|
||||
process.env.QA_EXIT_DIAGNOSTICS = '1';
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
|
||||
else process.env.QA_EXIT_DIAGNOSTICS = old;
|
||||
});
|
||||
const browser = fakeBrowser(t);
|
||||
const result = await closeBrowser(browser);
|
||||
assert.equal(result.closed, true);
|
||||
assert.equal(result.killed, false);
|
||||
assert.equal(process.kill.mock.callCount(), 0);
|
||||
assert.equal(browser.stdout.destroy.mock.callCount(), 0);
|
||||
assert.equal(browser.stderr.destroy.mock.callCount(), 0);
|
||||
let callback;
|
||||
const unref = t.mock.fn();
|
||||
t.mock.method(globalThis, 'setTimeout', (fn, delay) => {
|
||||
callback = fn;
|
||||
assert.equal(delay, 5000);
|
||||
return { unref };
|
||||
});
|
||||
t.mock.method(console, 'error', () => {});
|
||||
diagnoseExit([]);
|
||||
assert.equal(unref.mock.callCount(), 1);
|
||||
assert.equal(console.error.mock.callCount(), 0);
|
||||
callback();
|
||||
assert.equal(console.error.mock.calls[0].arguments[0], 'QA EXIT resources');
|
||||
assert.match(
|
||||
console.error.mock.calls[1].arguments[0],
|
||||
/pid ppid pgid elapsed command/,
|
||||
);
|
||||
});
|
||||
|
||||
test('the page clock moves to the nearest 18:30 UTC, at most 12 hours away', () => {
|
||||
const at = (iso) => daytimeNear(Date.parse(iso));
|
||||
assert.equal(at('2026-11-10T10:00:00Z'), '2026-11-10T18:30:00.000Z');
|
||||
assert.equal(at('2026-11-10T03:00:00Z'), '2026-11-09T18:30:00.000Z');
|
||||
assert.equal(at('2026-12-31T23:00:00Z'), '2026-12-31T18:30:00.000Z');
|
||||
assert.equal(at('2027-01-01T07:00:00Z'), '2027-01-01T18:30:00.000Z');
|
||||
assert.ok(Math.abs(Date.parse(DAYTIME) - Date.now()) <= 12 * 3_600_000);
|
||||
});
|
||||
|
||||
test('the shifted clock ticks, preserves explicit dates, and supports the real clock', () => {
|
||||
const context = vm.createContext({});
|
||||
vm.runInContext(
|
||||
`
|
||||
globalThis.tick = 1000;
|
||||
const NativeDate = Date;
|
||||
globalThis.Date = class extends NativeDate {
|
||||
static now() { return tick; }
|
||||
};
|
||||
globalThis.original = Date;
|
||||
(${shiftTime})(${JSON.stringify(DAYTIME)});
|
||||
`,
|
||||
context,
|
||||
);
|
||||
assert.equal(vm.runInContext('Date.now()', context), Date.parse(DAYTIME));
|
||||
assert.equal(vm.runInContext('new Date().toISOString()', context), DAYTIME);
|
||||
assert.equal(
|
||||
vm.runInContext('tick += 250; Date.now()', context),
|
||||
Date.parse(DAYTIME) + 250,
|
||||
);
|
||||
assert.equal(vm.runInContext('new Date(0).getTime()', context), 0);
|
||||
assert.equal(
|
||||
vm.runInContext('Date.UTC(2026, 0, 1)', context),
|
||||
Date.UTC(2026, 0, 1),
|
||||
);
|
||||
vm.runInContext(
|
||||
`globalThis.Date = original; (${shiftTime})('real');`,
|
||||
context,
|
||||
);
|
||||
assert.equal(vm.runInContext('Date === original', context), true);
|
||||
});
|
||||
|
||||
test('diagnostics keeps natural exit and the requested failure code', async (t) => {
|
||||
const old = process.env.QA_EXIT_DIAGNOSTICS;
|
||||
const code = process.exitCode;
|
||||
process.env.QA_EXIT_DIAGNOSTICS = '1';
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_EXIT_DIAGNOSTICS;
|
||||
else process.env.QA_EXIT_DIAGNOSTICS = old;
|
||||
process.exitCode = code;
|
||||
});
|
||||
t.mock.method(process, 'exit', () => {});
|
||||
await exitAfterFlush(1);
|
||||
assert.equal(process.exit.mock.callCount(), 0);
|
||||
assert.equal(process.exitCode, 1);
|
||||
});
|
||||
|
||||
test('each attached page gets its clock script before it is resumed', async (t) => {
|
||||
// Network installation creates a tile-cache directory; this test makes no requests.
|
||||
t.mock.method(fs, 'mkdirSync', () => {});
|
||||
for (const time of [undefined, '2026-07-15T18:00:00Z', 'real']) {
|
||||
const sessions = new Map();
|
||||
const root = new EventEmitter();
|
||||
root.send = async () => {};
|
||||
root.connection = () => ({ session: (id) => sessions.get(id) });
|
||||
await installStrictNetwork(
|
||||
{ target: () => ({ createCDPSession: async () => root }) },
|
||||
{
|
||||
origin: 'http://localhost:4242',
|
||||
ledger: networkLedger(),
|
||||
time,
|
||||
},
|
||||
);
|
||||
for (let id = 1; id <= 2; id++) {
|
||||
const session = new EventEmitter();
|
||||
const calls = [];
|
||||
const ready = Promise.withResolvers();
|
||||
session.send = async (method, params) => {
|
||||
calls.push({ method, params });
|
||||
if (method === 'Runtime.runIfWaitingForDebugger') ready.resolve();
|
||||
};
|
||||
sessions.set(id, session);
|
||||
root.emit('Target.attachedToTarget', {
|
||||
sessionId: id,
|
||||
targetInfo: { type: 'page' },
|
||||
waitingForDebugger: true,
|
||||
});
|
||||
await ready.promise;
|
||||
const script = calls.find(
|
||||
(call) => call.method === 'Page.addScriptToEvaluateOnNewDocument',
|
||||
);
|
||||
assert.ok(script);
|
||||
assert.equal(calls.at(-1).method, 'Runtime.runIfWaitingForDebugger');
|
||||
const context = vm.createContext({});
|
||||
vm.runInContext(
|
||||
'globalThis.window = globalThis; window.top = window; globalThis.original = Date;',
|
||||
context,
|
||||
);
|
||||
vm.runInContext(script.params.source, context);
|
||||
if (time === 'real')
|
||||
assert.equal(vm.runInContext('Date === original', context), true);
|
||||
else
|
||||
assert.ok(
|
||||
Math.abs(
|
||||
vm.runInContext('Date.now()', context) -
|
||||
Date.parse(time ?? DAYTIME),
|
||||
) < 1000,
|
||||
);
|
||||
assert.equal(
|
||||
vm.runInContext(
|
||||
'Object.getOwnPropertyDescriptor(window, "Cesium").get instanceof Function',
|
||||
context,
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('finish writes failing evidence for final validation and body failures', async (t) => {
|
||||
const dir = fs.mkdtempSync('tmp_acceptance_evidence_');
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const file = `${dir}/evidence.json`;
|
||||
const old = process.env.QA_STRICT_EVIDENCE;
|
||||
process.env.QA_STRICT_EVIDENCE = file;
|
||||
t.after(() => {
|
||||
if (old === undefined) delete process.env.QA_STRICT_EVIDENCE;
|
||||
else process.env.QA_STRICT_EVIDENCE = old;
|
||||
});
|
||||
t.mock.method(console, 'log', () => {});
|
||||
const strict = createStrict({
|
||||
origin: 'http://localhost:4242',
|
||||
layers: ['test'],
|
||||
});
|
||||
await assert.rejects(strict.finish(), /layers without an assertion/);
|
||||
let manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
assert.equal(manifest.status, 'fail');
|
||||
assert.match(manifest.error, /layers without an assertion/);
|
||||
await createStrict({ origin: 'http://localhost:4242', layers: [] }).finish({
|
||||
failed: new Error('body failed'),
|
||||
});
|
||||
manifest = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
assert.equal(manifest.status, 'fail');
|
||||
assert.match(manifest.error, /body failed/);
|
||||
assert.throws(
|
||||
() =>
|
||||
createStrict({
|
||||
origin: 'http://localhost:4242',
|
||||
layers: [],
|
||||
time: 'invalid',
|
||||
}),
|
||||
/valid page time/,
|
||||
);
|
||||
});
|
||||
|
||||
test('strict.run flushes large output and exits despite active resources on success or failure', async () => {
|
||||
const harness = new URL('./qa-strict-harness.mjs', import.meta.url).href;
|
||||
for (const fail of [false, true]) {
|
||||
const source = `
|
||||
import { createStrict } from ${JSON.stringify(harness)};
|
||||
const strict = createStrict({ origin: 'http://localhost:4242', layers: [] });
|
||||
strict.launch = async () => ({ close: async () => console.log('CLOSED'), process: () => null });
|
||||
strict.finish = async () => console.log('STRICT');
|
||||
setInterval(() => {}, 1000);
|
||||
await strict.run({}, async () => {
|
||||
process.stdout.write('x'.repeat(300000) + 'OUT_END\\n');
|
||||
process.stderr.write('y'.repeat(300000) + 'ERR_END\\n');
|
||||
if (${fail}) throw new Error('body failed');
|
||||
});
|
||||
`;
|
||||
const result = await promisify(execFile)(
|
||||
process.execPath,
|
||||
['--input-type=module', '-e', source],
|
||||
{
|
||||
timeout: 10000,
|
||||
maxBuffer: 2000000,
|
||||
env: { ...process.env, QA_EXIT_DIAGNOSTICS: '' },
|
||||
},
|
||||
).catch((error) => error);
|
||||
assert.equal(result.code ?? 0, fail ? 1 : 0);
|
||||
assert.match(result.stdout, /OUT_END\nCLOSED\nSTRICT\n$/);
|
||||
assert.match(result.stderr, /ERR_END/);
|
||||
if (fail) assert.match(result.stderr, /Error: body failed/);
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user