mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
Stop the strict acceptance from waiting for itself
A scenario whose body ran past about 19 s idled until 45 s after its start, and twice one never exited and hit the runner's 300 s timeout after it had passed: something of Chrome outlived Browser.close and kept its stdout and stderr open. - scripts/qa/browser.mjs closes Chrome within 5 s, kills its process group and closes its pipes; the harness then finishes and exits with the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and prints the open handles and Chrome's process tree 5 s after finish. - Every strict page starts with its clock at the 18:30 UTC nearest to the real time, so Navara's sun is up over the test cities whenever the acceptance runs; a scenario can ask for its own instant or the real clock (Wildlife does, for its freshness fixtures). - The runner records a scenario that passed but did not exit as hung, kills the whole process group on a timeout, runs a failed scenario once more and reports one that passes then as flaky, and appends each run to --history=FILE; scripts/report-acceptance-flake.mjs reads it.
This commit is contained in:
1 parent
a913e5466e
commit
ed6ae54ef6
19 files changed
+1039
-104
No files matched your search
@@ -0,0 +1,84 @@
|
||||
export const STATUSES = ['pass', 'flaky', 'hung', 'timeout', 'fail'];
|
||||
|
||||
export function classifyAttempt(run, manifest, { external = false } = {}) {
|
||||
if (run.error?.code === 'ETIMEDOUT')
|
||||
return manifest?.status === 'pass' ? 'hung' : 'timeout';
|
||||
if (run.error || run.signal || run.status !== 0) return 'fail';
|
||||
return manifest?.status === 'pass' || (external && !manifest)
|
||||
? 'pass'
|
||||
: 'fail';
|
||||
}
|
||||
|
||||
export function retryStatus(first, second) {
|
||||
return first !== 'pass' && second === 'pass' ? 'flaky' : (second ?? first);
|
||||
}
|
||||
|
||||
export function statusCounts(scenarios) {
|
||||
return Object.fromEntries(
|
||||
STATUSES.map((status) => [
|
||||
status,
|
||||
scenarios.filter((row) => row.status === status).length,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export function historyLine(report) {
|
||||
return {
|
||||
commit: report.commit,
|
||||
dirty: report.dirty,
|
||||
startedAt: report.startedAt,
|
||||
finishedAt: report.finishedAt,
|
||||
seconds:
|
||||
(Date.parse(report.finishedAt) - Date.parse(report.startedAt)) / 1000,
|
||||
counts: statusCounts(report.scenarios),
|
||||
// Keep the denominator correct for --only runs too.
|
||||
scenarios: report.scenarios.map((row) => row.label),
|
||||
nonPass: report.scenarios
|
||||
.filter((row) => row.status !== 'pass')
|
||||
.map(({ label, status }) => ({ label, status })),
|
||||
};
|
||||
}
|
||||
|
||||
export function flakeReport(history) {
|
||||
const rows = new Map();
|
||||
for (const run of history) {
|
||||
const nonPass = new Set(run.nonPass.map((row) => row.label));
|
||||
for (const label of run.scenarios) {
|
||||
const row = rows.get(label) ?? { label, runs: 0, passes: 0 };
|
||||
row.runs++;
|
||||
if (!nonPass.has(label)) row.passes++;
|
||||
rows.set(label, row);
|
||||
}
|
||||
}
|
||||
const share = (row) => ({
|
||||
...row,
|
||||
nonPassShare: row.runs ? (row.runs - row.passes) / row.runs : 0,
|
||||
});
|
||||
const scenarios = [...rows.values()]
|
||||
.map(share)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
b.nonPassShare - a.nonPassShare || a.label.localeCompare(b.label),
|
||||
);
|
||||
const overall = share(
|
||||
scenarios.reduce(
|
||||
(total, row) => ({
|
||||
runs: total.runs + row.runs,
|
||||
passes: total.passes + row.passes,
|
||||
}),
|
||||
{ runs: 0, passes: 0 },
|
||||
),
|
||||
);
|
||||
return { scenarios, overall };
|
||||
}
|
||||
|
||||
export function formatFlakeReport(history) {
|
||||
const { scenarios, overall } = flakeReport(history);
|
||||
return [
|
||||
'scenario\truns\tpasses\tnon-pass',
|
||||
...[...scenarios, { label: 'overall', ...overall }].map(
|
||||
(row) =>
|
||||
`${row.label}\t${row.runs}\t${row.passes}\t${(row.nonPassShare * 100).toFixed(1)}%`,
|
||||
),
|
||||
].join('\n');
|
||||
}
|
||||
Reference in new issue
Block a user