mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-15 15:52:21 +00:00
German Khokhlov
ed6ae54ef6
A scenario whose body ran past about 19 s idled until 45 s after its start, and twice one never exited and hit the runner's 300 s timeout after it had passed: something of Chrome outlived Browser.close and kept its stdout and stderr open. - scripts/qa/browser.mjs closes Chrome within 5 s, kills its process group and closes its pipes; the harness then finishes and exits with the scenario's code. QA_EXIT_DIAGNOSTICS=1 keeps the old ending and prints the open handles and Chrome's process tree 5 s after finish. - Every strict page starts with its clock at the 18:30 UTC nearest to the real time, so Navara's sun is up over the test cities whenever the acceptance runs; a scenario can ask for its own instant or the real clock (Wildlife does, for its freshness fixtures). - The runner records a scenario that passed but did not exit as hung, kills the whole process group on a timeout, runs a failed scenario once more and reports one that passes then as flaky, and appends each run to --history=FILE; scripts/report-acceptance-flake.mjs reads it.
85 lines
2.5 KiB
JavaScript
85 lines
2.5 KiB
JavaScript
export const STATUSES = ['pass', 'flaky', 'hung', 'timeout', 'fail'];
|
|
|
|
export function classifyAttempt(run, manifest, { external = false } = {}) {
|
|
if (run.error?.code === 'ETIMEDOUT')
|
|
return manifest?.status === 'pass' ? 'hung' : 'timeout';
|
|
if (run.error || run.signal || run.status !== 0) return 'fail';
|
|
return manifest?.status === 'pass' || (external && !manifest)
|
|
? 'pass'
|
|
: 'fail';
|
|
}
|
|
|
|
export function retryStatus(first, second) {
|
|
return first !== 'pass' && second === 'pass' ? 'flaky' : (second ?? first);
|
|
}
|
|
|
|
export function statusCounts(scenarios) {
|
|
return Object.fromEntries(
|
|
STATUSES.map((status) => [
|
|
status,
|
|
scenarios.filter((row) => row.status === status).length,
|
|
]),
|
|
);
|
|
}
|
|
|
|
export function historyLine(report) {
|
|
return {
|
|
commit: report.commit,
|
|
dirty: report.dirty,
|
|
startedAt: report.startedAt,
|
|
finishedAt: report.finishedAt,
|
|
seconds:
|
|
(Date.parse(report.finishedAt) - Date.parse(report.startedAt)) / 1000,
|
|
counts: statusCounts(report.scenarios),
|
|
// Keep the denominator correct for --only runs too.
|
|
scenarios: report.scenarios.map((row) => row.label),
|
|
nonPass: report.scenarios
|
|
.filter((row) => row.status !== 'pass')
|
|
.map(({ label, status }) => ({ label, status })),
|
|
};
|
|
}
|
|
|
|
export function flakeReport(history) {
|
|
const rows = new Map();
|
|
for (const run of history) {
|
|
const nonPass = new Set(run.nonPass.map((row) => row.label));
|
|
for (const label of run.scenarios) {
|
|
const row = rows.get(label) ?? { label, runs: 0, passes: 0 };
|
|
row.runs++;
|
|
if (!nonPass.has(label)) row.passes++;
|
|
rows.set(label, row);
|
|
}
|
|
}
|
|
const share = (row) => ({
|
|
...row,
|
|
nonPassShare: row.runs ? (row.runs - row.passes) / row.runs : 0,
|
|
});
|
|
const scenarios = [...rows.values()]
|
|
.map(share)
|
|
.sort(
|
|
(a, b) =>
|
|
b.nonPassShare - a.nonPassShare || a.label.localeCompare(b.label),
|
|
);
|
|
const overall = share(
|
|
scenarios.reduce(
|
|
(total, row) => ({
|
|
runs: total.runs + row.runs,
|
|
passes: total.passes + row.passes,
|
|
}),
|
|
{ runs: 0, passes: 0 },
|
|
),
|
|
);
|
|
return { scenarios, overall };
|
|
}
|
|
|
|
export function formatFlakeReport(history) {
|
|
const { scenarios, overall } = flakeReport(history);
|
|
return [
|
|
'scenario\truns\tpasses\tnon-pass',
|
|
...[...scenarios, { label: 'overall', ...overall }].map(
|
|
(row) =>
|
|
`${row.label}\t${row.runs}\t${row.passes}\t${(row.nonPassShare * 100).toFixed(1)}%`,
|
|
),
|
|
].join('\n');
|
|
}
|