From 05f668dfa7c6aea521f0b38d60f8158006c33bb0 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 13:08:35 +0100 Subject: [PATCH 1/3] fix(qa-cctv-v2): keep long headful waits under puppeteer's protocolTimeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The headful drain budget is uncapped (camCount * 800 + 30000 ms, ~50 min at N≈3,700), but each page.waitForFunction is one CDP call that puppeteer aborts after its 180 s default protocolTimeout. The .catch then recorded "timed out waiting for drain on the real-GPU sign-off path" after ~3 min. - Poll the drain in <=120 s slices (waitForFunctionSliced) so no single CDP call outlives protocolTimeout, whatever the catalog size. - Launch with protocolTimeout 240 s, headroom above the largest slice and the 60 s fixed waits. - Bound the sibling-owner 3-frame postRender wait at 10 s and report the frame count, so a renderer that stops producing frames fails with evidence instead of hanging the evaluate. - List --disable-backgrounding-occluded-windows explicitly. Puppeteer 25.10 already passes it by default, so this documents the headful contract rather than changing behaviour. Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 76 +++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 08995603f..9234093be 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -139,6 +139,41 @@ function record(name, ok, detail) { const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// Every page.waitForFunction / page.evaluate is a single CDP call that +// puppeteer aborts after `protocolTimeout` (default 180s). The headful drain +// budget is uncapped and reaches ~50 minutes on the full ~3,900-camera +// catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and +// protocolTimeout is set with headroom above the largest slice. +const WAIT_SLICE_MS = 120000; +const PROTOCOL_TIMEOUT_MS = 2 * WAIT_SLICE_MS; + +/** + * page.waitForFunction with an overall budget that may exceed + * protocolTimeout: waits in WAIT_SLICE_MS slices until the predicate holds or + * the budget runs out. + * @param {import('puppeteer').Page} page + * @param {Function} predicate - In-page predicate. + * @param {number} budgetMs - Overall budget. + * @returns {Promise} True if the predicate held within the budget. + */ +async function waitForFunctionSliced(page, predicate, budgetMs) { + const deadline = Date.now() + budgetMs; + for (;;) { + const remaining = deadline - Date.now(); + if (remaining <= 0) return false; + const ok = await page + .waitForFunction(predicate, { + timeout: Math.min(remaining, WAIT_SLICE_MS), + }) + .then(() => true) + .catch((err) => { + if (err?.name === 'TimeoutError') return false; + throw err; + }); + if (ok) return true; + } +} + // SwiftShader can spend tens of seconds servicing the hover drillPick emitted // by page.mouse.move(), before the LEFT_DOWN under test even runs. Position the // pointer while ADJUST is disabled, then re-enable it and explicitly render a @@ -364,6 +399,7 @@ async function main() { const chromeExecutable = findChromeExecutable(); const browser = await puppeteer.launch({ headless: HEADFUL ? false : 'new', + protocolTimeout: PROTOCOL_TIMEOUT_MS, ...(chromeExecutable ? { executablePath: chromeExecutable } : {}), args: [ '--no-sandbox', @@ -373,6 +409,9 @@ async function main() { '--disable-web-security', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', + // Puppeteer's defaults already include this; listed so the headful + // run keeps rendering when covered even if those defaults change. + '--disable-backgrounding-occluded-windows', '--window-size=1280,800', ], }); @@ -475,18 +514,15 @@ async function main() { console.log( `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`, ); - const drained = await page - .waitForFunction( - () => { - const mod = - window.__godsEyeView.dataManager.layers.get('cctv').module; - const ui = mod.getUIState(); - return ui.loading && ui.loading.active === false; - }, - { timeout: drainBudgetMs }, - ) - .then(() => true) - .catch(() => false); + const drained = await waitForFunctionSliced( + page, + () => { + const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; + const ui = mod.getUIState(); + return ui.loading && ui.loading.active === false; + }, + drainBudgetMs, + ); record( `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`, drained ? true : HEADFUL ? false : null, @@ -1358,16 +1394,23 @@ async function main() { }, }); window.__qaCctvSiblingOwner = owner; - await new Promise((resolve) => { + // Bounded: if the browser stops producing frames (headful window + // asleep or hidden), report it instead of hanging the CDP call. + const framesRendered = await new Promise((resolve) => { let frames = 0; + const finish = () => { + clearTimeout(timer); + remove(); + resolve(frames); + }; const remove = scene.postRender.addEventListener(() => { if (++frames < 3) { scene.requestRender(); return; } - remove(); - resolve(); + finish(); }); + const timer = setTimeout(finish, 10000); scene.requestRender(); }); const rect = scene.canvas.getBoundingClientRect(); @@ -1382,6 +1425,7 @@ async function main() { typeof pickedId === 'string' ? pickedId : (pickedId?.id ?? null), ownerId: owner.id, ownsExactObject: picked?.id === owner, + framesRendered, projectionDelta: Math.hypot( projected.x - canvasPoint.x, projected.y - canvasPoint.y, @@ -1395,7 +1439,7 @@ async function main() { siblingTarget.ownsExactObject : null, siblingTarget - ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}` + ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` : 'no previously verified empty canvas ray was available', ); if ( From 474d44c4db534f0133a7189745d100b2b10e352c Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 16:57:06 +0100 Subject: [PATCH 2/3] fix(qa-cctv-v2): report a stalled renderer apart from a missed sibling pick When the sibling-owner pick misses after fewer than 3 rendered frames, the cause is a stalled renderer, not a pick regression. Record it as a failure on the headful sign-off path and as inconclusive under headless GL, with "renderer stalled: N/3 frames" in the detail. A missed pick after 3 frames still fails in both modes. Also corrects the catalog-size comment (~3,700). Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 9234093be..b291c54f4 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -141,7 +141,7 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); // Every page.waitForFunction / page.evaluate is a single CDP call that // puppeteer aborts after `protocolTimeout` (default 180s). The headful drain -// budget is uncapped and reaches ~50 minutes on the full ~3,900-camera +// budget is uncapped and reaches ~50 minutes on the full ~3,700-camera // catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and // protocolTimeout is set with headroom above the largest slice. const WAIT_SLICE_MS = 120000; @@ -1432,20 +1432,28 @@ async function main() { ), }; }, emptyClickPoint); + const siblingOwnsPick = + siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && + siblingTarget.ownsExactObject; + // A missed pick after fewer than 3 frames means the renderer stalled, not + // that picking regressed: fail it on the real-GPU sign-off path, leave it + // inconclusive under headless GL. + const siblingRendererStalled = + siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered < 3; record( 'sibling test object owns its canvas pick', + !siblingTarget + ? null + : siblingRendererStalled + ? HEADFUL + ? false + : null + : siblingOwnsPick, siblingTarget - ? siblingTarget.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - : null, - siblingTarget - ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` + ? `${siblingRendererStalled ? `renderer stalled: ${siblingTarget.framesRendered}/3 frames in 10s; ` : ''}picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` : 'no previously verified empty canvas ray was available', ); - if ( - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - ) { + if (siblingOwnsPick) { await page.mouse.click(siblingTarget.x, siblingTarget.y); await sleep(250); } @@ -1470,8 +1478,7 @@ async function main() { }); record( 'sibling canvas click passes through without CCTV selection or deselection', - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject + siblingOwnsPick ? siblingClick.activeId === clickEvidenceSetup.activeId && siblingClick.transitions === siblingClick.baselineTransitions && siblingClick.focusEvents === 0 && From 12b8fd0c1d25ad4d402ee7a4004f745dff7c730f Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 18:59:51 +0100 Subject: [PATCH 3/3] fix(qa-cctv-v2): poll waits from Node so a hidden window can't stall them page.waitForFunction polls on requestAnimationFrame by default. In a minimised or hidden headful window rAF stops while timers keep running, so each 120 s slice only checked the predicate when it started: a drain that finished in the last slice was recorded as a FAIL on the sign-off path. - waitForPredicate replaces waitForFunctionSliced and the other three waitForFunction calls: one short page.evaluate per tick (1 s for the drain, 250 ms elsewhere), a monotonic performance.now() deadline, retry across a context swap, and a throwing predicate fails the run instead of silently spending the budget. - One in-page window.__qaRenderFrames(count, capMs) helper serves both the gizmo-drag and the sibling-owner frame waits. - The sibling pick counts as a stalled renderer only at 0 of 3 frames, so a slow SwiftShader frame still reports a real miss; the record flattens and prints the frame count once. - Review screenshots log and continue when a capture fails. - Launch with pipe: true (no localhost CDP port) and protocolTimeout 300_000 like the other harnesses; drop the occlusion flag that puppeteer already passes by default. Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 252 ++++++++++++++++++++++------------------- 1 file changed, 135 insertions(+), 117 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index b291c54f4..2c121e0d6 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -139,38 +139,53 @@ function record(name, ok, detail) { const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -// Every page.waitForFunction / page.evaluate is a single CDP call that -// puppeteer aborts after `protocolTimeout` (default 180s). The headful drain -// budget is uncapped and reaches ~50 minutes on the full ~3,700-camera -// catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and -// protocolTimeout is set with headroom above the largest slice. -const WAIT_SLICE_MS = 120000; -const PROTOCOL_TIMEOUT_MS = 2 * WAIT_SLICE_MS; +// Each page.evaluate is bounded by puppeteer's `protocolTimeout` (default +// 180s). Waits never come close: they poll from Node (below). The headroom is +// for the few long evaluates, such as enabling the full ~3,700-camera catalog +// on a real GPU, and matches the other qa-*.mjs harnesses. +const PROTOCOL_TIMEOUT_MS = 300_000; /** - * page.waitForFunction with an overall budget that may exceed - * protocolTimeout: waits in WAIT_SLICE_MS slices until the predicate holds or - * the budget runs out. + * Polls an in-page predicate from Node, one short page.evaluate per tick, + * until it holds or the budget runs out. Used instead of page.waitForFunction + * for two reasons: its default rAF polling stops in a minimised or hidden + * headful window (timers keep running, rAF does not), and its result is one + * CDP call that protocolTimeout cuts off, while the headful drain budget + * reaches ~50 minutes. A predicate that throws fails the run instead of + * silently spending the budget. * @param {import('puppeteer').Page} page * @param {Function} predicate - In-page predicate. - * @param {number} budgetMs - Overall budget. + * @param {object} opts + * @param {number} opts.timeoutMs - Overall budget. + * @param {number} [opts.intervalMs=250] - Delay between checks. + * @param {unknown[]} [opts.args=[]] - Arguments passed to the predicate. + * The predicate must return a boolean: its result crosses CDP by value. * @returns {Promise} True if the predicate held within the budget. */ -async function waitForFunctionSliced(page, predicate, budgetMs) { - const deadline = Date.now() + budgetMs; +async function waitForPredicate( + page, + predicate, + { timeoutMs, intervalMs = 250, args = [] }, +) { + // performance.now() is monotonic and doesn't advance while the machine + // sleeps, so neither a clock change nor a laptop sleep eats the budget. + const deadline = performance.now() + timeoutMs; for (;;) { - const remaining = deadline - Date.now(); - if (remaining <= 0) return false; - const ok = await page - .waitForFunction(predicate, { - timeout: Math.min(remaining, WAIT_SLICE_MS), - }) - .then(() => true) - .catch((err) => { - if (err?.name === 'TimeoutError') return false; + try { + if (await page.evaluate(predicate, ...args)) return true; + } catch (err) { + // A navigation or Vite reload replaces the execution context: poll again. + if ( + !/Execution context was destroyed|Cannot find context/.test( + err?.message ?? '', + ) + ) { throw err; - }); - if (ok) return true; + } + } + const remaining = deadline - performance.now(); + if (remaining <= 0) return false; + await sleep(Math.min(intervalMs, remaining)); } } @@ -190,29 +205,58 @@ async function positionPointerForGizmoDrag(page, point, frameCount = 3) { gev.dataManager.layers .get('cctv') .module.setParams({ calibrationMode: true }); - const scene = gev.viewer.scene; - let rendered = 0; - return new Promise((resolve) => { - let settled = false; - const finish = (complete) => { - if (settled) return; - settled = true; - clearTimeout(timer); - remove(); - resolve(complete); - }; - const remove = scene.postRender.addEventListener(() => { - rendered += 1; - if (rendered >= count) { - finish(true); - return; - } + return (await window.__qaRenderFrames(count, 5000)) >= count; + }, frameCount); +} + +/** + * Saves a screenshot for human review. These are never a gate, so a capture + * that fails (for example when the renderer stalls in a hidden headful window) + * is logged instead of ending the run. + * @param {import('puppeteer').Page} page + * @param {string} file + * @param {string} label + */ +async function saveReviewShot(page, file, label) { + try { + await page.screenshot({ path: file }); + console.log(` screenshot (${label}) → ${path.relative(REPO_ROOT, file)}`); + } catch (err) { + console.warn( + ` screenshot skipped (${label}): ${path.relative(REPO_ROOT, file)}: ${err?.message ?? err}`, + ); + } +} + +/** + * Installs window.__qaRenderFrames(count, capMs) in the page: requests renders + * until `count` frames have passed scene.postRender or capMs runs out, and + * resolves with the number of frames rendered. The cap keeps a stalled + * renderer (headful window asleep or hidden) from hanging the CDP call. + * @param {import('puppeteer').Page} page + */ +function installRenderFramesHelper(page) { + return page.evaluate(() => { + window.__qaRenderFrames = (count, capMs) => + new Promise((resolve) => { + const scene = window.__godsEyeView.viewer.scene; + let frames = 0; + const finish = () => { + clearTimeout(timer); + remove(); + resolve(frames); + }; + const remove = scene.postRender.addEventListener(() => { + if (++frames >= count) { + finish(); + return; + } + scene.requestRender(); + }); + const timer = setTimeout(finish, capMs); scene.requestRender(); }); - const timer = setTimeout(() => finish(false), 5000); - scene.requestRender(); - }); - }, frameCount); + }); } // In-page helper (stringified and re-defined inside each evaluate that needs @@ -315,23 +359,21 @@ const BEARING_SHIFT_SRC = ` * @returns {Promise} True if tilesLoaded was observed within the timeout. */ function waitForTilesLoaded(page, timeoutMs = 15000) { - return page - .waitForFunction( - () => { - const scene = window.__godsEyeView.viewer.scene; - const prims = scene.primitives; - for (let i = 0; i < prims.length; i++) { - const p = prims.get(i); - if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { - return p.tilesLoaded === true; - } + return waitForPredicate( + page, + () => { + const scene = window.__godsEyeView.viewer.scene; + const prims = scene.primitives; + for (let i = 0; i < prims.length; i++) { + const p = prims.get(i); + if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { + return p.tilesLoaded === true; } - return true; // no tileset in the scene — nothing to wait for - }, - { timeout: timeoutMs }, - ) - .then(() => true) - .catch(() => false); + } + return true; // no tileset in the scene — nothing to wait for + }, + { timeoutMs }, + ); } /** @@ -400,6 +442,9 @@ async function main() { const browser = await puppeteer.launch({ headless: HEADFUL ? false : 'new', protocolTimeout: PROTOCOL_TIMEOUT_MS, + // Talk CDP over a pipe, so no debugging port stays open on localhost for + // the length of a headful run. + pipe: true, ...(chromeExecutable ? { executablePath: chromeExecutable } : {}), args: [ '--no-sandbox', @@ -409,9 +454,6 @@ async function main() { '--disable-web-security', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', - // Puppeteer's defaults already include this; listed so the headful - // run keeps rendering when covered even if those defaults change. - '--disable-backgrounding-occluded-windows', '--window-size=1280,800', ], }); @@ -436,13 +478,18 @@ async function main() { }); console.log('Loading app...'); await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); - await page.waitForFunction( + const appReady = await waitForPredicate( + page, () => - window.__godsEyeView && - window.__godsEyeView.viewer && - window.__godsEyeView.dataManager, - { timeout: 60000 }, + Boolean( + window.__godsEyeView && + window.__godsEyeView.viewer && + window.__godsEyeView.dataManager, + ), + { timeoutMs: 60000 }, ); + if (!appReady) throw new Error('app did not initialise within 60s'); + await installRenderFramesHelper(page); // Let the initial fly-to Austin and first tiles settle. await sleep(4000); @@ -514,14 +561,14 @@ async function main() { console.log( `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`, ); - const drained = await waitForFunctionSliced( + const drained = await waitForPredicate( page, () => { const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; const ui = mod.getUIState(); - return ui.loading && ui.loading.active === false; + return Boolean(ui.loading) && ui.loading.active === false; }, - drainBudgetMs, + { timeoutMs: drainBudgetMs, intervalMs: 1000 }, ); record( `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`, @@ -615,14 +662,11 @@ async function main() { const tilesSeenLoaded = await waitForTilesLoaded(page, 45000); let sampleFloorOk = false; if (tilesSeenLoaded) { - sampleFloorOk = await page - .waitForFunction( - (base) => window.__qaCounters.sampleHeight - base >= 1, - { timeout: 30000 }, - c0.sampleHeight, - ) - .then(() => true) - .catch(() => false); + sampleFloorOk = await waitForPredicate( + page, + (base) => window.__qaCounters.sampleHeight - base >= 1, + { timeoutMs: 30000, args: [c0.sampleHeight] }, + ); } const cFloor = await readCounters(); record( @@ -1394,25 +1438,7 @@ async function main() { }, }); window.__qaCctvSiblingOwner = owner; - // Bounded: if the browser stops producing frames (headful window - // asleep or hidden), report it instead of hanging the CDP call. - const framesRendered = await new Promise((resolve) => { - let frames = 0; - const finish = () => { - clearTimeout(timer); - remove(); - resolve(frames); - }; - const remove = scene.postRender.addEventListener(() => { - if (++frames < 3) { - scene.requestRender(); - return; - } - finish(); - }); - const timer = setTimeout(finish, 10000); - scene.requestRender(); - }); + const framesRendered = await window.__qaRenderFrames(3, 10000); const rect = scene.canvas.getBoundingClientRect(); const picked = scene.pick(canvasPoint); const pickedId = picked?.id?.id ?? picked?.id ?? null; @@ -1435,22 +1461,20 @@ async function main() { const siblingOwnsPick = siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && siblingTarget.ownsExactObject; - // A missed pick after fewer than 3 frames means the renderer stalled, not - // that picking regressed: fail it on the real-GPU sign-off path, leave it - // inconclusive under headless GL. + // No frame at all within the 10s cap points at a stalled renderer rather + // than at picking, though the harness can't rule out both. Fail it on the + // real-GPU sign-off path; leave it inconclusive under headless GL. A slow + // SwiftShader frame (1-2 of 3) still counts as a real miss. const siblingRendererStalled = - siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered < 3; + siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered === 0; record( 'sibling test object owns its canvas pick', !siblingTarget ? null - : siblingRendererStalled - ? HEADFUL - ? false - : null - : siblingOwnsPick, + : siblingOwnsPick || + (siblingRendererStalled && !HEADFUL ? null : false), siblingTarget - ? `${siblingRendererStalled ? `renderer stalled: ${siblingTarget.framesRendered}/3 frames in 10s; ` : ''}picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` + ? `${siblingRendererStalled ? 'renderer stalled: ' : ''}frames=${siblingTarget.framesRendered}/3 picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}` : 'no previously verified empty canvas ray was available', ); if (siblingOwnsPick) { @@ -1677,10 +1701,7 @@ async function main() { ); const viewshedShot = path.join(SHOTS_DIR, 'cctv-v3-viewshed.png'); - await page.screenshot({ path: viewshedShot }); - console.log( - ` screenshot (visual review only) → ${path.relative(REPO_ROOT, viewshedShot)}`, - ); + await saveReviewShot(page, viewshedShot, 'visual review only'); // ========================================================================= // Group 7: calibration gizmo (v3 design §3c — ADJUST mode + drag) @@ -2014,10 +2035,7 @@ async function main() { // ----------------------------------------------------------------------- await sleep(1000); const shot = path.join(SHOTS_DIR, 'cctv-v2-active.png'); - await page.screenshot({ path: shot }); - console.log( - ` screenshot (visual review only, not a gate) → ${path.relative(REPO_ROOT, shot)}`, - ); + await saveReviewShot(page, shot, 'visual review only, not a gate'); // No console errors during the whole exercise. record(