Merge pull request #1133 from asturksever/fix/qa-cctv-v2-headful

fix(qa-cctv-v2): keep long headful waits under puppeteer's protocolTimeout

scripts/qa-cctv-v2.mjs

@@ -139,6 +139,56 @@ function record(name, ok, detail) {
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
+// Each page.evaluate is bounded by puppeteer's `protocolTimeout` (default
+// 180s). Waits never come close: they poll from Node (below). The headroom is
+// for the few long evaluates, such as enabling the full ~3,700-camera catalog
+// on a real GPU, and matches the other qa-*.mjs harnesses.
+const PROTOCOL_TIMEOUT_MS = 300_000;
+
+/**
+ * Polls an in-page predicate from Node, one short page.evaluate per tick,
+ * until it holds or the budget runs out. Used instead of page.waitForFunction
+ * for two reasons: its default rAF polling stops in a minimised or hidden
+ * headful window (timers keep running, rAF does not), and its result is one
+ * CDP call that protocolTimeout cuts off, while the headful drain budget
+ * reaches ~50 minutes. A predicate that throws fails the run instead of
+ * silently spending the budget.
+ * @param {import('puppeteer').Page} page
+ * @param {Function} predicate - In-page predicate.
+ * @param {object} opts
+ * @param {number} opts.timeoutMs - Overall budget.
+ * @param {number} [opts.intervalMs=250] - Delay between checks.
+ * @param {unknown[]} [opts.args=[]] - Arguments passed to the predicate.
+ *   The predicate must return a boolean: its result crosses CDP by value.
+ * @returns {Promise<boolean>} True if the predicate held within the budget.
+ */
+async function waitForPredicate(
+  page,
+  predicate,
+  { timeoutMs, intervalMs = 250, args = [] },
+) {
+  // performance.now() is monotonic and doesn't advance while the machine
+  // sleeps, so neither a clock change nor a laptop sleep eats the budget.
+  const deadline = performance.now() + timeoutMs;
+  for (;;) {
+    try {
+      if (await page.evaluate(predicate, ...args)) return true;
+    } catch (err) {
+      // A navigation or Vite reload replaces the execution context: poll again.
+      if (
+        !/Execution context was destroyed|Cannot find context/.test(
+          err?.message ?? '',
+        )
+      ) {
+        throw err;
+      }
+    }
+    const remaining = deadline - performance.now();
+    if (remaining <= 0) return false;
+    await sleep(Math.min(intervalMs, remaining));
+  }
+}
+
 // SwiftShader can spend tens of seconds servicing the hover drillPick emitted
 // by page.mouse.move(), before the LEFT_DOWN under test even runs. Position the
 // pointer while ADJUST is disabled, then re-enable it and explicitly render a
@@ -155,29 +205,58 @@ async function positionPointerForGizmoDrag(page, point, frameCount = 3) {
     gev.dataManager.layers
       .get('cctv')
       .module.setParams({ calibrationMode: true });
-    const scene = gev.viewer.scene;
+    return (await window.__qaRenderFrames(count, 5000)) >= count;
-    let rendered = 0;
+  }, frameCount);
-    return new Promise((resolve) => {
+}
-      let settled = false;
+
-      const finish = (complete) => {
+/**
-        if (settled) return;
+ * Saves a screenshot for human review. These are never a gate, so a capture
-        settled = true;
+ * that fails (for example when the renderer stalls in a hidden headful window)
-        clearTimeout(timer);
+ * is logged instead of ending the run.
-        remove();
+ * @param {import('puppeteer').Page} page
-        resolve(complete);
+ * @param {string} file
-      };
+ * @param {string} label
-      const remove = scene.postRender.addEventListener(() => {
+ */
-        rendered += 1;
+async function saveReviewShot(page, file, label) {
-        if (rendered >= count) {
+  try {
-          finish(true);
+    await page.screenshot({ path: file });
-          return;
+    console.log(`  screenshot (${label}) → ${path.relative(REPO_ROOT, file)}`);
-        }
+  } catch (err) {
+    console.warn(
+      `  screenshot skipped (${label}): ${path.relative(REPO_ROOT, file)}: ${err?.message ?? err}`,
+    );
+  }
+}
+
+/**
+ * Installs window.__qaRenderFrames(count, capMs) in the page: requests renders
+ * until `count` frames have passed scene.postRender or capMs runs out, and
+ * resolves with the number of frames rendered. The cap keeps a stalled
+ * renderer (headful window asleep or hidden) from hanging the CDP call.
+ * @param {import('puppeteer').Page} page
+ */
+function installRenderFramesHelper(page) {
+  return page.evaluate(() => {
+    window.__qaRenderFrames = (count, capMs) =>
+      new Promise((resolve) => {
+        const scene = window.__godsEyeView.viewer.scene;
+        let frames = 0;
+        const finish = () => {
+          clearTimeout(timer);
+          remove();
+          resolve(frames);
+        };
+        const remove = scene.postRender.addEventListener(() => {
+          if (++frames >= count) {
+            finish();
+            return;
+          }
+          scene.requestRender();
+        });
+        const timer = setTimeout(finish, capMs);
         scene.requestRender();
       });
-      const timer = setTimeout(() => finish(false), 5000);
+  });
-      scene.requestRender();
-    });
-  }, frameCount);
 }
 
 // In-page helper (stringified and re-defined inside each evaluate that needs
@@ -280,23 +359,21 @@ const BEARING_SHIFT_SRC = `
  * @returns {Promise<boolean>} True if tilesLoaded was observed within the timeout.
  */
 function waitForTilesLoaded(page, timeoutMs = 15000) {
-  return page
+  return waitForPredicate(
-    .waitForFunction(
+    page,
-      () => {
+    () => {
-        const scene = window.__godsEyeView.viewer.scene;
+      const scene = window.__godsEyeView.viewer.scene;
-        const prims = scene.primitives;
+      const prims = scene.primitives;
-        for (let i = 0; i < prims.length; i++) {
+      for (let i = 0; i < prims.length; i++) {
-          const p = prims.get(i);
+        const p = prims.get(i);
-          if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') {
+        if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') {
-            return p.tilesLoaded === true;
+          return p.tilesLoaded === true;
-          }
         }
-        return true; // no tileset in the scene — nothing to wait for
+      }
-      },
+      return true; // no tileset in the scene — nothing to wait for
-      { timeout: timeoutMs },
+    },
-    )
+    { timeoutMs },
-    .then(() => true)
+  );
-    .catch(() => false);
 }
 
 /**
@@ -364,6 +441,10 @@ async function main() {
   const chromeExecutable = findChromeExecutable();
   const browser = await puppeteer.launch({
     headless: HEADFUL ? false : 'new',
+    protocolTimeout: PROTOCOL_TIMEOUT_MS,
+    // Talk CDP over a pipe, so no debugging port stays open on localhost for
+    // the length of a headful run.
+    pipe: true,
     ...(chromeExecutable ? { executablePath: chromeExecutable } : {}),
     args: [
       '--no-sandbox',
@@ -397,13 +478,18 @@ async function main() {
     });
     console.log('Loading app...');
     await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
-    await page.waitForFunction(
+    const appReady = await waitForPredicate(
+      page,
       () =>
-        window.__godsEyeView &&
+        Boolean(
-        window.__godsEyeView.viewer &&
+          window.__godsEyeView &&
-        window.__godsEyeView.dataManager,
+          window.__godsEyeView.viewer &&
-      { timeout: 60000 },
+          window.__godsEyeView.dataManager,
+        ),
+      { timeoutMs: 60000 },
     );
+    if (!appReady) throw new Error('app did not initialise within 60s');
+    await installRenderFramesHelper(page);
     // Let the initial fly-to Austin and first tiles settle.
     await sleep(4000);
 
@@ -475,18 +561,15 @@ async function main() {
     console.log(
       `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`,
     );
-    const drained = await page
+    const drained = await waitForPredicate(
-      .waitForFunction(
+      page,
-        () => {
+      () => {
-          const mod =
+        const mod = window.__godsEyeView.dataManager.layers.get('cctv').module;
-            window.__godsEyeView.dataManager.layers.get('cctv').module;
+        const ui = mod.getUIState();
-          const ui = mod.getUIState();
+        return Boolean(ui.loading) && ui.loading.active === false;
-          return ui.loading && ui.loading.active === false;
+      },
-        },
+      { timeoutMs: drainBudgetMs, intervalMs: 1000 },
-        { timeout: drainBudgetMs },
+    );
-      )
-      .then(() => true)
-      .catch(() => false);
     record(
       `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`,
       drained ? true : HEADFUL ? false : null,
@@ -579,14 +662,11 @@ async function main() {
     const tilesSeenLoaded = await waitForTilesLoaded(page, 45000);
     let sampleFloorOk = false;
     if (tilesSeenLoaded) {
-      sampleFloorOk = await page
+      sampleFloorOk = await waitForPredicate(
-        .waitForFunction(
+        page,
-          (base) => window.__qaCounters.sampleHeight - base >= 1,
+        (base) => window.__qaCounters.sampleHeight - base >= 1,
-          { timeout: 30000 },
+        { timeoutMs: 30000, args: [c0.sampleHeight] },
-          c0.sampleHeight,
+      );
-        )
-        .then(() => true)
-        .catch(() => false);
     }
     const cFloor = await readCounters();
     record(
@@ -1358,18 +1438,7 @@ async function main() {
         },
       });
       window.__qaCctvSiblingOwner = owner;
-      await new Promise((resolve) => {
+      const framesRendered = await window.__qaRenderFrames(3, 10000);
-        let frames = 0;
-        const remove = scene.postRender.addEventListener(() => {
-          if (++frames < 3) {
-            scene.requestRender();
-            return;
-          }
-          remove();
-          resolve();
-        });
-        scene.requestRender();
-      });
       const rect = scene.canvas.getBoundingClientRect();
       const picked = scene.pick(canvasPoint);
       const pickedId = picked?.id?.id ?? picked?.id ?? null;
@@ -1382,26 +1451,33 @@ async function main() {
           typeof pickedId === 'string' ? pickedId : (pickedId?.id ?? null),
         ownerId: owner.id,
         ownsExactObject: picked?.id === owner,
+        framesRendered,
         projectionDelta: Math.hypot(
           projected.x - canvasPoint.x,
           projected.y - canvasPoint.y,
         ),
       };
     }, emptyClickPoint);
+    const siblingOwnsPick =
+      siblingTarget?.pickedId === 'qa-cctv-sibling-owner' &&
+      siblingTarget.ownsExactObject;
+    // No frame at all within the 10s cap points at a stalled renderer rather
+    // than at picking, though the harness can't rule out both. Fail it on the
+    // real-GPU sign-off path; leave it inconclusive under headless GL. A slow
+    // SwiftShader frame (1-2 of 3) still counts as a real miss.
+    const siblingRendererStalled =
+      siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered === 0;
     record(
       'sibling test object owns its canvas pick',
+      !siblingTarget
+        ? null
+        : siblingOwnsPick ||
+            (siblingRendererStalled && !HEADFUL ? null : false),
       siblingTarget
-        ? siblingTarget.pickedId === 'qa-cctv-sibling-owner' &&
+        ? `${siblingRendererStalled ? 'renderer stalled: ' : ''}frames=${siblingTarget.framesRendered}/3 picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}`
-            siblingTarget.ownsExactObject
-        : null,
-      siblingTarget
-        ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}`
         : 'no previously verified empty canvas ray was available',
     );
-    if (
+    if (siblingOwnsPick) {
-      siblingTarget?.pickedId === 'qa-cctv-sibling-owner' &&
-      siblingTarget.ownsExactObject
-    ) {
       await page.mouse.click(siblingTarget.x, siblingTarget.y);
       await sleep(250);
     }
@@ -1426,8 +1502,7 @@ async function main() {
     });
     record(
       'sibling canvas click passes through without CCTV selection or deselection',
-      siblingTarget?.pickedId === 'qa-cctv-sibling-owner' &&
+      siblingOwnsPick
-        siblingTarget.ownsExactObject
         ? siblingClick.activeId === clickEvidenceSetup.activeId &&
             siblingClick.transitions === siblingClick.baselineTransitions &&
             siblingClick.focusEvents === 0 &&
@@ -1626,10 +1701,7 @@ async function main() {
     );
 
     const viewshedShot = path.join(SHOTS_DIR, 'cctv-v3-viewshed.png');
-    await page.screenshot({ path: viewshedShot });
+    await saveReviewShot(page, viewshedShot, 'visual review only');
-    console.log(
-      `  screenshot (visual review only) → ${path.relative(REPO_ROOT, viewshedShot)}`,
-    );
 
     // =========================================================================
     // Group 7: calibration gizmo (v3 design §3c — ADJUST mode + drag)
@@ -1963,10 +2035,7 @@ async function main() {
     // -----------------------------------------------------------------------
     await sleep(1000);
     const shot = path.join(SHOTS_DIR, 'cctv-v2-active.png');
-    await page.screenshot({ path: shot });
+    await saveReviewShot(page, shot, 'visual review only, not a gate');
-    console.log(
-      `  screenshot (visual review only, not a gate) → ${path.relative(REPO_ROOT, shot)}`,
-    );
 
     // No console errors during the whole exercise.
     record(