Run the layer acceptance under a default-deny network policy with per-layer assertion manifests and strict switch contracts

Every scenario runs through scripts/qa-strict-harness.mjs: each page, frame and worker gets an ordered block list before it runs (paid hosts and routes first, then only the preview, declared fixtures, keyless probes and cached free tile hosts), switches with real input on the picker, and records its assertions per layer and engine. The runner reads the catalog from the live page, fails any layer without an assertion on either engine, can build and serve a keyless checkout itself, and adds rapid, hidden-tab, context-loss, failure-injection, cold-chunk and soak scenarios. The main chunk gets a 1% size budget.

scripts/qa-strict-soak.mjs

@@ -0,0 +1,378 @@
+// Soak and switch timing under a mixed workload: radio playing (750
+// stations), earthquakes, 10 000 flights, 200 military aircraft, wind with
+// its GPU particles, and the traffic simulation, all from fixtures.
+//
+// Each run starts a fresh browser on one engine and switches with real
+// input on the picker. Run r starts on Navara when r is odd and on Cesium
+// when it is even, so the first switch of each run is a cold one in either
+// direction.
+//
+// "First frame" of a switch: the first postRender of the incoming engine
+// whose picture, read back from its canvas in that callback, is not blank,
+// then the next animation frame (presented). Timed in the page from the
+// picker's change event, so no CDP round trip is included.
+//
+// Checked on every switch: the same session (manager and every layer
+// model), the radio still playing the same station with one play() call.
+// With --assert (the long run), after the warm-up (switch 20): live WebGL
+// objects per engine context grow by at most 0.1 per switch (least-squares
+// slope over the checkpoints) and by at most 64 overall; no new WebGL
+// context; JS heap slope at most 10 KiB per switch; no new document; the
+// webglcontextlost/restored listener counts on both engine canvases stay as
+// they were (each canvas keeps its webglcontextlost listener).
+//
+//   node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50
+//   node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert
+
+import assert from 'node:assert/strict';
+import {
+  createStrict,
+  localPreview,
+  waitForEngine,
+} from './qa-strict-harness.mjs';
+import {
+  installFirstFrameProbe,
+  installSessionFixtures,
+  installWebGlCounters,
+  openApp,
+  quantile,
+  rememberEngine,
+  slope,
+  startSession,
+} from './qa-strict-session.mjs';
+import { installRadioFixture, stations } from './fixtures/w2/radio.mjs';
+import { trafficFixtureResponse } from './traffic-fixtures.mjs';
+
+const base = localPreview(process.argv[2]);
+const option = (name, fallback) => {
+  const arg = process.argv.find((a) => a.startsWith(`--${name}=`));
+  return arg ? Number(arg.slice(name.length + 3)) : fallback;
+};
+const runs = option('runs', 6);
+const swaps = option('swaps', 50);
+const strictMode = process.argv.includes('--assert');
+assert.ok(
+  Number.isInteger(swaps) && swaps >= 4 && swaps % 2 === 0,
+  'even --swaps',
+);
+const WARM_UP = 20;
+const CHECKPOINTS = [
+  2, 10, 20, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500,
+];
+const HASH =
+  'v=2&lat=30.2670&lon=-97.7440&alt=3000&heading=0&pitch=-90&map=osm&l=';
+const LAYERS = [
+  'earthquakes',
+  'flights',
+  'military',
+  'wind',
+  'traffic',
+  'radio',
+];
+
+const strict = createStrict({
+  origin: base.origin,
+  layers: [],
+  // Road tiles of the traffic simulation, from scripts/traffic-fixtures.mjs.
+  fixtures: ['https://tiles.openfreemap.org/*'],
+});
+
+/** Wind and traffic fixtures on their own CDP Fetch patterns. */
+async function installWorkloadFixtures(page) {
+  const session = await page.createCDPSession();
+  const grid = (u) => {
+    const kind = u.searchParams.get('overlay');
+    const count = 36 * 19;
+    const values = new Float32Array(count * (kind ? 3 : 2));
+    values.fill(8, 0, count);
+    values.fill(6, count, count * 2);
+    if (kind) values.fill(40, count * 2);
+    return Buffer.from(values.buffer);
+  };
+  session.on('Fetch.requestPaused', async ({ requestId, request }) => {
+    const u = new URL(request.url);
+    let answer;
+    try {
+      if (u.pathname.startsWith('/api/wind/')) {
+        const model = u.searchParams.get('model');
+        answer = u.pathname.endsWith('/manifest')
+          ? {
+              status: 200,
+              contentType: 'application/json',
+              body: JSON.stringify({
+                model,
+                overlay: 'none',
+                grid: { nx: 36, ny: 19, lo1: 0, la1: 90, dx: 10, dy: 10 },
+                gridUrl: '/api/wind/grid/' + model + '-qa.bin' + u.search,
+                cycle: {
+                  validIso: '2026-11-02T12:00:00Z',
+                  runIso: '2026-11-02T06:00:00Z',
+                },
+              }),
+            }
+          : {
+              status: 200,
+              contentType: 'application/octet-stream',
+              body: grid(u),
+            };
+      } else if (u.pathname === '/api/tomtom/status')
+        answer = {
+          status: 200,
+          contentType: 'application/json',
+          body: '{"hasKey":false}',
+        };
+      else answer = trafficFixtureResponse({ url: () => request.url });
+      await session.send('Fetch.fulfillRequest', {
+        requestId,
+        responseCode: answer.status,
+        responseHeaders: [
+          { name: 'Content-Type', value: answer.contentType },
+          { name: 'Access-Control-Allow-Origin', value: '*' },
+        ],
+        body: Buffer.from(answer.body).toString('base64'),
+      });
+    } catch {
+      await session
+        .send('Fetch.failRequest', { requestId, errorReason: 'Failed' })
+        .catch(() => {});
+    }
+  });
+  await session.send('Fetch.enable', {
+    patterns: [
+      { urlPattern: `${base.origin}/api/wind/*` },
+      { urlPattern: `${base.origin}/api/tomtom/*` },
+      { urlPattern: 'https://tiles.openfreemap.org/*' },
+    ],
+  });
+}
+
+const report = { runs: [], swaps, strict: strictMode };
+strict.extra.soak = report;
+
+for (let run = 1; run <= runs; run++) {
+  const start = run % 2 ? 'navara' : 'cesium';
+  const browser = await strict.launch({
+    args: ['--use-angle=metal', '--enable-gpu'],
+  });
+  const row = { run, start, swaps: [], checkpoints: [], errors: [] };
+  report.runs.push(row);
+  try {
+    const p = await browser.newPage();
+    await p.setViewport({ width: 1280, height: 800 });
+    p.on('pageerror', (e) => row.errors.push(e.message.slice(0, 300)));
+    await installRadioFixture(p);
+    await installSessionFixtures(p, { flights: 10000, military: 200 });
+    await installWebGlCounters(p);
+    await installFirstFrameProbe(p);
+    await installWorkloadFixtures(p);
+    const cdp = await p.createCDPSession();
+    await openApp(p, base.origin, start, HASH);
+    await startSession(p, LAYERS);
+    await p.waitForFunction(
+      () => {
+        const layers = window.__godsEyeView.dataManager.layers;
+        const stat = (id) => layers.get(id).module.getStats();
+        return (
+          stat('radio').count === 750 &&
+          stat('flights').count > 0 &&
+          stat('military').count > 0 &&
+          stat('traffic').count >= 10 &&
+          layers.get('wind').module.getDiagnostics().gpu?.ready
+        );
+      },
+      { timeout: 90000, polling: 500 },
+    );
+    await p.evaluate((id) => {
+      const radio = window.__godsEyeView.dataManager.layers.get('radio').module;
+      radio.selectStation(id, { autoplay: true, focus: false, origin: 'user' });
+      window.__qaRadio.station = id;
+    }, stations[10].id);
+    await p.waitForFunction(
+      () =>
+        window.__qaRadio.plays.length === 1 &&
+        window.__godsEyeView.dataManager.layers.get('radio').module.getUIState()
+          .audioState === 'playing',
+    );
+    row.workload = await p.evaluate(() => {
+      const layers = window.__godsEyeView.dataManager.layers;
+      return Object.fromEntries(
+        ['flights', 'military', 'traffic', 'radio', 'earthquakes'].map((id) => [
+          id,
+          layers.get(id).module.getStats().count,
+        ]),
+      );
+    });
+    const listeners = async () => {
+      const { result } = await cdp.send('Runtime.evaluate', {
+        includeCommandLineAPI: true,
+        returnByValue: true,
+        expression: `(() => Object.fromEntries([...window.__qaEngines].map(([kind, engine]) => {
+          const events = getEventListeners(engine.host.canvas);
+          return [kind, [events.webglcontextlost?.length ?? 0, events.webglcontextrestored?.length ?? 0]];
+        })))()`,
+      });
+      return result.value;
+    };
+    const seen = new Set([start]);
+    let kind = start;
+    for (let i = 1; i <= swaps; i++) {
+      kind = kind === 'navara' ? 'cesium' : 'navara';
+      const cold = !seen.has(kind);
+      seen.add(kind);
+      await p.evaluate((kind) => {
+        window.__qaFrame = window.__qaFirstFrame(kind);
+      }, kind);
+      const pick = await strict.pick(p, kind);
+      await waitForEngine(p, kind);
+      const frame = await p.evaluate(() => window.__qaFrame);
+      await rememberEngine(p);
+      const state = await p.evaluate(() => {
+        const { dataManager } = window.__godsEyeView;
+        const session = window.__qaSession;
+        const radio = dataManager.layers.get('radio').module;
+        return {
+          session:
+            dataManager === session.manager &&
+            [...session.models].every(
+              ([id, model]) => dataManager.layers.get(id)?.module === model,
+            ),
+          radio:
+            window.__qaRadio.plays.length === 1 &&
+            radio.getUIState().audioState === 'playing' &&
+            radio.getUIState().selected?.id === window.__qaRadio.station,
+        };
+      });
+      assert.ok(
+        state.session && state.radio,
+        `swap ${i}: ${JSON.stringify(state)}`,
+      );
+      row.swaps.push({
+        i,
+        to: kind,
+        cold,
+        firstFrameMs: Math.round((frame.presented - pick.at) * 10) / 10,
+        drawnMs: Math.round((frame.drawn - pick.at) * 10) / 10,
+        settledMs:
+          pick.idleAt == null
+            ? null
+            : Math.round((pick.idleAt - pick.at) * 10) / 10,
+      });
+      if (CHECKPOINTS.includes(i) || i === swaps) {
+        await cdp.send('HeapProfiler.collectGarbage');
+        const heap = await cdp.send('Runtime.getHeapUsage');
+        const dom = await cdp.send('Memory.getDOMCounters');
+        row.checkpoints.push({
+          swap: i,
+          engine: kind,
+          heapMiB: Math.round((heap.usedSize / 1048576) * 100) / 100,
+          ...dom,
+          listeners: await listeners(),
+          webgl: await p.evaluate(() => window.__qaGlCounts()),
+        });
+      }
+    }
+  } finally {
+    await browser.close().catch(() => {});
+  }
+  const warm = (to) =>
+    row.swaps.filter((s) => !s.cold && s.to === to).map((s) => s.firstFrameMs);
+  row.summary = Object.fromEntries(
+    ['navara', 'cesium'].map((to) => [
+      to,
+      {
+        cold:
+          row.swaps.find((s) => s.cold && s.to === to)?.firstFrameMs ?? null,
+        warmP50: quantile(warm(to), 0.5),
+        warmP95: quantile(warm(to), 0.95),
+      },
+    ]),
+  );
+  console.log(
+    'RUN',
+    JSON.stringify({ run, start, ...row.summary, workload: row.workload }),
+  );
+}
+
+// Pooled over the runs, per direction.
+const pooled = (to, cold) =>
+  report.runs.flatMap((r) =>
+    r.swaps
+      .filter((s) => s.to === to && s.cold === cold)
+      .map((s) => s.firstFrameMs),
+  );
+report.summary = Object.fromEntries(
+  ['navara', 'cesium'].map((to) => [
+    to === 'navara' ? 'cesium→navara' : 'navara→cesium',
+    {
+      cold: pooled(to, true),
+      warmP50: quantile(pooled(to, false), 0.5),
+      warmP95: quantile(pooled(to, false), 0.95),
+      warmSwitches: pooled(to, false).length,
+    },
+  ]),
+);
+console.log('SOAK', JSON.stringify(report.summary));
+strict.timings.push(
+  ...report.runs.flatMap((r) => r.swaps.map((s) => ({ run: r.run, ...s }))),
+);
+
+if (strictMode) {
+  const checks = [];
+  for (const row of report.runs) {
+    const late = row.checkpoints.filter((c) => c.swap >= WARM_UP);
+    assert.ok(late.length >= 3, 'enough checkpoints after the warm-up');
+    const first = late[0];
+    const last = late.at(-1);
+    for (const owner of ['navara', 'cesium']) {
+      const totals = late.map((c) => [
+        c.swap,
+        c.webgl
+          .filter((g) => g.owner === owner && !g.lost)
+          .reduce((s, g) => s + g.total, 0),
+      ]);
+      checks.push({
+        check: `${owner} live WebGL objects`,
+        slopePerSwap: Math.round(slope(totals) * 1000) / 1000,
+        from: totals[0][1],
+        to: totals.at(-1)[1],
+        pass: slope(totals) <= 0.1 && totals.at(-1)[1] - totals[0][1] <= 64,
+      });
+    }
+    checks.push({
+      check: 'WebGL contexts',
+      from: first.webgl.length,
+      to: last.webgl.length,
+      pass: last.webgl.length === first.webgl.length,
+    });
+    const heap = late.map((c) => [c.swap, c.heapMiB * 1024]);
+    checks.push({
+      check: 'JS heap KiB per switch',
+      slopePerSwap: Math.round(slope(heap) * 100) / 100,
+      fromMiB: first.heapMiB,
+      toMiB: last.heapMiB,
+      pass: slope(heap) <= 10,
+    });
+    checks.push({
+      check: 'documents',
+      from: first.documents,
+      to: last.documents,
+      pass: last.documents <= first.documents,
+    });
+    checks.push({
+      check: 'context-loss listeners on both canvases',
+      from: first.listeners,
+      to: last.listeners,
+      pass:
+        JSON.stringify(last.listeners) === JSON.stringify(first.listeners) &&
+        Object.keys(last.listeners).length === 2 &&
+        Object.values(last.listeners).every(([lost]) => lost > 0),
+    });
+  }
+  report.checks = checks;
+  for (const check of checks)
+    console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check));
+  await strict.finish({
+    failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null,
+  });
+  if (checks.some((c) => !c.pass)) process.exitCode = 1;
+} else await strict.finish();