Assert the long soak's JS object growth from heap snapshots, and report compiled code, native memory and the used heap's slope apart

scripts/qa-strict-soak.mjs

@@ -18,7 +18,9 @@
 // below), then after the warm-up (switch 20): live WebGL
 // objects per engine context grow by at most 0.1 per switch (least-squares
 // slope over the checkpoints) and by at most 64 overall; no new WebGL
-// context; JS heap slope at most 10 KiB per switch; no new document; the
+// context; the JS objects grow by at most 10 KiB per switch (heap snapshots
+// at the warm-up and the end, V8's compiled code and native memory left out
+// and reported apart, with the used heap's slope); no new document; the
 // webglcontextlost/restored listener counts on both engine canvases stay as
 // they were (each canvas keeps its webglcontextlost listener).
 //
@@ -44,6 +46,7 @@ import {
 import { installRadioFixture, stations } from './fixtures/w2/radio.mjs';
 import { trafficFixtureResponse } from './traffic-fixtures.mjs';
 import { POSITION_HISTORY_LIMIT } from '../src/layers/flights/recordPolicy.js';
+import { heapSizesByType } from './qa/heapSnapshotSizes.mjs';
 
 const base = localPreview(process.argv[2]);
 const option = (name, fallback) => {
@@ -231,6 +234,8 @@ for (let run = 1; run <= runs; run++) {
     };
     const seen = new Set([start]);
     let kind = start;
+    // The switch after a heap snapshot is slower; timings leave it out.
+    let afterSnapshot = false;
     for (let i = 1; i <= swaps; i++) {
       kind = kind === 'navara' ? 'cesium' : 'navara';
       const cold = !seen.has(kind);
@@ -272,15 +277,34 @@ for (let run = 1; run <= runs; run++) {
           pick.idleAt == null
             ? null
             : Math.round((pick.idleAt - pick.at) * 10) / 10,
+        ...(afterSnapshot ? { afterSnapshot } : {}),
       });
+      afterSnapshot = false;
       if (CHECKPOINTS.includes(i) || i === swaps) {
         await cdp.send('HeapProfiler.collectGarbage');
         const heap = await cdp.send('Runtime.getHeapUsage');
+        // The heap by snapshot node type, at the first checkpoint after the
+        // warm-up and at the last: the asserted growth is the JS objects'
+        // (V8's compiled code grows with tier-ups over a long run; native
+        // memory, array buffer data included, is reported apart). The switch
+        // after a snapshot is left out of the timings.
+        let typesMiB = null;
+        if (strictMode && (i === WARM_UP || i === swaps)) {
+          const sizes = await heapSizesByType(cdp);
+          typesMiB = Object.fromEntries(
+            Object.entries(sizes).map(([type, bytes]) => [
+              type,
+              Math.round((bytes / 1048576) * 100) / 100,
+            ]),
+          );
+          afterSnapshot = true;
+        }
         const dom = await cdp.send('Memory.getDOMCounters');
         row.checkpoints.push({
           swap: i,
           engine: kind,
           heapMiB: Math.round((heap.usedSize / 1048576) * 100) / 100,
+          typesMiB,
           ...dom,
           listeners: await listeners(),
           webgl: await p.evaluate(() => window.__qaGlCounts()),
@@ -291,7 +315,9 @@ for (let run = 1; run <= runs; run++) {
     await browser.close().catch(() => {});
   }
   const warm = (to) =>
-    row.swaps.filter((s) => !s.cold && s.to === to).map((s) => s.firstFrameMs);
+    row.swaps
+      .filter((s) => !s.cold && !s.afterSnapshot && s.to === to)
+      .map((s) => s.firstFrameMs);
   row.summary = Object.fromEntries(
     ['navara', 'cesium'].map((to) => [
       to,
@@ -313,7 +339,7 @@ for (let run = 1; run <= runs; run++) {
 const pooled = (to, cold) =>
   report.runs.flatMap((r) =>
     r.swaps
-      .filter((s) => s.to === to && s.cold === cold)
+      .filter((s) => s.to === to && s.cold === cold && !s.afterSnapshot)
       .map((s) => s.firstFrameMs),
   );
 report.summary = Object.fromEntries(
@@ -361,12 +387,32 @@ if (strictMode) {
       pass: last.webgl.length === first.webgl.length,
     });
     const heap = late.map((c) => [c.swap, c.heapMiB * 1024]);
+    const typed = late.filter((c) => c.typesMiB);
+    assert.equal(typed.length, 2, 'heap snapshots at the warm-up and the end');
+    const [from, to] = typed;
+    const outside = new Set(['code', 'native', 'synthetic']);
+    const objects = (c) =>
+      Object.entries(c.typesMiB)
+        .filter(([type]) => !outside.has(type))
+        .reduce((sum, [, mib]) => sum + mib, 0);
+    const perSwitch = (mib) =>
+      Math.round(((mib * 1024) / (to.swap - from.swap)) * 100) / 100;
+    const objectGrowth = perSwitch(objects(to) - objects(from));
     checks.push({
       check: 'JS heap KiB per switch',
-      slopePerSwap: Math.round(slope(heap) * 100) / 100,
-      fromMiB: first.heapMiB,
-      toMiB: last.heapMiB,
-      pass: slope(heap) <= 10,
+      objectsKiBPerSwitch: objectGrowth,
+      fromMiB: Math.round(objects(from) * 100) / 100,
+      toMiB: Math.round(objects(to) * 100) / 100,
+      // Reported, not asserted: the used heap's slope (compiled code
+      // included), compiled code's and native memory's growth.
+      usedSlopePerSwap: Math.round(slope(heap) * 100) / 100,
+      codeKiBPerSwitch: perSwitch(
+        (to.typesMiB.code ?? 0) - (from.typesMiB.code ?? 0),
+      ),
+      nativeKiBPerSwitch: perSwitch(
+        (to.typesMiB.native ?? 0) - (from.typesMiB.native ?? 0),
+      ),
+      pass: objectGrowth <= 10,
     });
     checks.push({
       check: 'documents',

scripts/qa/heapSnapshotSizes.mjs

@@ -0,0 +1,96 @@
+// The live heap by V8 node type, read from a heap snapshot as it streams in:
+// only the nodes array is parsed, number by number, so a snapshot of a large
+// page is never held whole. The long soak asserts on the heap without the
+// `code` nodes (compiled code grows with V8's tier-ups over a long run, and
+// the browser drops it under memory pressure).
+
+/**
+ * A parser fed with a heap snapshot's JSON text in chunks. `result()` gives
+ * the self size summed per node type name.
+ */
+export function createHeapSnapshotSizes() {
+  let head = '';
+  let fields = null;
+  let typeOffset = -1;
+  let sizeOffset = -1;
+  let typeNames = null;
+  let inNodes = false;
+  let done = false;
+  let number = '';
+  let field = 0;
+  let type = 0;
+  const sizes = new Map();
+
+  function take(value) {
+    if (field === typeOffset) type = value;
+    if (field === sizeOffset) {
+      const name = typeNames[type] ?? String(type);
+      sizes.set(name, (sizes.get(name) ?? 0) + value);
+    }
+    field = (field + 1) % fields.length;
+  }
+
+  function nodes(text, from) {
+    for (let i = from; i < text.length; i++) {
+      const c = text.charCodeAt(i);
+      if (c >= 48 && c <= 57) number += text[i];
+      else {
+        if (number) {
+          take(Number(number));
+          number = '';
+        }
+        if (c === 93) {
+          done = true;
+          return;
+        }
+      }
+    }
+  }
+
+  return {
+    push(chunk) {
+      if (done) return;
+      if (inNodes) {
+        nodes(chunk, 0);
+        return;
+      }
+      head += chunk;
+      const start = head.indexOf('"nodes":[');
+      if (start < 0) return;
+      const meta = JSON.parse(
+        head.slice(head.indexOf('"meta":') + 7, head.indexOf(',"node_count"')),
+      );
+      fields = meta.node_fields;
+      typeOffset = fields.indexOf('type');
+      sizeOffset = fields.indexOf('self_size');
+      [typeNames] = meta.node_types;
+      inNodes = true;
+      const rest = head.slice(start + '"nodes":['.length);
+      head = '';
+      nodes(rest, 0);
+    },
+    result() {
+      return Object.fromEntries(sizes);
+    },
+  };
+}
+
+/** Take a heap snapshot over a CDP session and size it by node type. */
+export async function heapSizesByType(cdp) {
+  const sizes = createHeapSnapshotSizes();
+  const onChunk = ({ chunk }) => sizes.push(chunk);
+  cdp.on('HeapProfiler.addHeapSnapshotChunk', onChunk);
+  try {
+    await cdp.send('HeapProfiler.takeHeapSnapshot', {
+      reportProgress: false,
+      captureNumericValue: false,
+    });
+  } finally {
+    cdp.off('HeapProfiler.addHeapSnapshotChunk', onChunk);
+    // A snapshot leaves the profiler tracking object ids, which slows the
+    // page for the rest of the run (Navara → Cesium switches about 2.5 times
+    // slower in the long soak); it stops here.
+    await cdp.send('HeapProfiler.disable').catch(() => {});
+  }
+  return sizes.result();
+}

scripts/qa/heapSnapshotSizes.test.mjs

@@ -0,0 +1,24 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createHeapSnapshotSizes } from './heapSnapshotSizes.mjs';
+
+test('a streamed heap snapshot is sized per node type, however it is chunked', () => {
+  const snapshot = JSON.stringify({
+    snapshot: {
+      meta: {
+        node_fields: ['type', 'name', 'id', 'self_size', 'edge_count'],
+        node_types: [['hidden', 'object', 'code'], 'string', 'number'],
+      },
+      node_count: 4,
+    },
+    nodes: [1, 0, 1, 100, 0, 2, 0, 2, 4000, 0, 1, 0, 3, 23, 0, 0, 0, 4, 7, 0],
+    edges: [9, 9],
+    strings: ['x'],
+  });
+  for (const size of [snapshot.length, 7, 1]) {
+    const sizes = createHeapSnapshotSizes();
+    for (let i = 0; i < snapshot.length; i += size)
+      sizes.push(snapshot.slice(i, i + size));
+    assert.deepEqual(sizes.result(), { object: 123, code: 4000, hidden: 7 });
+  }
+});