Measure engine coupling, QA harness use and contract coverage

Counters that may only fall, each with a committed baseline and a test
that fails on growth:

- report-engine-coupling.mjs gains layer modules that take the viewer
  in attachRenderer (33), layer folders with their own "view being
  restored" flag (9), engine.surface calls outside src/globe (41) and
  layer ids named in the engine switch (3).
- report-qa-harness.mjs counts QA scripts that launch Chrome themselves
  (98), wait with fixed sleeps (117), take the hidden loading screen as
  readiness (9), read Cesium-only objects (75) or define their own
  waitFor helpers (19).
- withCallLog records every contract member the contract suite calls on
  the fake engine; contractCoverage.test.mjs fails when a member outside
  contract-uncalled-baseline.json goes uncalled, and when a listed one
  is called now, so the list of 12 only shrinks.

scripts/engine-coupling-baseline.json

@@ -1,5 +1,9 @@
 {
   "metrics": {
+    "attachRendererWithViewer": 33,
+    "restoreFlagCopies": 9,
+    "surfaceCallsOutsideGlobe": 41,
+    "engineSwitchLayerIds": 3,
     "cesiumLayerRendererLines": 0,
     "navaraLayerRendererLines": 0,
     "cesiumImportFiles": 3,
@@ -11,6 +15,5 @@
     "cesiumSdkLoads": 3,
     "screenSpaceHandlerFiles": 2,
     "engineNameLines": 21
-  },
-  "measuredAt": "Orbital cockpit and bridge mesh sampling"
+  }
 }

scripts/engineCoupling.test.mjs

@@ -39,3 +39,85 @@ test('a metric that grows is reported, one that falls is not', () => {
     ['viewerLines'],
   );
 });
+
+test('renderer attachment counts definitions once per module, not calls', () => {
+  const { measure } = ENGINE_COUPLING_METRICS.find(
+    ({ id }) => id === 'attachRendererWithViewer',
+  );
+  assert.equal(
+    measure([
+      {
+        text: 'attachRenderer(viewer, options = {}) {}\nattachRenderer(scene) {}',
+      },
+      { text: 'function attachRenderer(\n _viewer, options\n) {}' },
+      { text: 'const attachRenderer = (nextViewer) => {};' },
+      {
+        text: 'attachRenderer(_scene); layer.attachRenderer(viewer, options);',
+      },
+      { text: 'attachRenderer(engine) {}' },
+      { text: 'attachRenderer(viewerState) {}' },
+    ]),
+    3,
+  );
+});
+
+test('restore flags count layer directories and surface calls exclude globe', () => {
+  const measure = (id, files) =>
+    ENGINE_COUPLING_METRICS.find((metric) => metric.id === id).measure(files);
+  assert.equal(
+    measure('restoreFlagCopies', [
+      {
+        file: 'src/layers/traffic/lifecycle.js',
+        text: 'state._restoringViewport = true;',
+      },
+      {
+        file: 'src/layers/traffic/viewport.js',
+        text: 'state._restoringViewport',
+      },
+      { file: 'src/layers/alpr/index.js', text: 'let restoringView = false;' },
+      {
+        file: 'src/layers/streetLevel/index.js',
+        text: 'let waitingForView = false;',
+      },
+      { file: 'src/app/view.js', text: 'let restoringView = false;' },
+    ]),
+    3,
+  );
+  const text = [
+    'terrainHeight',
+    'sampleHeight',
+    'sampleHeightDetailed',
+    'sampleTerrainMostDetailed',
+    'clampToHeight',
+    'raycast',
+  ]
+    .map((name) => `engine.surface.${name}(point);`)
+    .join('\n');
+  assert.equal(
+    measure('surfaceCallsOutsideGlobe', [
+      { file: 'src/globe/navara/surface.js', text },
+      { file: 'src/layers/traffic/viewport.js', text },
+      {
+        file: 'src/app/view.js',
+        text: 'engine.surface?.raycast?.(ray); surface.tilesLoaded();',
+      },
+    ]),
+    7,
+  );
+});
+
+test('engine switch layer ids count lines and follow the catalog', () => {
+  const { measure } = ENGINE_COUPLING_METRICS.find(
+    ({ id }) => id === 'engineSwitchLayerIds',
+  );
+  assert.equal(
+    measure([
+      {
+        file: 'src/standalone/engineSwitch.js',
+        text: "get('directions'); get('satellites');\nget('traffic');\nget('not-a-layer');",
+      },
+      { file: 'src/app/view.js', text: "get('directions');" },
+    ]),
+    2,
+  );
+});

scripts/qa-harness-baseline.json

@@ -0,0 +1,9 @@
+{
+  "metrics": {
+    "ownLaunch": 98,
+    "fixedSleeps": 117,
+    "loadingScreenReadiness": 9,
+    "cesiumOnlyReads": 75,
+    "ownWaitHelpers": 19
+  }
+}

scripts/qaHarness.test.mjs

@@ -0,0 +1,100 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  QA_HARNESS_METRICS,
+  harnessGrowth,
+  measureQaHarness,
+  readBaseline,
+  readQaSources,
+} from './report-qa-harness.mjs';
+
+test('QA harness duplication does not grow past the baseline', () => {
+  assert.deepEqual(
+    harnessGrowth(measureQaHarness(), readBaseline().metrics),
+    [],
+    'QA harness duplication grew; use scripts/qa-strict-harness.mjs helpers for browser launch, engine readiness and camera waits instead.',
+  );
+});
+
+test('the baseline names every metric', () => {
+  assert.deepEqual(
+    Object.keys(readBaseline().metrics).sort(),
+    QA_HARNESS_METRICS.map(({ id }) => id).sort(),
+  );
+});
+
+test('a metric that grows is reported, one that falls is not', () => {
+  const baseline = Object.fromEntries(
+    QA_HARNESS_METRICS.map(({ id }) => [id, 10]),
+  );
+  assert.deepEqual(
+    harnessGrowth({ ...baseline, ownLaunch: 11, fixedSleeps: 3 }, baseline).map(
+      ({ id }) => id,
+    ),
+    ['ownLaunch'],
+  );
+  assert.equal(harnessGrowth(baseline, {}).length, QA_HARNESS_METRICS.length);
+});
+
+test('QA sources include the harness and exclude tests and other reports', () => {
+  const names = readQaSources().map(({ file }) => file);
+  assert.ok(names.includes('scripts/qa-strict-harness.mjs'));
+  assert.ok(!names.some((name) => name.endsWith('.test.mjs')));
+  assert.ok(!names.includes('scripts/report-qa-harness.mjs'));
+});
+
+test('metrics count scripts once and distinguish helpers from their callers', () => {
+  const measure = (id, ...sources) =>
+    QA_HARNESS_METRICS.find((metric) => metric.id === id).measure(
+      sources.map((text) => ({ text })),
+    );
+  assert.equal(
+    measure(
+      'ownLaunch',
+      'puppeteer.launch({}); puppeteer.launch({});',
+      'launchBrowser();',
+    ),
+    1,
+  );
+  assert.equal(
+    measure(
+      'fixedSleeps',
+      'const sleep = (ms) => new Promise((r) => setTimeout(r, ms));',
+      'await new Promise((resolve) => { setTimeout(resolve, 1000); });',
+      'await new Promise((resolve) => setTimeout(resolve));',
+      'new Promise((resolve) => requestAnimationFrame(resolve));',
+      'setTimeout(() => reject(new Error()), 1000);',
+    ),
+    3,
+  );
+  assert.equal(
+    measure(
+      'loadingScreenReadiness',
+      "page.waitForSelector('#loading-screen');",
+      "page.waitForSelector('#loading-screen'); await waitForEngine(page);",
+      "document.querySelector('#loading-screen');",
+    ),
+    1,
+  );
+  assert.equal(
+    measure(
+      'cesiumOnlyReads',
+      'window.__godsEyeView?.viewer; window.__godsEyeView.tileset;',
+      'const { scene } = window.__godsEyeView;',
+      'const app = window.__godsEyeView; app.viewer;',
+      'window.__godsEyeView.sceneDirector;',
+      'window.__godsEyeView.engine;',
+    ),
+    3,
+  );
+  assert.equal(
+    measure(
+      'ownWaitHelpers',
+      'async function waitForReady() {}',
+      'const waitForReady = async (page) => page.ready();',
+      'await waitForEngine(page);',
+      "import { waitForEngine } from './qa-strict-harness.mjs';",
+    ),
+    2,
+  );
+});

scripts/report-engine-coupling.mjs

@@ -16,6 +16,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { APPLICATION_LAYER_DIRECTORY } from '../src/app/layers/directory.js';
 
 const ROOT = fileURLToPath(new URL('..', import.meta.url));
 export const BASELINE_PATH = path.join(
@@ -37,12 +38,72 @@ const countLines = (text, pattern) =>
   lines(text).filter((line) => pattern.test(line)).length;
 const countMatches = (text, pattern) =>
   (text.match(new RegExp(pattern.source, 'g')) ?? []).length;
+const layerIds = new Set(APPLICATION_LAYER_DIRECTORY.map(({ id }) => id));
 
 /**
  * The metrics, in report order. `measure(files)` gets every production
  * source as `{ file, text }` (file relative to the repository, with `/`).
  */
 export const ENGINE_COUPLING_METRICS = Object.freeze([
+  {
+    id: 'attachRendererWithViewer',
+    label: 'Modules defining attachRenderer with a viewer or scene parameter',
+    measure: (files) =>
+      files.filter(({ text }) =>
+        /\battachRenderer\s*(?:(?:=|:)\s*(?:async\s+)?(?:function\s*)?)?\(\s*(?:viewer|_viewer|nextViewer|_scene|scene)\s*(?:,[^)]*|=[^)]*)?\)\s*(?:\{|=>)/.test(
+          text,
+        ),
+      ).length,
+  },
+  {
+    id: 'restoreFlagCopies',
+    label: 'Layer directories keeping their own view-restore flag',
+    measure: (files) =>
+      new Set(
+        files
+          .filter(
+            ({ file, text }) =>
+              file.startsWith('src/layers/') &&
+              /\b(?:_restoringViewport|restoringView|waitingForView)\b/.test(
+                text,
+              ),
+          )
+          .map(({ file }) => file.split('/')[2]),
+      ).size,
+  },
+  {
+    id: 'surfaceCallsOutsideGlobe',
+    label: 'Surface sampling calls outside src/globe',
+    measure: (files) =>
+      files
+        .filter(({ file }) => !inGlobe(file))
+        .reduce(
+          (sum, { text }) =>
+            sum +
+            countMatches(
+              text,
+              /\bsurface\??\.\s*(?:terrainHeight|sampleHeight|sampleHeightDetailed|sampleTerrainMostDetailed|clampToHeight|raycast)\s*(?:\?\.)?\(/,
+            ),
+          0,
+        ),
+  },
+  {
+    id: 'engineSwitchLayerIds',
+    label: 'Lines naming catalog layer ids in src/standalone/engineSwitch.js',
+    measure: (files) =>
+      files
+        .filter(({ file }) => file === 'src/standalone/engineSwitch.js')
+        .reduce(
+          (sum, { text }) =>
+            sum +
+            lines(text).filter((line) =>
+              [...line.matchAll(/(['"])([^'"\n]+)\1/g)].some((match) =>
+                layerIds.has(match[2]),
+              ),
+            ).length,
+          0,
+        ),
+  },
   {
     id: 'cesiumLayerRendererLines',
     label: 'Lines in src/layers/**/render/cesium*',
@@ -170,7 +231,11 @@ export function readProductionSources(root = ROOT) {
     for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
       const absolute = path.join(directory, entry.name);
       if (entry.isDirectory()) visit(absolute);
-      else if (entry.isFile() && /\.(?:js|vue)$/.test(entry.name))
+      else if (
+        entry.isFile() &&
+        /\.(?:js|vue)$/.test(entry.name) &&
+        !/\.test\./.test(entry.name)
+      )
         files.push({
           file: path.relative(root, absolute).split(path.sep).join('/'),
           text: fs.readFileSync(absolute, 'utf8'),

scripts/report-qa-harness.mjs

@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+/**
+ * Browser QA still doing work owned by scripts/qa-strict-harness.mjs.
+ * Each metric counts scripts, so repeated uses in one script count once.
+ *
+ *   node scripts/report-qa-harness.mjs
+ *   node scripts/report-qa-harness.mjs --json
+ *   node scripts/report-qa-harness.mjs --write-baseline
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const ROOT = fileURLToPath(new URL('..', import.meta.url));
+export const BASELINE_PATH = path.join(
+  ROOT,
+  'scripts',
+  'qa-harness-baseline.json',
+);
+
+const readsCesiumState = (text) => {
+  const names = [
+    '__godsEyeView',
+    ...[
+      ...text.matchAll(
+        /\b(?:const|let|var)\s+(\w+)\s*=\s*(?:window\.)?__godsEyeView\s*[;,]/g,
+      ),
+    ].map((match) => match[1]),
+  ];
+  return names.some((name) =>
+    new RegExp(
+      `\\b${name}\\s*\\??\\.\\s*(?:viewer|tileset|scene)\\b|\\{[^{}]*\\b(?:viewer|tileset|scene)\\b[^{}]*\\}\\s*=\\s*(?:window\\.)?${name}\\b`,
+    ).test(text),
+  );
+};
+
+export const QA_HARNESS_METRICS = Object.freeze([
+  {
+    id: 'ownLaunch',
+    label: 'Scripts launching puppeteer themselves',
+    measure: (files) =>
+      files.filter(({ text }) => /\bpuppeteer\.launch\s*\(/.test(text)).length,
+  },
+  {
+    id: 'fixedSleeps',
+    label: 'Scripts with setTimeout-based pauses',
+    measure: (files) =>
+      files.filter(({ text }) =>
+        /new\s+Promise\s*\(\s*\(?\s*(\w+)\s*\)?\s*=>\s*(?:\{\s*(?:return\s+)?)?setTimeout\s*\(\s*\1\s*[,)]/.test(
+          text,
+        ),
+      ).length,
+  },
+  {
+    id: 'loadingScreenReadiness',
+    label: 'Scripts waiting for #loading-screen without waitForEngine',
+    measure: (files) =>
+      files.filter(
+        ({ text }) =>
+          /\bwaitFor(?:Selector|Function)\s*\([^;]*#loading-screen\b/.test(
+            text,
+          ) && !/\bwaitForEngine\s*\(/.test(text),
+      ).length,
+  },
+  {
+    id: 'cesiumOnlyReads',
+    label: 'Scripts reading __godsEyeView viewer, tileset or scene',
+    measure: (files) =>
+      files.filter(({ text }) => readsCesiumState(text)).length,
+  },
+  {
+    id: 'ownWaitHelpers',
+    label: 'Scripts defining their own waitFor functions',
+    measure: (files) =>
+      files.filter(({ text }) =>
+        /\bfunction\s+waitFor\w*\s*\(|\b(?:const|let|var)\s+waitFor\w*\s*=\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*=>|\w+\s*=>)/.test(
+          text,
+        ),
+      ).length,
+  },
+]);
+
+/** QA scripts, including the shared harness itself, but excluding tests. */
+export function readQaSources(root = ROOT) {
+  return fs
+    .readdirSync(path.join(root, 'scripts'))
+    .filter((name) => /^qa-.*\.mjs$/.test(name) && !name.endsWith('.test.mjs'))
+    .sort()
+    .map((name) => ({
+      file: `scripts/${name}`,
+      text: fs.readFileSync(path.join(root, 'scripts', name), 'utf8'),
+    }));
+}
+
+export function measureQaHarness(root = ROOT) {
+  const files = readQaSources(root);
+  return Object.fromEntries(
+    QA_HARNESS_METRICS.map(({ id, measure }) => [id, measure(files)]),
+  );
+}
+
+export function harnessGrowth(measured, baseline) {
+  return QA_HARNESS_METRICS.filter(
+    ({ id }) => !(measured[id] <= (baseline[id] ?? -Infinity)),
+  ).map(({ id, label }) => ({
+    id,
+    label,
+    measured: measured[id],
+    baseline: baseline[id],
+  }));
+}
+
+export function readBaseline(file = BASELINE_PATH) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function main(args) {
+  const measured = measureQaHarness();
+  if (args.includes('--write-baseline')) {
+    const note = args.find((arg) => arg.startsWith('--note='))?.slice(7);
+    fs.writeFileSync(
+      BASELINE_PATH,
+      `${JSON.stringify({ metrics: measured, ...(note ? { measuredAt: note } : {}) }, null, 2)}\n`,
+    );
+  }
+  if (args.includes('--json')) {
+    console.log(JSON.stringify(measured, null, 2));
+    return;
+  }
+  const baseline = fs.existsSync(BASELINE_PATH) ? readBaseline().metrics : {};
+  for (const { id, label } of QA_HARNESS_METRICS) {
+    const was = baseline[id];
+    const change =
+      was === undefined
+        ? ''
+        : ` (baseline ${was}, ${measured[id] - was >= 0 ? '+' : ''}${measured[id] - was})`;
+    console.log(`${String(measured[id]).padStart(7)}  ${label}${change}`);
+  }
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href)
+  main(process.argv.slice(2));