Check the globe contract by behaviour on every engine

The contract suite is split into one module per area and grows from 87
to 114 checks, each named after the member it exercises: a gesture during
a flight does not cut it, a render request inside a frame draws the next
one, a pure rotation raises changed, ground corridors are as wide in CSS
pixels at device scale 2, imagery replans on resize, drapes draw only on
their surface, sampleTerrainMostDetailed keeps order and passes on a
rejection, terrainRevision changes on a new terrain and on fallback,
follows raise followChanged on retarget, and a mesh, a ground fill and an
imagery pixel draw the colour asked for. In the browser Cesium passes all
113 it runs at device scale 1 and 2; Navara passes 110 of 111 and reports
its single-colour mesh, drawn before tone mapping, as a known failure.

scripts/contract-mutants.test.mjs breaks one promise of the fake engine
at a time, 30 mutants in all, and names the check that catches each; the
list of contract members the suite never calls is down to five.

maps.errors reports tile and terrain errors where an engine has them
(the tileErrors capability: Cesium yes, Navara no), and the map
controller falls back to another map only where it does; movers.list()
rows tell whether a set is shown, as the other families do; FollowHandle
.name and PickResult.primitive are marked deprecated. qa-contract-suite
takes --dpr and the ANGLE backend from QA_ANGLE.

scripts/contract-mutants.test.mjs

@@ -0,0 +1,367 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createFakeEngine } from '../src/globe/testing/fakeEngine.js';
+import { fakeContractOptions } from '../src/globe/testing/contractFixtures.js';
+import { runContractSuite } from '../src/globe/testing/contractSuite/index.mjs';
+
+const replace = (object, member, wrap) => {
+  object[member] = wrap(object[member].bind(object));
+};
+const handle = (engine, area, factory, mutate) =>
+  replace(engine[area], factory, (create) => (...args) => {
+    const value = create(...args);
+    mutate(value);
+    return value;
+  });
+
+const mutants = [
+  [
+    'retarget omits followChanged',
+    'camera.following: set, retarget',
+    (e) => {
+      let previous;
+      replace(e.camera.followChanged, 'raiseEvent', (raise) => (value) => {
+        if (value !== previous) raise(value);
+        previous = value;
+      });
+    },
+  ],
+  [
+    'pointer cancels an active flight',
+    'camera.flyTo: a pointer gesture',
+    (e) => {
+      replace(e.fake, 'input', (input) => (type, event) => {
+        if (type === 'down') e.camera.cancelFlight();
+        input(type, event);
+      });
+    },
+  ],
+  [
+    'terrain errors are labelled as tiles',
+    'maps.errors: terrain failures',
+    (e) => {
+      replace(
+        e.maps.errors,
+        'raiseEvent',
+        (raise) => (error) => raise({ ...error, kind: 'tile' }),
+      );
+    },
+  ],
+  [
+    'interrupted flight omits cancel',
+    'camera.flyTo: a newer flight',
+    (e) => {
+      replace(
+        e.camera,
+        'flyTo',
+        (fly) => (options) => fly({ ...options, cancel: undefined }),
+      );
+    },
+  ],
+  [
+    'pure rotation omits changed',
+    'camera.changed: a pure heading',
+    (e) => {
+      let previous = e.camera.pose().position;
+      replace(e.camera.changed, 'raiseEvent', (raise) => (...args) => {
+        const position = e.camera.pose().position;
+        const moved =
+          Math.hypot(
+            position.x - previous.x,
+            position.y - previous.y,
+            position.z - previous.z,
+          ) > 0.01;
+        previous = position;
+        if (moved) raise(...args);
+      });
+    },
+  ],
+  [
+    'request during render is lost',
+    'render.request: a request inside',
+    (e) => {
+      let inFrame = false;
+      e.frames.preRender.addEventListener(() => {
+        inFrame = true;
+      });
+      e.frames.postRender.addEventListener(() => {
+        inFrame = false;
+      });
+      replace(e.render, 'request', (request) => (...args) => {
+        if (!inFrame) request(...args);
+      });
+    },
+  ],
+  [
+    'terrain revision stays fixed',
+    'maps.terrainRevision: changes when setTerrain',
+    (e) => {
+      e.maps.terrainRevision = () => 0;
+    },
+  ],
+  [
+    'removed mover remains listed',
+    'movers.list: shown follows',
+    (e) => {
+      const removed = [];
+      const list = e.movers.list;
+      handle(e, 'movers', 'createSet', (set) => {
+        replace(set, 'destroy', (destroy) => () => {
+          const row = list().find(({ id }) => id === set.id);
+          if (row) removed.push(row);
+          destroy();
+        });
+      });
+      e.movers.list = () => [...list(), ...removed];
+    },
+  ],
+  [
+    'stop omits followChanged',
+    'camera.following: set, retarget',
+    (e) => {
+      replace(e.camera.followChanged, 'raiseEvent', (raise) => (value) => {
+        if (value !== null) raise(value);
+      });
+    },
+  ],
+  [
+    'following forgets the handle',
+    'camera.following: set, retarget',
+    (e) => {
+      e.camera.following = () => null;
+    },
+  ],
+  [
+    'retarget keeps the old target',
+    'camera.following: set, retarget',
+    (e) => {
+      handle(e, 'camera', 'follow', (follow) => {
+        follow.retarget = () => {};
+      });
+    },
+  ],
+  [
+    'texture limit is not a power of two',
+    'render.maxTextureSize:',
+    (e) => {
+      e.render.maxTextureSize = () => 3;
+    },
+  ],
+  [
+    'continuous mode never changes',
+    'render.setContinuous: continuous',
+    (e) => {
+      e.render.setContinuous = () => {};
+    },
+  ],
+  [
+    'clock is never latched',
+    'clock.frameTime: advances before',
+    (e) => {
+      e.clock.frameTime = () => 0;
+    },
+  ],
+  [
+    'terrain samples are reversed',
+    'surface.sampleTerrainMostDetailed: preserves',
+    (e) => {
+      replace(
+        e.surface,
+        'sampleTerrainMostDetailed',
+        (sample) =>
+          async (...args) =>
+            (await sample(...args)).reverse(),
+      );
+    },
+  ],
+  [
+    'unknown terrain becomes zero',
+    'surface.sampleTerrainMostDetailed: preserves',
+    (e) => {
+      replace(
+        e.surface,
+        'sampleTerrainMostDetailed',
+        (sample) =>
+          async (...args) =>
+            (await sample(...args)).map((height) => height ?? 0),
+      );
+    },
+  ],
+  [
+    'terrain rejection is swallowed',
+    'surface.sampleTerrainMostDetailed: passes',
+    (e) => {
+      replace(
+        e.surface,
+        'sampleTerrainMostDetailed',
+        (sample) =>
+          async (...args) => {
+            try {
+              return await sample(...args);
+            } catch {
+              return [];
+            }
+          },
+      );
+    },
+  ],
+  [
+    'failed model reports no failure',
+    'MoverSet.failed:',
+    (e) => {
+      handle(e, 'movers', 'createSet', (set) => {
+        set.failed = () => false;
+      });
+    },
+  ],
+  [
+    'failed model reports ready',
+    'MoverSet.failed:',
+    (e) => {
+      handle(e, 'movers', 'createSet', (set) => {
+        set.ready = () => true;
+      });
+    },
+  ],
+  [
+    'hidden mover reports shown',
+    'movers.list: shown follows',
+    (e) => {
+      replace(
+        e.movers,
+        'list',
+        (list) => () => list().map((row) => ({ ...row, shown: true })),
+      );
+    },
+  ],
+  [
+    'map error loses HTTP status',
+    'maps.errors: forwards source',
+    (e) => {
+      replace(
+        e.maps.errors,
+        'raiseEvent',
+        (raise) => (error) => raise({ ...error, status: undefined }),
+      );
+    },
+  ],
+  [
+    'map errors are swallowed',
+    'maps.errors: forwards source',
+    (e) => {
+      e.maps.errors.raiseEvent = () => {};
+    },
+  ],
+  [
+    'old map providers keep reporting',
+    'maps.errors: forwards source',
+    (e) => {
+      replace(e.maps, 'setBaseImagery', (set) => async (source) => {
+        await set(source);
+        source.provider?.errorEvent?.addEventListener((error) => {
+          e.maps.errors.raiseEvent({
+            source,
+            kind: 'tile',
+            status: error?.statusCode,
+          });
+        });
+      });
+    },
+  ],
+  [
+    'flight remains active after completion',
+    'camera.flyTo: duration 0',
+    (e) => {
+      e.camera.flightActive = () => true;
+    },
+  ],
+  [
+    'setView leaves the flight running',
+    'camera.setView: cancels an active',
+    (e) => {
+      e.camera.setView = ({ destination, orientation }) =>
+        e.fake.setPose({ position: destination, ...orientation });
+    },
+  ],
+  [
+    'instant flight omits complete',
+    'camera.flyTo: duration 0',
+    (e) => {
+      replace(
+        e.camera,
+        'flyTo',
+        (fly) => (options) => fly({ ...options, complete: undefined }),
+      );
+    },
+  ],
+  [
+    'hidden meshes remain shown',
+    'meshes.createMesh: picks use',
+    (e) => {
+      handle(e, 'meshes', 'createMesh', (mesh) => {
+        replace(
+          mesh,
+          'show',
+          (show) => (value) => show(value === false ? true : value),
+        );
+      });
+    },
+  ],
+  [
+    'imagery ignores alpha changes',
+    'imagery.createLayer: layers are',
+    (e) => {
+      handle(e, 'imagery', 'createLayer', (layer) => {
+        layer.setAlpha = () => {};
+      });
+    },
+  ],
+  [
+    'clearing tiles forgets base imagery',
+    'maps.setBaseImagery: setPhotorealTiles',
+    (e) => {
+      replace(e.maps, 'setPhotorealTiles', (set) => async (source) => {
+        await set(source);
+        if (source === null) e.fake.maps.imagery = null;
+      });
+    },
+  ],
+  [
+    'tiles-only ground also draws on terrain',
+    'ground.createSet: tiles drape',
+    (e) => {
+      replace(
+        e.ground,
+        'createSet',
+        (create) => (options) =>
+          create({
+            ...options,
+            drape: options.drape === 'tiles' ? 'both' : options.drape,
+          }),
+      );
+    },
+  ],
+];
+
+for (const [name, killedBy, mutate] of mutants) {
+  test(`${name} — killed by ${killedBy}`, async () => {
+    const engine = createFakeEngine();
+    try {
+      mutate(engine);
+      const results = await runContractSuite(engine, {
+        ...fakeContractOptions(engine),
+        maxWaitFrames: 160,
+      });
+      const check = results.find(({ name }) => name.startsWith(killedBy));
+      assert.ok(check, `No check named ${killedBy}`);
+      assert.equal(check.ok, false, `Surviving mutant: ${name}`);
+      assert.ok(
+        check.error.name === 'ContractError' ||
+          check.error.message === 'No frame requested',
+        `Unexpected failure: ${check.error.stack}`,
+      );
+    } finally {
+      engine.destroy();
+    }
+  });
+}

scripts/package-boundaries.json

@@ -2285,7 +2285,8 @@
       "src/i18n/locales/ru.json",
       "src/i18n/locales/uk.json",
       "src/i18n/locales/zh-CN.json",
-      "src/i18n/locales/zh-Hant.json"
+      "src/i18n/locales/zh-Hant.json",
+      "src/globe/contract.js"
     ],
     "external": ["cesium"]
   },

scripts/qa-contract-suite.mjs

@@ -26,13 +26,22 @@ const option = (name) =>
   args.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3);
 if (!origin) {
   console.error(
-    'Usage: node scripts/qa-contract-suite.mjs <origin> [--engine=cesium,navara] [--only=prefix,...] [--json=FILE]',
+    'Usage: node scripts/qa-contract-suite.mjs <origin> [--engine=cesium,navara] [--only=prefix,...] [--dpr=1|2] [--json=FILE]',
   );
   process.exit(2);
 }
 const engines = (option('engine') ?? 'cesium,navara').split(',');
 // --only=prefix,prefix runs just the checks whose names start with one.
-const only = option('only')?.split(',') ?? null;
+const only =
+  option('only')
+    ?.split(',')
+    .map((prefix) =>
+      /^(ground|imagery|movers|lines|meshes|maps):$/.test(prefix)
+        ? prefix.replace(':', '.')
+        : prefix,
+    ) ?? null;
+const dpr = Number(option('dpr') ?? 1);
+if (![1, 2].includes(dpr)) throw new Error('--dpr must be 1 or 2');
 
 // Checks of state the running app drives itself, by name prefix, with the
 // owner. The app sets contact sonar on every frame (src/cyberSonarEngine.js),
@@ -43,13 +52,15 @@ const APP_OWNED = Object.freeze([
 
 async function runOn(browser, kind) {
   const page = await browser.newPage();
-  await page.setViewport({ width: 1280, height: 800 });
+  await page.setViewport({ width: 1280, height: 800, deviceScaleFactor: dpr });
   const pageErrors = [];
   page.on('pageerror', (error) => pageErrors.push(error.message));
   await page.setRequestInterception(true);
   page.on('request', (request) => {
     const { hostname } = new URL(request.url());
-    if (BLOCKED_HOSTS.test(hostname)) request.abort();
+    if (new URL(request.url()).pathname === '/__contract_missing_model.glb')
+      request.respond({ status: 404, body: 'Model unavailable' });
+    else if (BLOCKED_HOSTS.test(hostname)) request.abort();
     else request.continue();
   });
   // A view in the link: the app starts there instead of flying to a city.
@@ -139,25 +150,53 @@ async function runOn(browser, kind) {
       await page.mouse.move(...at);
     }
   });
+  await page.exposeFunction('__contractDrag', async (x, y, endX, endY) => {
+    const box = await page.evaluate(() => {
+      const rect =
+        window.__godsEyeView.engine.host.canvas.getBoundingClientRect();
+      return { left: rect.left, top: rect.top };
+    });
+    await page.mouse.move(box.left + x, box.top + y);
+    await page.mouse.down();
+    await page.mouse.move(box.left + endX, box.top + endY, { steps: 4 });
+    await page.mouse.up();
+  });
+  await page.exposeFunction('__contractResize', async (width, height) => {
+    await page.setViewport({
+      width: Math.round(width),
+      height: Math.round(height),
+      deviceScaleFactor: dpr,
+    });
+  });
   const results = await page.evaluate(
     async (appOwned, only) => {
       const { engine } = window.__godsEyeView;
       const waitFrames = (count) =>
-        new Promise((resolve) => {
+        new Promise((resolve, reject) => {
+          const timer = setTimeout(() => {
+            remove();
+            reject(new Error('Timed out waiting for a rendered frame'));
+          }, 10_000);
           let left = count;
           const remove = engine.frames.postRender.addEventListener(() => {
             if (--left > 0) return;
             remove();
+            clearTimeout(timer);
             resolve();
           });
-          engine.render.request('contract-suite');
         });
       const pointer = {
         click: (x, y) => window.__contractPointer('click', x, y),
         move: (x, y) => window.__contractPointer('move', x, y),
+        drag: (x, y, endX, endY) => window.__contractDrag(x, y, endX, endY),
       };
       const { createContractSuite } =
         await import('/src/globe/testing/contractSuite.mjs');
+      const fixtureModule =
+        engine.kind === 'cesium'
+          ? await import('/src/globe/cesium/testing/contractFixtures.js')
+          : await import('/src/globe/navara/testing/contractFixtures.js');
+      const fixtures = await fixtureModule.createContractFixtures();
       const outcome = [];
       // A real 2D canvas: a sprite atlas the engines can draw and pick.
       const makeCanvas = (width, height) =>
@@ -189,6 +228,8 @@ async function runOn(browser, kind) {
         pointer,
         readRegion,
         makeCanvas,
+        fixtures,
+        resize: (width, height) => window.__contractResize(width, height),
       })) {
         if (only && !only.some((prefix) => name.startsWith(prefix))) continue;
         const owner = appOwned.find(([prefix]) => name.startsWith(prefix));
@@ -196,6 +237,12 @@ async function runOn(browser, kind) {
           outcome.push({ name, skipped: owner[1] });
           continue;
         }
+        const knownFailure =
+          engine.kind === 'navara' &&
+          name ===
+            'meshes.createMesh: a single colour draws as asked within 10 per channel'
+            ? 'Meshes are drawn before tone mapping'
+            : null;
         try {
           await run();
           outcome.push({ name, ok: true });
@@ -203,24 +250,33 @@ async function runOn(browser, kind) {
           outcome.push({
             name,
             ok: false,
+            ...(knownFailure &&
+              String(error?.message).startsWith(
+                'mesh colour within 10 per channel',
+              ) && { knownFailure }),
             error: String(error?.message ?? error),
           });
         }
       }
+      fixtures.destroy?.();
       return outcome;
     },
     APP_OWNED,
     only,
   );
   await page.close();
-  return { engine: kind, results, pageErrors };
+  return { engine: kind, dpr, results, pageErrors };
 }
 
 const browser = await puppeteer.launch({
   headless: true,
   // A Navara page can hold its main thread for minutes while it starts.
   protocolTimeout: 600_000,
-  args: ['--use-angle=metal', '--enable-gpu', '--no-sandbox'],
+  args: [
+    `--use-angle=${process.env.QA_ANGLE ?? (process.platform === 'darwin' ? 'metal' : 'swiftshader')}`,
+    '--enable-gpu',
+    '--no-sandbox',
+  ],
 });
 const report = [];
 try {
@@ -229,13 +285,16 @@ try {
     report.push(run);
     const ran = run.results.filter(({ skipped }) => !skipped);
     const failed = ran.filter(({ ok }) => !ok);
+    const known = failed.filter(({ knownFailure }) => knownFailure);
     console.log(
-      `${kind}: ${ran.length - failed.length}/${ran.length} checks pass`,
+      `${kind} DPR ${dpr}: ${ran.length - failed.length}/${ran.length} checks pass; ${known.length} known failures; ${run.results.length - ran.length} skipped`,
     );
     for (const { name, skipped } of run.results)
       if (skipped) console.log(`  skipped ${name}: ${skipped}`);
-    for (const { name, error } of failed)
-      console.log(`  FAIL ${name}: ${error}`);
+    for (const { name, error, knownFailure } of failed)
+      console.log(
+        `  ${knownFailure ? 'KNOWN FAIL' : 'FAIL'} ${name}: ${error}`,
+      );
     for (const message of run.pageErrors)
       console.log(`  page error: ${message}`);
   }
@@ -246,7 +305,9 @@ const json = option('json');
 if (json) fs.writeFileSync(json, `${JSON.stringify(report, null, 2)}\n`);
 process.exit(
   report.every(({ results }) =>
-    results.every(({ ok, skipped }) => ok || skipped),
+    results.every(
+      ({ ok, skipped, knownFailure }) => ok || skipped || knownFailure,
+    ),
   )
     ? 0
     : 1,