test(qa): noon sky and stars, surface heights across engines, tile cache

qa-globe-contract checks the noon sky facing the horizon (daylight blue,
no star pixels) and stars at midnight; qa-engine-shell compares
sampleHeight at the Austin startup pose across engines and reports the
wheel floor's lowest frame. QA pages load OSM and keyless terrain tiles
through a disk cache with backoff on 429. Report updated.

scripts/format-scope.json

@@ -1008,6 +1008,7 @@
   "scripts/qa-engine-shell.mjs",
   "scripts/qa/navara-contract.html",
   "scripts/qa/navaraContractPage.js",
+  "scripts/qa/tileCache.mjs",
   "src/annotations/resolverEngine.test.mjs",
   "src/app/controls.test.mjs",
   "src/app/engineChoice.js",
@@ -1032,6 +1033,7 @@
   "src/globe/cesium/render.js",
   "src/globe/cesium/surface.js",
   "src/globe/contract.js",
+  "src/globe/navara/atmosphereLuminance.js",
   "src/globe/navara/cameraMath.js",
   "src/globe/navara/cameraMath.test.mjs",
   "src/globe/navara/cameraModel.js",
@@ -1060,5 +1062,6 @@
   "src/orbit.test.mjs",
   "src/services/groundSnap.test.mjs",
   "src/sharelinkPose.test.mjs",
-  "src/tooling/navaraAssets.test.mjs"
+  "src/tooling/navaraAssets.test.mjs",
+  "src/tooling/tileCache.test.mjs"
 ]

scripts/qa/tileCache.mjs

@@ -0,0 +1,147 @@
+// Map tiles for headless QA: a disk cache shared by every run (in the system
+// temp directory) and one fetch per missing tile, with backoff on HTTP 429,
+// for the free tile hosts the QA pages load (OpenStreetMap imagery, the
+// keyless re:earth terrain). Back-to-back QA runs otherwise reload the same
+// tiles until those hosts rate-limit them. Only requests to these hosts are
+// paused (CDP Fetch with URL patterns); page-wide request interception also
+// held the Navara workers' requests and kept workers alive past their globe.
+
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export const CACHED_TILE_HOSTS = Object.freeze([
+  'tile.openstreetmap.org',
+  'terrain.reearth.land',
+]);
+const RETRIES = 4;
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * The cache entry for a GET of `url` with `accept` (terrain answers by its
+ * Accept header), or null for a host that is not cached.
+ */
+export function tileCacheFile(url, accept, dir) {
+  let host;
+  try {
+    const parsed = new URL(url);
+    if (parsed.protocol !== 'https:') return null;
+    host = parsed.host;
+  } catch {
+    return null;
+  }
+  if (!CACHED_TILE_HOSTS.includes(host)) return null;
+  const key = createHash('sha256')
+    .update(`${url}\n${accept ?? ''}`)
+    .digest('hex')
+    .slice(0, 32);
+  return path.join(dir, `${host}-${key}`);
+}
+
+/**
+ * Waits before retry `attempt` (0-based) after a 429: the server's
+ * Retry-After in seconds when it sends one (at most 30 s), else 1, 2, 4, 8 s.
+ */
+export function backoffMs(attempt, retryAfter) {
+  const seconds =
+    retryAfter == null || retryAfter === '' ? Number.NaN : Number(retryAfter);
+  if (Number.isFinite(seconds) && seconds >= 0)
+    return Math.min(seconds, 30) * 1000;
+  return 1000 * 2 ** attempt;
+}
+
+/** GETs a tile, retrying on 429; resolves to { status, contentType, body }. */
+export async function fetchTile(
+  url,
+  { accept, fetchImpl = fetch, wait = sleep } = {},
+) {
+  for (let attempt = 0; ; attempt++) {
+    const response = await fetchImpl(url, {
+      headers: {
+        'User-Agent': 'gods-eye-view-qa (headless test, cached)',
+        ...(accept ? { Accept: accept } : {}),
+      },
+    });
+    if (response.status !== 429 || attempt >= RETRIES)
+      return {
+        status: response.status,
+        contentType: response.headers.get('content-type'),
+        body: Buffer.from(await response.arrayBuffer()),
+      };
+    await wait(backoffMs(attempt, response.headers.get('retry-after')));
+  }
+}
+
+/**
+ * Serves the page's requests to CACHED_TILE_HOSTS from the cache (filling
+ * it on a miss). Returns the counts.
+ */
+export async function installTileCache(
+  page,
+  { dir = path.join(os.tmpdir(), 'gev-qa-tiles') } = {},
+) {
+  fs.mkdirSync(dir, { recursive: true });
+  const stats = { hits: 0, fetched: 0, cancelled: 0, failed: 0, errors: {} };
+  const client = await page.createCDPSession();
+  client.on('Fetch.requestPaused', async ({ requestId, request }) => {
+    const accept = Object.entries(request.headers ?? {}).find(
+      ([name]) => name.toLowerCase() === 'accept',
+    )?.[1];
+    const file =
+      request.method === 'GET' ? tileCacheFile(request.url, accept, dir) : null;
+    try {
+      if (!file)
+        return await client.send('Fetch.continueRequest', { requestId });
+      let entry = null;
+      if (fs.existsSync(`${file}.json`)) {
+        entry = {
+          ...JSON.parse(fs.readFileSync(`${file}.json`, 'utf8')),
+          body: fs.readFileSync(file),
+        };
+        stats.hits++;
+      } else {
+        const tile = await fetchTile(request.url, { accept });
+        // Only successes are cached; a 404 (no tile there) is passed on.
+        if (tile.status === 200) {
+          fs.writeFileSync(file, tile.body);
+          fs.writeFileSync(
+            `${file}.json`,
+            JSON.stringify({ status: 200, contentType: tile.contentType }),
+          );
+          stats.fetched++;
+        } else if (tile.status === 429) stats.failed++;
+        entry = tile;
+      }
+      await client.send('Fetch.fulfillRequest', {
+        requestId,
+        responseCode: entry.status,
+        responseHeaders: [
+          ...(entry.contentType
+            ? [{ name: 'Content-Type', value: entry.contentType }]
+            : []),
+          { name: 'Access-Control-Allow-Origin', value: '*' },
+        ],
+        body: entry.body.toString('base64'),
+      });
+    } catch (error) {
+      // The page dropped the request meanwhile (a tile no longer in view).
+      if (/Invalid InterceptionId/.test(String(error?.message))) {
+        stats.cancelled++;
+        return;
+      }
+      stats.failed++;
+      const reason = String(error?.message ?? error).slice(0, 80);
+      stats.errors[reason] = (stats.errors[reason] ?? 0) + 1;
+      await client
+        .send('Fetch.failRequest', { requestId, errorReason: 'Failed' })
+        .catch(() => {});
+    }
+  });
+  await client.send('Fetch.enable', {
+    patterns: CACHED_TILE_HOSTS.map((host) => ({
+      urlPattern: `https://${host}/*`,
+    })),
+  });
+  return stats;
+}

src/tooling/tileCache.test.mjs

@@ -0,0 +1,71 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  backoffMs,
+  fetchTile,
+  tileCacheFile,
+} from '../../scripts/qa/tileCache.mjs';
+
+test('only the free tile hosts are cached, per URL and Accept header', () => {
+  const osm = 'https://tile.openstreetmap.org/12/1205/1539.png';
+  const file = tileCacheFile(osm, 'image/png', '/c');
+  assert.match(file, /^\/c\/tile\.openstreetmap\.org-[0-9a-f]{32}$/);
+  assert.equal(tileCacheFile(osm, 'image/png', '/c'), file, 'stable');
+  assert.notEqual(tileCacheFile(osm, 'image/webp', '/c'), file);
+  assert.match(
+    tileCacheFile('https://terrain.reearth.land/1/2/3.terrain', null, '/c'),
+    /^\/c\/terrain\.reearth\.land-/,
+  );
+  for (const url of [
+    'https://tile.googleapis.com/v1/3dtiles/root.json?key=x',
+    'http://tile.openstreetmap.org/1/2/3.png',
+    'not a url',
+  ])
+    assert.equal(tileCacheFile(url, null, '/c'), null, url);
+});
+
+test('429 backs off by Retry-After (at most 30 s) or 1, 2, 4, 8 s', () => {
+  assert.deepEqual(
+    [0, 1, 2, 3].map((attempt) => backoffMs(attempt, null)),
+    [1000, 2000, 4000, 8000],
+  );
+  assert.equal(backoffMs(0, '5'), 5000);
+  assert.equal(backoffMs(0, '600'), 30000);
+});
+
+const response = (status, headers = {}) => ({
+  status,
+  headers: { get: (name) => headers[name] ?? null },
+  arrayBuffer: async () => new Uint8Array([status % 256]).buffer,
+});
+
+test('a tile fetch retries 429s, stops after four retries, keeps Accept', async () => {
+  const waits = [];
+  const seen = [];
+  const replies = [
+    response(429, { 'retry-after': '2' }),
+    response(429),
+    response(200, { 'content-type': 'image/png' }),
+  ];
+  const tile = await fetchTile('u', {
+    accept: 'application/vnd.quantized-mesh',
+    fetchImpl: async (url, init) => (
+      seen.push(init.headers.Accept),
+      replies.shift()
+    ),
+    wait: async (ms) => waits.push(ms),
+  });
+  assert.equal(tile.status, 200);
+  assert.equal(tile.contentType, 'image/png');
+  assert.deepEqual(waits, [2000, 2000]);
+  assert.ok(
+    seen.every((accept) => accept === 'application/vnd.quantized-mesh'),
+  );
+  let calls = 0;
+  const limited = await fetchTile('u', {
+    fetchImpl: async () => (calls++, response(429)),
+    wait: async () => {},
+  });
+  assert.equal(limited.status, 429);
+  assert.equal(calls, 5);
+});