diff --git a/scripts/format-scope.json b/scripts/format-scope.json index d35da2367..8e0aa8922 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -1008,6 +1008,7 @@ "scripts/qa-engine-shell.mjs", "scripts/qa/navara-contract.html", "scripts/qa/navaraContractPage.js", + "scripts/qa/tileCache.mjs", "src/annotations/resolverEngine.test.mjs", "src/app/controls.test.mjs", "src/app/engineChoice.js", @@ -1032,6 +1033,7 @@ "src/globe/cesium/render.js", "src/globe/cesium/surface.js", "src/globe/contract.js", + "src/globe/navara/atmosphereLuminance.js", "src/globe/navara/cameraMath.js", "src/globe/navara/cameraMath.test.mjs", "src/globe/navara/cameraModel.js", @@ -1060,5 +1062,6 @@ "src/orbit.test.mjs", "src/services/groundSnap.test.mjs", "src/sharelinkPose.test.mjs", - "src/tooling/navaraAssets.test.mjs" + "src/tooling/navaraAssets.test.mjs", + "src/tooling/tileCache.test.mjs" ] diff --git a/scripts/qa/tileCache.mjs b/scripts/qa/tileCache.mjs new file mode 100644 index 000000000..b345e96e5 --- /dev/null +++ b/scripts/qa/tileCache.mjs @@ -0,0 +1,147 @@ +// Map tiles for headless QA: a disk cache shared by every run (in the system +// temp directory) and one fetch per missing tile, with backoff on HTTP 429, +// for the free tile hosts the QA pages load (OpenStreetMap imagery, the +// keyless re:earth terrain). Back-to-back QA runs otherwise reload the same +// tiles until those hosts rate-limit them. Only requests to these hosts are +// paused (CDP Fetch with URL patterns); page-wide request interception also +// held the Navara workers' requests and kept workers alive past their globe. + +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export const CACHED_TILE_HOSTS = Object.freeze([ + 'tile.openstreetmap.org', + 'terrain.reearth.land', +]); +const RETRIES = 4; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * The cache entry for a GET of `url` with `accept` (terrain answers by its + * Accept header), or null for a host that is not cached. + */ +export function tileCacheFile(url, accept, dir) { + let host; + try { + const parsed = new URL(url); + if (parsed.protocol !== 'https:') return null; + host = parsed.host; + } catch { + return null; + } + if (!CACHED_TILE_HOSTS.includes(host)) return null; + const key = createHash('sha256') + .update(`${url}\n${accept ?? ''}`) + .digest('hex') + .slice(0, 32); + return path.join(dir, `${host}-${key}`); +} + +/** + * Waits before retry `attempt` (0-based) after a 429: the server's + * Retry-After in seconds when it sends one (at most 30 s), else 1, 2, 4, 8 s. + */ +export function backoffMs(attempt, retryAfter) { + const seconds = + retryAfter == null || retryAfter === '' ? Number.NaN : Number(retryAfter); + if (Number.isFinite(seconds) && seconds >= 0) + return Math.min(seconds, 30) * 1000; + return 1000 * 2 ** attempt; +} + +/** GETs a tile, retrying on 429; resolves to { status, contentType, body }. */ +export async function fetchTile( + url, + { accept, fetchImpl = fetch, wait = sleep } = {}, +) { + for (let attempt = 0; ; attempt++) { + const response = await fetchImpl(url, { + headers: { + 'User-Agent': 'gods-eye-view-qa (headless test, cached)', + ...(accept ? { Accept: accept } : {}), + }, + }); + if (response.status !== 429 || attempt >= RETRIES) + return { + status: response.status, + contentType: response.headers.get('content-type'), + body: Buffer.from(await response.arrayBuffer()), + }; + await wait(backoffMs(attempt, response.headers.get('retry-after'))); + } +} + +/** + * Serves the page's requests to CACHED_TILE_HOSTS from the cache (filling + * it on a miss). Returns the counts. + */ +export async function installTileCache( + page, + { dir = path.join(os.tmpdir(), 'gev-qa-tiles') } = {}, +) { + fs.mkdirSync(dir, { recursive: true }); + const stats = { hits: 0, fetched: 0, cancelled: 0, failed: 0, errors: {} }; + const client = await page.createCDPSession(); + client.on('Fetch.requestPaused', async ({ requestId, request }) => { + const accept = Object.entries(request.headers ?? {}).find( + ([name]) => name.toLowerCase() === 'accept', + )?.[1]; + const file = + request.method === 'GET' ? tileCacheFile(request.url, accept, dir) : null; + try { + if (!file) + return await client.send('Fetch.continueRequest', { requestId }); + let entry = null; + if (fs.existsSync(`${file}.json`)) { + entry = { + ...JSON.parse(fs.readFileSync(`${file}.json`, 'utf8')), + body: fs.readFileSync(file), + }; + stats.hits++; + } else { + const tile = await fetchTile(request.url, { accept }); + // Only successes are cached; a 404 (no tile there) is passed on. + if (tile.status === 200) { + fs.writeFileSync(file, tile.body); + fs.writeFileSync( + `${file}.json`, + JSON.stringify({ status: 200, contentType: tile.contentType }), + ); + stats.fetched++; + } else if (tile.status === 429) stats.failed++; + entry = tile; + } + await client.send('Fetch.fulfillRequest', { + requestId, + responseCode: entry.status, + responseHeaders: [ + ...(entry.contentType + ? [{ name: 'Content-Type', value: entry.contentType }] + : []), + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + body: entry.body.toString('base64'), + }); + } catch (error) { + // The page dropped the request meanwhile (a tile no longer in view). + if (/Invalid InterceptionId/.test(String(error?.message))) { + stats.cancelled++; + return; + } + stats.failed++; + const reason = String(error?.message ?? error).slice(0, 80); + stats.errors[reason] = (stats.errors[reason] ?? 0) + 1; + await client + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await client.send('Fetch.enable', { + patterns: CACHED_TILE_HOSTS.map((host) => ({ + urlPattern: `https://${host}/*`, + })), + }); + return stats; +} diff --git a/src/tooling/tileCache.test.mjs b/src/tooling/tileCache.test.mjs new file mode 100644 index 000000000..19ffcb879 --- /dev/null +++ b/src/tooling/tileCache.test.mjs @@ -0,0 +1,71 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + backoffMs, + fetchTile, + tileCacheFile, +} from '../../scripts/qa/tileCache.mjs'; + +test('only the free tile hosts are cached, per URL and Accept header', () => { + const osm = 'https://tile.openstreetmap.org/12/1205/1539.png'; + const file = tileCacheFile(osm, 'image/png', '/c'); + assert.match(file, /^\/c\/tile\.openstreetmap\.org-[0-9a-f]{32}$/); + assert.equal(tileCacheFile(osm, 'image/png', '/c'), file, 'stable'); + assert.notEqual(tileCacheFile(osm, 'image/webp', '/c'), file); + assert.match( + tileCacheFile('https://terrain.reearth.land/1/2/3.terrain', null, '/c'), + /^\/c\/terrain\.reearth\.land-/, + ); + for (const url of [ + 'https://tile.googleapis.com/v1/3dtiles/root.json?key=x', + 'http://tile.openstreetmap.org/1/2/3.png', + 'not a url', + ]) + assert.equal(tileCacheFile(url, null, '/c'), null, url); +}); + +test('429 backs off by Retry-After (at most 30 s) or 1, 2, 4, 8 s', () => { + assert.deepEqual( + [0, 1, 2, 3].map((attempt) => backoffMs(attempt, null)), + [1000, 2000, 4000, 8000], + ); + assert.equal(backoffMs(0, '5'), 5000); + assert.equal(backoffMs(0, '600'), 30000); +}); + +const response = (status, headers = {}) => ({ + status, + headers: { get: (name) => headers[name] ?? null }, + arrayBuffer: async () => new Uint8Array([status % 256]).buffer, +}); + +test('a tile fetch retries 429s, stops after four retries, keeps Accept', async () => { + const waits = []; + const seen = []; + const replies = [ + response(429, { 'retry-after': '2' }), + response(429), + response(200, { 'content-type': 'image/png' }), + ]; + const tile = await fetchTile('u', { + accept: 'application/vnd.quantized-mesh', + fetchImpl: async (url, init) => ( + seen.push(init.headers.Accept), + replies.shift() + ), + wait: async (ms) => waits.push(ms), + }); + assert.equal(tile.status, 200); + assert.equal(tile.contentType, 'image/png'); + assert.deepEqual(waits, [2000, 2000]); + assert.ok( + seen.every((accept) => accept === 'application/vnd.quantized-mesh'), + ); + let calls = 0; + const limited = await fetchTile('u', { + fetchImpl: async () => (calls++, response(429)), + wait: async () => {}, + }); + assert.equal(limited.status, 429); + assert.equal(calls, 5); +});