Files
gods-eye-view/scripts/qa/tileCache.mjs
T
German Khokhlov 1773494116 Run the layer acceptance under a default-deny network policy with per-layer assertion manifests and strict switch contracts
Every scenario runs through scripts/qa-strict-harness.mjs: each page, frame and worker gets an ordered block list before it runs (paid hosts and routes first, then only the preview, declared fixtures, keyless probes and cached free tile hosts), switches with real input on the picker, and records its assertions per layer and engine. The runner reads the catalog from the live page, fails any layer without an assertion on either engine, can build and serve a keyless checkout itself, and adds rapid, hidden-tab, context-loss, failure-injection, cold-chunk and soak scenarios. The main chunk gets a 1% size budget.
2026-11-03 20:15:50 +04:00

201 lines
6.5 KiB
JavaScript

// Map tiles for headless QA: a disk cache shared by every run (in the system
// temp directory) and one fetch per missing tile, with backoff on HTTP 429,
// for the free tile hosts the QA pages load (OpenStreetMap imagery, the
// keyless re:earth terrain). Back-to-back QA runs otherwise reload the same
// tiles until those hosts rate-limit them. Only requests to these hosts are
// paused (CDP Fetch with URL patterns); page-wide request interception also
// held the Navara workers' requests and kept workers alive past their globe.
import { createHash } from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
export const CACHED_TILE_HOSTS = Object.freeze([
'tile.openstreetmap.org',
'terrain.reearth.land',
]);
const RETRIES = 4;
/** Who is asking, with a contact URL (the tile hosts' usage policies). */
export const QA_USER_AGENT =
'gods-eye-view-qa/1 (headless tests with a disk cache; https://github.com/bilawalsidhu/gods-eye-view)';
/** Fetches to the tile hosts in flight at once. */
export const MAX_MISSES_IN_FLIGHT = 2;
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
/**
* The cache entry for a GET of `url` with `accept` (terrain answers by its
* Accept header), or null for a host that is not cached.
*/
export function tileCacheFile(url, accept, dir, hosts = CACHED_TILE_HOSTS) {
let host;
try {
const parsed = new URL(url);
if (parsed.protocol !== 'https:') return null;
host = parsed.host;
} catch {
return null;
}
if (!hosts.includes(host)) return null;
const key = createHash('sha256')
.update(`${url}\n${accept ?? ''}`)
.digest('hex')
.slice(0, 32);
return path.join(dir, `${host}-${key}`);
}
/**
* Waits before retry `attempt` (0-based) after a 429: the server's
* Retry-After in seconds when it sends one (at most 30 s), else 1, 2, 4, 8 s.
*/
export function backoffMs(attempt, retryAfter) {
const seconds =
retryAfter == null || retryAfter === '' ? Number.NaN : Number(retryAfter);
if (Number.isFinite(seconds) && seconds >= 0)
return Math.min(seconds, 30) * 1000;
return 1000 * 2 ** attempt;
}
/** GETs a tile, retrying on 429; resolves to { status, contentType, body }. */
export async function fetchTile(
url,
{ accept, fetchImpl = fetch, wait = sleep } = {},
) {
for (let attempt = 0; ; attempt++) {
const response = await fetchImpl(url, {
headers: {
'User-Agent': QA_USER_AGENT,
...(accept ? { Accept: accept } : {}),
},
});
if (response.status !== 429 || attempt >= RETRIES)
return {
status: response.status,
contentType: response.headers.get('content-type'),
body: Buffer.from(await response.arrayBuffer()),
};
await wait(backoffMs(attempt, response.headers.get('retry-after')));
}
}
/**
* Loads missing tiles politely: at most MAX_MISSES_IN_FLIGHT fetches at
* once (the rest queue), and one fetch for identical misses in flight (same
* URL and Accept). `load(url, accept)` resolves to fetchTile's result.
*/
export function createMissLoader({
fetchImpl = fetch,
wait = sleep,
maxInFlight = MAX_MISSES_IN_FLIGHT,
} = {}) {
const pending = new Map();
const queue = [];
let running = 0;
const next = () => {
while (running < maxInFlight && queue.length) {
const job = queue.shift();
running++;
fetchTile(job.url, { accept: job.accept, fetchImpl, wait })
.then(job.resolve, job.reject)
.finally(() => {
running--;
pending.delete(job.key);
next();
});
}
};
return {
load(url, accept) {
const key = `${url}\n${accept ?? ''}`;
let promise = pending.get(key);
if (!promise) {
promise = new Promise((resolve, reject) =>
queue.push({ key, url, accept, resolve, reject }),
);
pending.set(key, promise);
next();
}
return promise;
},
};
}
/**
* Serves the page's requests to CACHED_TILE_HOSTS (or `hosts`) from the
* cache (filling it on a miss). Returns the counts.
*/
export async function installTileCache(
page,
{
dir = path.join(os.tmpdir(), 'gev-qa-tiles'),
hosts = CACHED_TILE_HOSTS,
} = {},
) {
fs.mkdirSync(dir, { recursive: true });
const stats = { hits: 0, fetched: 0, cancelled: 0, failed: 0, errors: {} };
const client = await page.createCDPSession();
const misses = createMissLoader();
client.on('Fetch.requestPaused', async ({ requestId, request }) => {
const accept = Object.entries(request.headers ?? {}).find(
([name]) => name.toLowerCase() === 'accept',
)?.[1];
const file =
request.method === 'GET'
? tileCacheFile(request.url, accept, dir, hosts)
: null;
try {
if (!file)
return await client.send('Fetch.continueRequest', { requestId });
let entry = null;
if (fs.existsSync(`${file}.json`)) {
entry = {
...JSON.parse(fs.readFileSync(`${file}.json`, 'utf8')),
body: fs.readFileSync(file),
};
stats.hits++;
} else {
const tile = await misses.load(request.url, accept);
// Only successes are cached; a 404 (no tile there) is passed on.
if (tile.status === 200) {
fs.writeFileSync(file, tile.body);
fs.writeFileSync(
`${file}.json`,
JSON.stringify({ status: 200, contentType: tile.contentType }),
);
stats.fetched++;
} else if (tile.status === 429) stats.failed++;
entry = tile;
}
await client.send('Fetch.fulfillRequest', {
requestId,
responseCode: entry.status,
responseHeaders: [
...(entry.contentType
? [{ name: 'Content-Type', value: entry.contentType }]
: []),
{ name: 'Access-Control-Allow-Origin', value: '*' },
],
body: entry.body.toString('base64'),
});
} catch (error) {
// The page dropped the request meanwhile (a tile no longer in view).
if (/Invalid InterceptionId/.test(String(error?.message))) {
stats.cancelled++;
return;
}
stats.failed++;
const reason = String(error?.message ?? error).slice(0, 80);
stats.errors[reason] = (stats.errors[reason] ?? 0) + 1;
await client
.send('Fetch.failRequest', { requestId, errorReason: 'Failed' })
.catch(() => {});
}
});
await client.send('Fetch.enable', {
patterns: hosts.map((host) => ({
urlPattern: `https://${host}/*`,
})),
});
return stats;
}