From 177349411654628f2d4a6fca7128eba6b8fa4717 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 3 Nov 2026 20:15:50 +0400 Subject: [PATCH] Run the layer acceptance under a default-deny network policy with per-layer assertion manifests and strict switch contracts Every scenario runs through scripts/qa-strict-harness.mjs: each page, frame and worker gets an ordered block list before it runs (paid hosts and routes first, then only the preview, declared fixtures, keyless probes and cached free tile hosts), switches with real input on the picker, and records its assertions per layer and engine. The runner reads the catalog from the live page, fails any layer without an assertion on either engine, can build and serve a keyless checkout itself, and adds rapid, hidden-tab, context-loss, failure-injection, cold-chunk and soak scenarios. The main chunk gets a 1% size budget. --- scripts/main-chunk-baseline.json | 6 + scripts/qa-all-layer-switches.mjs | 519 ++++++++-- scripts/qa-cctv-engine-switch.mjs | 59 +- scripts/qa-contact-switch.mjs | 179 +++- scripts/qa-directions-engine-switch.mjs | 85 +- scripts/qa-director-engine-switch.mjs | 35 +- scripts/qa-engine-switch.mjs | 891 +++++++++--------- scripts/qa-fire-camera-switch.mjs | 206 ++-- scripts/qa-ghostmaps-switch.mjs | 95 +- scripts/qa-ground-layer-switch.mjs | 439 +++++++-- scripts/qa-infrastructure-engine-switch.mjs | 149 ++- scripts/qa-layer-handoff.mjs | 863 ++++++++++------- scripts/qa-local-adsb-engine-switch.mjs | 59 +- scripts/qa-map-policy.mjs | 102 ++ scripts/qa-map-policy.test.mjs | 58 +- scripts/qa-nepal-event-switch.mjs | 86 +- scripts/qa-nepal-locator-switch.mjs | 52 +- scripts/qa-radio-engine-switch.mjs | 247 ++--- scripts/qa-space-engine-switch.mjs | 66 +- scripts/qa-street-level-engine-switch.mjs | 344 ++++--- scripts/qa-strict-cold-chunks.mjs | 95 ++ scripts/qa-strict-context-loss.mjs | 235 +++++ scripts/qa-strict-failure-matrix.mjs | 283 ++++++ scripts/qa-strict-harness.mjs | 736 +++++++++++++++ scripts/qa-strict-hidden-switch.mjs | 142 +++ scripts/qa-strict-rapid-switch.mjs | 162 ++++ scripts/qa-strict-session.mjs | 390 ++++++++ scripts/qa-strict-session.test.mjs | 47 + scripts/qa-strict-soak.mjs | 378 ++++++++ scripts/qa-submarine-cables-engine-switch.mjs | 84 +- scripts/qa-traffic-engine-switch.mjs | 76 +- scripts/qa-urban-engine-switch.mjs | 272 ++++-- scripts/qa-w2-imagery.mjs | 46 +- scripts/qa-weather-engine-switch.mjs | 184 +++- scripts/qa-wind-engine-switch.mjs | 88 +- scripts/qa/tileCache.mjs | 19 +- 36 files changed, 6003 insertions(+), 1774 deletions(-) create mode 100644 scripts/main-chunk-baseline.json create mode 100644 scripts/qa-strict-cold-chunks.mjs create mode 100644 scripts/qa-strict-context-loss.mjs create mode 100644 scripts/qa-strict-failure-matrix.mjs create mode 100644 scripts/qa-strict-harness.mjs create mode 100644 scripts/qa-strict-hidden-switch.mjs create mode 100644 scripts/qa-strict-rapid-switch.mjs create mode 100644 scripts/qa-strict-session.mjs create mode 100644 scripts/qa-strict-session.test.mjs create mode 100644 scripts/qa-strict-soak.mjs diff --git a/scripts/main-chunk-baseline.json b/scripts/main-chunk-baseline.json new file mode 100644 index 000000000..d584895c7 --- /dev/null +++ b/scripts/main-chunk-baseline.json @@ -0,0 +1,6 @@ +{ + "gzipBytes": 1920969, + "allowanceBytes": 19210, + "measuredAt": "a5d78b00, keyless build (GOOGLE_MAPS_API_KEY and CESIUM_ION_TOKEN empty), entry index-CV-avk6m.js, 5 933 796 bytes, gzip level 9", + "allowance": "1% of the baseline; growth past it fails npm run check:main-chunk until the baseline is raised on purpose" +} diff --git a/scripts/qa-all-layer-switches.mjs b/scripts/qa-all-layer-switches.mjs index e9c32683f..8b4cafa12 100644 --- a/scripts/qa-all-layer-switches.mjs +++ b/scripts/qa-all-layer-switches.mjs @@ -1,28 +1,65 @@ -import { execFileSync, spawnSync } from 'node:child_process'; +import { execFileSync, spawn, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import assert from 'node:assert/strict'; import puppeteer from 'puppeteer'; import { APPLICATION_LAYER_DIRECTORY } from '../src/app/layers/directory.js'; +import { + createStrict, + ENGINES, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; -// Runs every browser scenario against one keyless local preview and writes -// /acceptance.json plus one log per scenario. +// The strict acceptance: every browser scenario against one keyless local +// preview, under the strict network policy (scripts/qa-strict-harness.mjs), +// writing /acceptance.json, one log and one assertion manifest per +// scenario. +// +// node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 +// [--serve[=]] build (default HEAD) keyless in a clean +// worktree and serve it with every .env key blank +// [--scratch=DIR] where --serve puts its checkout and build +// [--only=a,b] run only these labels +// [--quick] the soak with 2 runs of 10 switches and 100 +// switches for the long run +// +// The run fails when a scenario fails, when any catalog layer (read from +// the live page and cross-checked with src/app/layers/directory.js) has no +// assertion on either engine, or when any network policy violation shows. -const base = new URL(process.argv[2]); -if ( - !['127.0.0.1', 'localhost'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw Error('Supply an unreserved keyless local preview URL'); -const output = process.argv[3] - ? resolve(process.argv[3]) - : mkdtempSync(join(tmpdir(), 'gev-layer-switch-')); -mkdirSync(output, { recursive: true }); +const base = localPreview(process.argv[2]); +const output = process.argv[3]?.startsWith('--') + ? mkdtempSync(join(tmpdir(), 'gev-acceptance-')) + : resolve(process.argv[3] ?? mkdtempSync(join(tmpdir(), 'gev-acceptance-'))); +const flag = (name) => + process.argv + .find((arg) => arg === `--${name}` || arg.startsWith(`--${name}=`)) + ?.split('=')[1] ?? (process.argv.includes(`--${name}`) ? true : null); +const only = flag('only') ? String(flag('only')).split(',') : null; +const quick = Boolean(flag('quick')); +mkdirSync(join(output, 'evidence'), { recursive: true }); +const root = fileURLToPath(new URL('..', import.meta.url)); +const git = (...args) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }); +const dirty = () => + git('status', '--porcelain') + .split('\n') + .filter(Boolean) + .map((line) => line.slice(3)); // Fail once, with the fix, instead of once per scenario. let chrome; @@ -42,6 +79,136 @@ try { ); process.exit(2); } + +// Every name the checkout's .env files define, plus key-like names in this +// environment: all blank for the build and the preview. +const envNames = () => { + const names = new Set(); + for (const file of ['.env', '.env.local']) + if (existsSync(join(root, file))) + for (const line of readFileSync(join(root, file), 'utf8').split('\n')) { + const name = line.match(/^\s*([A-Z0-9_]+)\s*=/)?.[1]; + if (name) names.add(name); + } + for (const name of Object.keys(process.env)) + if (/(_KEY|_TOKEN|_SECRET|_CLIENT_ID|PASSWORD)$/.test(name)) + names.add(name); + return [...names].sort(); +}; +const envValues = () => { + const values = []; + for (const file of ['.env', '.env.local']) + if (existsSync(join(root, file))) + for (const line of readFileSync(join(root, file), 'utf8').split('\n')) { + const value = line + .match(/^\s*[A-Z0-9_]+\s*=\s*(.*)$/)?.[1] + ?.trim() + .replace(/^["']|["']$/g, ''); + // Long enough to be a credential rather than a setting. + if (value && value.length >= 16) values.push(value); + } + return values; +}; +const scrubbed = () => ({ + ...process.env, + ...Object.fromEntries(envNames().map((name) => [name, ''])), +}); + +let served = null; +let preview = null; +const cleanup = () => { + if (preview && !preview.killed) preview.kill('SIGTERM'); + if (served?.worktree) + try { + git('worktree', 'remove', '--force', served.worktree); + } catch { + // Already gone. + } +}; +process.on('exit', cleanup); +process.on('SIGINT', () => process.exit(130)); + +if (flag('serve')) { + const commit = git( + 'rev-parse', + flag('serve') === true ? 'HEAD' : String(flag('serve')), + ).trim(); + // --scratch=DIR keeps the checkout and build out of the system temp. + const scratchRoot = flag('scratch') + ? resolve(String(flag('scratch'))) + : tmpdir(); + mkdirSync(scratchRoot, { recursive: true }); + const scratch = mkdtempSync(join(scratchRoot, 'gev-acceptance-build-')); + const worktree = join(scratch, 'checkout'); + const dist = join(scratch, 'dist'); + git('worktree', 'add', '--detach', worktree, commit); + symlinkSync(join(root, 'node_modules'), join(worktree, 'node_modules')); + served = { + commit, + worktree, + dist, + blankedEnv: envNames(), + envFilesInCheckout: ['.env', '.env.local'].filter((f) => + existsSync(join(worktree, f)), + ), + }; + console.log('Building', commit, 'keyless into', dist); + const build = spawnSync( + process.execPath, + [join(root, 'node_modules/vite/bin/vite.js'), 'build', '--outDir', dist], + { cwd: worktree, env: scrubbed(), encoding: 'utf8', maxBuffer: 1 << 26 }, + ); + if (build.status !== 0) { + console.error(build.stdout + build.stderr); + process.exit(2); + } + // No .env credential value may reach the built files. + const values = envValues(); + let leaks = 0; + const walk = (dir) => { + for (const name of readdirSync(dir)) { + const file = join(dir, name); + if (statSync(file).isDirectory()) walk(file); + else if (/\.(js|html|json|css|webmanifest)$/.test(name)) { + const text = readFileSync(file, 'utf8'); + for (const value of values) if (text.includes(value)) leaks++; + } + } + }; + walk(dist); + served.distCredentialValuesFound = leaks; + served.mainChunk = JSON.parse( + spawnSync( + process.execPath, + [join(root, 'scripts/check-main-chunk.mjs'), `--dist=${dist}`, '--json'], + { cwd: root, encoding: 'utf8' }, + ).stdout || 'null', + ); + preview = spawn( + process.execPath, + [ + join(root, 'node_modules/vite/bin/vite.js'), + 'preview', + '--outDir', + dist, + '--host', + base.hostname, + '--port', + base.port, + '--strictPort', + ], + { cwd: worktree, env: scrubbed(), stdio: 'ignore' }, + ); + for (let i = 0; i < 60; i++) { + try { + if ((await fetch(base.origin + '/')).ok) break; + } catch { + // Not up yet. + } + await new Promise((r) => setTimeout(r, 500)); + } +} + try { const response = await fetch(base.origin + '/', { signal: AbortSignal.timeout(10000), @@ -53,29 +220,56 @@ try { ); process.exit(2); } -const root = fileURLToPath(new URL('..', import.meta.url)); -const git = (...args) => - execFileSync('git', args, { cwd: root, encoding: 'utf8' }); -// Other work may change the tree while the run lasts; record both ends. -const dirty = () => - git('status', '--porcelain') - .split('\n') - .filter(Boolean) - .map((line) => line.slice(3)); + +// The preview reports no paid key: its own keyless status routes. +const probe = async (path) => { + try { + const r = await fetch(base.origin + path, { + headers: { 'Sec-Fetch-Site': 'same-origin' }, + }); + return { status: r.status, body: await r.json().catch(() => null) }; + } catch (error) { + return { error: error.message }; + } +}; +const keys = { + openaiHudSummary: await probe('/api/openai/hud-summary/status'), + firms: await probe('/api/firms/status'), + tomtom: await probe('/api/tomtom/status'), + mapillary: await probe('/api/mapillary/status'), +}; +const paidKeysConfigured = [ + keys.openaiHudSummary.body?.configured !== false && 'OpenAI', + keys.firms.body?.hasKey !== false && 'FIRMS', + keys.tomtom.body?.hasKey !== false && 'TomTom', + keys.mapillary.body?.configured !== false && 'Mapillary', +].filter(Boolean); + const dirtyFiles = dirty(); +const soakRuns = quick + ? ['--runs=2', '--swaps=10'] + : ['--runs=6', '--swaps=50']; +const longRun = quick ? '--swaps=100' : '--swaps=500'; +// label, script, layers, extra arguments, timeout in seconds. const runs = [ - ['engine-switch', ['earthquakes']], - ['director-engine-switch', []], - ['layer-handoff', ['flights']], - ['layer-handoff', ['military'], '--military'], + ['engine-switch', 'engine-switch', ['earthquakes']], + ['director-engine-switch', 'director-engine-switch', []], + ['layer-handoff', 'layer-handoff', ['flights']], + ['layer-handoff--military', 'layer-handoff', ['military'], ['--military']], [ + 'ground-layer-switch', 'ground-layer-switch', ['ais-live-vessels', 'air-quality', 'gnss-integrity', 'fire-perimeters'], ], - ['contact-switch', ['military-awareness', 'military-installations']], - ['local-adsb-engine-switch', ['local-adsb']], - ['ghostmaps-switch', ['s2-ghostmaps']], [ + 'contact-switch', + 'contact-switch', + ['military-awareness', 'military-installations'], + ], + ['local-adsb-engine-switch', 'local-adsb-engine-switch', ['local-adsb']], + ['ghostmaps-switch', 'ghostmaps-switch', ['s2-ghostmaps']], + [ + 'weather-engine-switch', 'weather-engine-switch', [ 'weather-radar', @@ -84,32 +278,113 @@ const runs = [ 'weather-cyclones', ], ], - ['wind-engine-switch', ['wind']], - ['infrastructure-engine-switch', ['local-dams', 'local-datacenters']], - ['submarine-cables-engine-switch', ['telegeography-submarine-cables']], - ['directions-engine-switch', ['directions']], - ['nepal-locator-switch', ['bhote-koshi-locator']], - ['nepal-event-switch', ['bhote-koshi-2026']], - ['traffic-engine-switch', ['traffic']], - ['traffic-engine-switch', ['traffic'], '--simulation'], - ['street-level-engine-switch', ['street-level']], - ['street-level-engine-switch', ['street-level'], '--panoramax'], - ['radio-engine-switch', ['radio']], - ['w2-imagery', ['recent-imagery']], - ['fire-camera-switch', ['alpr-cameras']], - ['fire-camera-switch', ['local-firms'], '--fires'], - ['urban-engine-switch', ['bikeshare']], - ['urban-engine-switch', ['transit'], '--transit'], - ['cctv-engine-switch', ['cctv']], - ['space-engine-switch', ['satellites']], - ['space-engine-switch', ['rocket-launches'], '--missions'], + ['wind-engine-switch', 'wind-engine-switch', ['wind']], + [ + 'infrastructure-engine-switch', + 'infrastructure-engine-switch', + ['local-dams', 'local-datacenters'], + ], + [ + 'submarine-cables-engine-switch', + 'submarine-cables-engine-switch', + ['telegeography-submarine-cables'], + ], + ['directions-engine-switch', 'directions-engine-switch', ['directions']], + ['nepal-locator-switch', 'nepal-locator-switch', ['bhote-koshi-locator']], + ['nepal-event-switch', 'nepal-event-switch', ['bhote-koshi-2026']], + ['traffic-engine-switch', 'traffic-engine-switch', ['traffic']], + [ + 'traffic-engine-switch--simulation', + 'traffic-engine-switch', + ['traffic'], + ['--simulation'], + ], + [ + 'street-level-engine-switch', + 'street-level-engine-switch', + ['street-level'], + ], + [ + 'street-level-engine-switch--panoramax', + 'street-level-engine-switch', + ['street-level'], + ['--panoramax'], + ], + ['radio-engine-switch', 'radio-engine-switch', ['radio']], + ['w2-imagery', 'w2-imagery', ['recent-imagery']], + ['fire-camera-switch', 'fire-camera-switch', ['alpr-cameras']], + [ + 'fire-camera-switch--fires', + 'fire-camera-switch', + ['local-firms'], + ['--fires'], + ], + ['urban-engine-switch', 'urban-engine-switch', ['bikeshare']], + [ + 'urban-engine-switch--transit', + 'urban-engine-switch', + ['transit'], + ['--transit'], + ], + ['cctv-engine-switch', 'cctv-engine-switch', ['cctv']], + ['space-engine-switch', 'space-engine-switch', ['satellites']], + [ + 'space-engine-switch--missions', + 'space-engine-switch', + ['rocket-launches'], + ['--missions'], + ], + // Stage 3's own checks (one application per page, voice across a switch). + // They block paid routes themselves but do not run under the strict + // harness, so they add no assertion manifest and no network ledger. + ['app-once', 'app-once', [], [], 600, true], + ['voice-engine-switch', 'voice-engine-switch', [], [], 300, true], + ['strict-rapid-switch', 'strict-rapid-switch', []], + ['strict-hidden-switch', 'strict-hidden-switch', []], + ['strict-context-loss', 'strict-context-loss', []], + ['strict-failure-matrix', 'strict-failure-matrix', [], [], 600], + ['strict-cold-chunks', 'strict-cold-chunks', []], + ['strict-soak', 'strict-soak', [], soakRuns, 1200], + [ + 'strict-soak--long', + 'strict-soak', + [], + ['--runs=1', longRun, '--assert'], + 1800, + ], ]; -const ids = [...new Set(runs.flatMap(([, ids]) => ids))].sort(); -assert.deepEqual( - ids, - APPLICATION_LAYER_DIRECTORY.map((l) => l.id).sort(), - 'Every catalog layer must have a browser scenario', -); +const scenarioIds = [...new Set(runs.flatMap(([, , ids]) => ids))].sort(); +const directoryIds = APPLICATION_LAYER_DIRECTORY.map((l) => l.id).sort(); + +// The catalog as the live page has it, on both engines. +const liveCatalog = {}; +{ + const strict = createStrict({ + origin: base.origin, + layers: [], + name: 'catalog', + }); + const browser = await strict.launch({ + args: ['--use-angle=metal', '--enable-gpu'], + }); + try { + for (const engine of ENGINES) { + const page = await browser.newPage(); + await page.goto( + `${base.origin}/?engine=${engine}&welcome=0#v=2&lat=30.27&lon=-97.74&alt=12000&pitch=-90&map=osm&l=`, + ); + await waitForEngine(page, engine); + liveCatalog[engine] = await page.evaluate(() => + [...window.__godsEyeView.dataManager.layers.keys()].sort(), + ); + await page.close(); + } + } finally { + await browser.close(); + } + liveCatalog.violations = strict.ledger.violations; +} + const scenarios = []; const report = { commit: git('rev-parse', 'HEAD').trim(), @@ -121,11 +396,41 @@ const report = { node: process.version, platform: process.platform + ' ' + process.arch, baseUrl: base.origin, + preview: served + ? { + servedByRunner: true, + appCommit: served.commit, + appCheckoutClean: true, + envFilesInCheckout: served.envFilesInCheckout, + blankedEnv: served.blankedEnv, + distCredentialValuesFound: served.distCredentialValuesFound, + } + : { servedByRunner: false }, + keyProbes: keys, + paidKeysConfigured, + mainChunk: served?.mainChunk ?? null, + networkPolicy: { + blockedFirst: + 'Google (*.googleapis.com, *.google.com, maps.google.*, *.gstatic.com), Cesium ion (*.cesium.com), preview /api/openai/*, /api/realtime/*, /api/google/*', + allowed: + 'the preview origin, each scenario’s declared fixture URLs, keyless probes answered by the harness, and the free tile hosts answered from the disk cache', + keylessProbes: (await import('./qa-map-policy.mjs')).KEYLESS_PROBES.map( + (p) => p.pattern, + ), + tileHosts: (await import('./qa-map-policy.mjs')).STRICT_TILE_HOSTS, + }, + catalog: { + directory: directoryIds, + live: liveCatalog, + scenarios: scenarioIds, + }, startedAt: new Date().toISOString(), finishedAt: null, - layers: ids, passed: 0, failed: 0, + networkViolations: 0, + coverage: null, + uncovered: null, scenarios, }; const save = () => @@ -133,26 +438,32 @@ const save = () => join(output, 'acceptance.json'), JSON.stringify(report, null, 2) + '\n', ); -for (const [name, layers, flag] of runs) { - const label = name + (flag ?? ''); +save(); + +for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { + if (only && !only.includes(label)) continue; const script = 'scripts/qa-' + name + '.mjs'; + const evidence = join(output, 'evidence', label + '.json'); + rmSync(evidence, { force: true }); const args = name === 'w2-imagery' ? [ '--url', base.origin + '/?engine=navara', '--report', - join(output, 'imagery.json'), + join(output, 'evidence', 'imagery-report.json'), '--switch', '--preview', ] - : [base.origin, ...(flag ? [flag] : [])]; + : [base.origin, ...extra]; + if (external) args.splice(0, 1, base.origin + '/'); + else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`); const start = Date.now(); const run = spawnSync(process.execPath, [join(root, script), ...args], { encoding: 'utf8', - timeout: 240000, - maxBuffer: 8388608, - env: { ...process.env, GOOGLE_MAPS_API_KEY: '', CESIUM_ION_TOKEN: '' }, + timeout: seconds * 1000, + maxBuffer: 1 << 26, + env: scrubbed(), }); const status = run.status === 0 @@ -163,34 +474,94 @@ for (const [name, layers, flag] of runs) { const log = (run.stdout || '') + (run.stderr || '') + (run.error?.message || ''); writeFileSync(join(output, label + '.log'), log); + const manifest = existsSync(evidence) + ? JSON.parse(readFileSync(evidence, 'utf8')) + : null; scenarios.push({ label, script, args: args.map((arg) => - arg.startsWith(output) ? '' + arg.slice(output.length) : arg, + arg.includes(output) ? arg.replace(output, '') : arg, ), layers, + external: Boolean(external), status, exitCode: run.status, seconds: Math.round((Date.now() - start) / 100) / 10, log: label + '.log', logSha256: createHash('sha256').update(log).digest('hex'), + evidence: manifest ? 'evidence/' + label + '.json' : null, + assertions: manifest?.assertions.length ?? 0, + coverage: manifest?.coverage ?? null, + picks: manifest?.picks ?? [], + network: manifest + ? { + requests: manifest.network.requests, + violations: manifest.network.violations, + external: manifest.network.external, + tileCache: manifest.network.tileCache, + probes: manifest.network.probes, + routes: manifest.network.routes, + previewApi: manifest.network.previewApi, + fixtures: manifest.network.fixtures, + } + : null, + contracts: manifest?.contracts ?? null, + error: manifest?.error ?? null, }); + if (manifest?.soak) scenarios.at(-1).soak = manifest.soak; report.passed = scenarios.filter((row) => row.status === 'pass').length; report.failed = scenarios.length - report.passed; + report.networkViolations = + scenarios.reduce( + (sum, row) => sum + (row.network?.violations.length ?? 0), + 0, + ) + liveCatalog.violations.length; save(); console.log(status.toUpperCase(), label, scenarios.at(-1).seconds); } + +// Coverage that cannot be faked: every catalog id needs an assertion on each +// engine from some scenario's manifest. +const catalogIds = [ + ...new Set([...directoryIds, ...(liveCatalog.navara ?? [])]), +].sort(); +report.coverage = Object.fromEntries( + catalogIds.map((id) => [ + id, + Object.fromEntries( + ENGINES.map((engine) => [ + engine, + scenarios.reduce( + (sum, row) => sum + (row.coverage?.[id]?.[engine] ?? 0), + 0, + ), + ]), + ), + ]), +); +report.uncovered = Object.entries(report.coverage).flatMap(([id, counts]) => + ENGINES.filter((engine) => !counts[engine]).map( + (engine) => `${id}@${engine}`, + ), +); +report.catalogMatches = + JSON.stringify(liveCatalog.navara) === JSON.stringify(directoryIds) && + JSON.stringify(liveCatalog.cesium) === JSON.stringify(directoryIds) && + JSON.stringify(scenarioIds) === JSON.stringify(directoryIds); report.finishedAt = new Date().toISOString(); report.dirtyFilesAtEnd = dirty(); save(); console.log( - ids.length + - ' layers, ' + - report.passed + - '/' + - runs.length + - ' browser scenarios. ' + - join(output, 'acceptance.json'), + `${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` + + `${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` + + `paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`, ); -if (report.failed) process.exitCode = 1; +if ( + report.failed || + (!only && report.uncovered.length) || + report.networkViolations || + paidKeysConfigured.length || + !report.catalogMatches +) + process.exitCode = 1; diff --git a/scripts/qa-cctv-engine-switch.mjs b/scripts/qa-cctv-engine-switch.mjs index 08bc07b9a..8608de968 100644 --- a/scripts/qa-cctv-engine-switch.mjs +++ b/scripts/qa-cctv-engine-switch.mjs @@ -1,18 +1,9 @@ import assert from 'node:assert/strict'; import { promisify } from 'node:util'; import { execFile } from 'node:child_process'; -import puppeteer from 'puppeteer'; import { installCctvFixture } from './fixtures/w2/cctv.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.2718&lon=-97.744&alt=900&pitch=-70&heading=0&map=sentinel2&l='; @@ -45,13 +36,9 @@ const { stdout: video } = await promisify(execFile)( ], { encoding: 'buffer', maxBuffer: 16000000 }, ); -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: url.origin, layers: ['cctv'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); const errors = []; p.on('pageerror', (e) => { errors.push(e.message); @@ -65,7 +52,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async () => { @@ -95,7 +82,7 @@ try { let previous = first; const count = catalogs; for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); await p.waitForFunction( (n) => window.qaCctv.getActiveVideoElement()?.getVideoPlaybackQuality() @@ -114,6 +101,16 @@ try { plane: !!window.qaCctv ._cctvRecordsForTest() .find((r) => r.camera.id === 'w2-cctv-video').projection?.planeEntity, + projection: (() => { + const r = window.qaCctv + ._cctvRecordsForTest() + .find((r) => r.camera.id === 'w2-cctv-video'); + return { + show: r.projection?.planeEntity?.show === true, + visible: r.projection?.planeVisible === true, + calibration: r.camera.calibration, + }; + })(), })); console.log( kind, @@ -128,10 +125,28 @@ try { assert.equal(v.params.selectedCameraId, 'w2-cctv-video'); assert.equal(v.params.coverageMode, 'viewshed'); assert.equal(catalogs, count); + await strict.passed( + p, + 'cctv', + 'same model and video element decoding new frames; textured projection plane; no catalog read', + ); + // The calibration patch (31 m mount, 20° pitch) and the plane's + // visibility are kept on the new engine. + const { calibration } = v.projection; + await strict.check( + p, + 'cctv', + 'calibration patch kept (31 m, 20° pitch); monitor plane visible', + v.projection.show && + v.projection.visible && + calibration?.heightM === 31 && + calibration?.pitchDeg === 20 && + v.params.calibration?.values?.heightM === 31 && + v.params.calibration?.values?.pitchDeg === 20, + v.projection, + ); previous = v.frames; } assert.deepEqual(errors, []); console.log('PASS retained CCTV catalog, decoder and textured projection'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-contact-switch.mjs b/scripts/qa-contact-switch.mjs index 2226ee420..16b40a76e 100644 --- a/scripts/qa-contact-switch.mjs +++ b/scripts/qa-contact-switch.mjs @@ -1,25 +1,17 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { fromDegrees } from '../src/geo/cartesian3.js'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l=4'; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: ['military-awareness', 'military-installations'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -43,6 +35,14 @@ try { })), }), ); + // Contacts reads air quality near the subject; no station here. + if ( + new URL(input.url ?? input, document.baseURI).hostname === + 'api.weather.gc.ca' + ) + return Promise.resolve( + Response.json({ type: 'FeatureCollection', features: [] }), + ); if (pathname === '/api/vessels/track') return Promise.resolve(Response.json({ samples: [] })); if (pathname === '/api/flights') @@ -51,16 +51,48 @@ try { ); if (pathname === '/api/military') return Promise.resolve(Response.json({ now: Date.now(), ac: [] })); - if (pathname === '/api/military-installations') - return Promise.resolve(Response.json({ elements: [], status: 'ok' })); + // One mapped station next to the Contacts subject, with a footprint. + if (pathname === '/api/military-installations') { + window.qaInstallationReads = (window.qaInstallationReads ?? 0) + 1; + const ring = [ + [43.665, -79.345], + [43.665, -79.315], + [43.68, -79.315], + [43.68, -79.345], + [43.665, -79.345], + ]; + return Promise.resolve( + Response.json({ + status: 'ok', + elements: [ + { + type: 'way', + id: 424242, + tags: { military: 'naval_base', name: 'QA Harbour Station' }, + center: { lat: 43.6725, lon: -79.33 }, + geometry: ring.map(([lat, lon]) => ({ lat, lon })), + }, + ], + }), + ); + } return fetch(input, options); }; + window.qaSelected = []; + for (const type of ['gev:entity-selected', 'gev:entity-selection-cleared']) + window.addEventListener(type, ({ detail }) => + window.qaSelected.push({ + type, + id: detail?.id, + layerId: detail?.layerId, + }), + ); }); await p.goto(base.href); const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -110,14 +142,100 @@ try { : null, }; }); + // The station zone: the fixture's installation is the one mapped site + // within the Contacts radius of the subject, and the engine draws it where + // a pointer can reach it (the engine's own picker returns it at an + // uncovered canvas point at its marker or inside its footprint). + const inside = [fromDegrees(-79.33, 43.6725)]; + for (let i = 1; i < 6; i++) + for (let k = 1; k < 6; k++) + inside.push(fromDegrees(-79.345 + 0.005 * k, 43.665 + 0.0025 * i)); + const zone = async () => { + await p.waitForFunction( + () => + !window.qaSites.getStats().loading && + window.qaSites + .getAnalystRecords() + .some((r) => r.name === 'QA Harbour Station'), + { timeout: 30000 }, + ); + const drawn = await p + .waitForFunction( + (points) => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (const world of points) { + const centre = engine.projector.worldToWindow(world, {}); + if (!centre) continue; + for (const [dx, dy] of [ + [0, 0], + [0, -8], + [8, 0], + [-8, 0], + [0, 8], + ]) { + const at = { x: centre.x + dx, y: centre.y + dy }; + if ( + !canvas.contains( + document.elementFromPoint(box.left + at.x, box.top + at.y), + ) + ) + continue; + const id = engine.picker.pick(at)?.id; + if (String(id?.id ?? id) === 'osm:way:424242') + return [Math.round(at.x), Math.round(at.y)]; + } + } + return false; + }, + { timeout: 20000 }, + inside, + ) + .then((handle) => handle.jsonValue()) + .catch(() => null); + return p.evaluate( + (drawn) => ({ + drawn, + records: window.qaSites.getAnalystRecords().map((r) => r.name), + coverage: window.qaSites.getStats().coverage, + count: window.qaSites.getStats().count, + reads: window.qaInstallationReads, + subject: window.qaContext.getContextSnapshot()?.subject?.id, + }), + drawn, + ); + }; + const checkZone = async () => { + const station = await zone(); + await strict.check( + p, + 'military-installations', + 'the station zone holds the fixture station, drawn and pickable', + station.count === 1 && + station.records.length === 1 && + station.records[0] === 'QA Harbour Station' && + station.coverage?.kind === 'subject' && + Array.isArray(station.drawn), + station, + ); + return station; + }; + const firstZone = await checkZone(); const baseline = await sample(); console.log('baseline', JSON.stringify(baseline)); assert.equal(baseline.mode, 'flights'); assert.equal(baseline.subject, '223456789'); assert.ok(baseline.history.includes('ais-live-vessels:123456789')); assert.ok(baseline.ring >= 1); + await strict.passed( + p, + 'military-awareness', + 'Contacts subject, navigation history and awareness rim', + baseline, + ); for (const kind of ['cesium', 'navara']) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); const current = await sample(); console.log('handoff', kind, JSON.stringify(current)); @@ -125,7 +243,26 @@ try { ...baseline, ring: kind === 'navara' ? baseline.ring : null, }); + await strict.passed( + p, + 'military-awareness', + 'same awareness model, Contacts mode, subject, history and enabled layers', + ); + await strict.passed( + p, + 'military-installations', + 'same installations model and enabled through the switch', + ); + const station = await checkZone(); + await strict.check( + p, + 'military-installations', + 'the switch reads no installation snapshot', + station.reads === firstZone.reads, + { before: firstZone.reads, after: station.reads }, + ); } + await checkZone(); const restored = await p.evaluate(async () => { const { dataManager, styleManager } = window.__godsEyeView; const result = await styleManager.setContextMode('off'); @@ -143,6 +280,4 @@ try { console.log( 'PASS Contacts history, dependency ownership and exit restore across both globes', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-directions-engine-switch.mjs b/scripts/qa-directions-engine-switch.mjs index 6fb95e802..f1ce4bdc5 100644 --- a/scripts/qa-directions-engine-switch.mjs +++ b/scripts/qa-directions-engine-switch.mjs @@ -1,25 +1,13 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=55.3241&lon=10.7995&alt=25000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['directions'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -70,7 +58,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -98,7 +86,12 @@ try { await new Promise((resolve) => setTimeout(resolve, 450)); } assert.equal(routeReads, 1); - await p.select('#globe-engine', 'cesium'); + await strict.passed( + p, + 'directions', + 'A and B set by clicks on the globe; one route request', + ); + await strict.pick(p, 'cesium'); await ready('cesium'); release(); await p.waitForFunction(() => window.qaRoute.getStats().count === 3); @@ -106,8 +99,13 @@ try { window.qaRoute.setParams({ step: 1 }); return window.qaRoute.getRowControls().list.items[1]?.active; }); + await strict.passed( + p, + 'directions', + 'the route asked for on Navara arrives on Cesium; a step is selectable', + ); const transfer = async (kind) => { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); assert.equal( await p.evaluate( @@ -118,6 +116,11 @@ try { true, ); assert.equal(routeReads, 1); + await strict.passed( + p, + 'directions', + 'same model; the switch requests no route', + ); }; await transfer('navara'); assert.equal( @@ -153,13 +156,51 @@ try { progress = resumed; console.log(kind, 'moved', d(at, next), 'handoff', d(at, before)); assert.ok(d(at, next) > 1, 'dolly advances after switch to ' + kind); + await strict.passed( + p, + 'directions', + 'FLY continues with the same motion and advances', + { moved: d(at, next) }, + ); } await p.evaluate(() => window.qaRoute.setParams({ clear: true })); assert.equal(await p.evaluate(() => window.qaRoute.getStats().count), 0); + // CLEAR releases the camera: the route's motion ends, the camera stays + // where it is, and a wheel zoom by the person moves it and is kept. + await new Promise((resolve) => setTimeout(resolve, 500)); + const released = await p.evaluate(() => ({ + motion: window.qaRoute.getFlightProgress(), + fly: window.qaRoute.getRowControls().chips.find((c) => c.id === 'fly') + ?.active, + })); + const still = await pos(); + await new Promise((resolve) => setTimeout(resolve, 1500)); + const later = await pos(); + const height = () => + p.evaluate(() => window.__godsEyeView.engine.camera.height()); + const beforeZoom = await height(); + await p.mouse.move(512, 300); + for (let i = 0; i < 4; i++) { + await p.mouse.wheel({ deltaY: -240 }); + await new Promise((resolve) => setTimeout(resolve, 120)); + } + await new Promise((resolve) => setTimeout(resolve, 1500)); + const zoomed = await height(); + await new Promise((resolve) => setTimeout(resolve, 1500)); + const kept = await height(); + await strict.check( + p, + 'directions', + 'CLEAR removes the route and releases the camera: no route motion, the camera stays put, a wheel zoom moves it and is kept', + released.motion === null && + !released.fly && + d(still, later) < 1 && + zoomed < beforeZoom * 0.95 && + Math.abs(kept - zoomed) < 0.02 * zoomed, + { released, drift: d(still, later), beforeZoom, zoomed, kept }, + ); assert.deepEqual(errors, []); console.log( 'PASS route delivery during handoff, A/B clicks, step selection, ongoing FLY, clear and single route request', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-director-engine-switch.mjs b/scripts/qa-director-engine-switch.mjs index ca74a18ff..947cd8e24 100644 --- a/scripts/qa-director-engine-switch.mjs +++ b/scripts/qa-director-engine-switch.mjs @@ -1,17 +1,8 @@ import assert from 'node:assert/strict'; import sharp from 'sharp'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { fromDegrees } from '../src/geo/cartesian3.js'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +const url = localPreview(process.argv[2]); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.2672&lon=-97.742&alt=4000&pitch=-90&heading=0&map=sentinel2&l='; @@ -137,13 +128,15 @@ const project = { }, ], }; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Scene Director is not a catalog layer; the scenario answers the scene's +// earthquake feed itself. +const strict = createStrict({ + origin: url.origin, + layers: [], + fixtures: ['https://earthquake.usgs.gov/*'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => { @@ -223,7 +216,7 @@ try { }, world); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) - await switchGlobe(p, kind); + await strict.switchTo(p, kind); await new Promise((r) => setTimeout(r, 500)); const st = await p.evaluate(() => ({ same: window.director === window.__godsEyeView.sceneDirector, @@ -290,7 +283,7 @@ try { window.director.running && window.director.getPlaybackTimingState().snapshot?.shotId === 'one', ); - await switchGlobe(p, 'cesium'); + await strict.switchTo(p, 'cesium'); assert.ok( await p.evaluate( () => @@ -302,13 +295,11 @@ try { () => window.director.getPlaybackTimingState().snapshot?.shotId === 'two', { timeout: 20000 }, ); - await switchGlobe(p, 'navara'); + await strict.switchTo(p, 'navara'); const complete = await p.evaluate(() => window.qaRun); assert.ok(complete.started && complete.shots === 2); assert.deepEqual(errors, []); console.log( 'PASS authored packs, geographic picks, same player, original multi-shot timeline', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-engine-switch.mjs b/scripts/qa-engine-switch.mjs index b1f886651..a69b05e69 100644 --- a/scripts/qa-engine-switch.mjs +++ b/scripts/qa-engine-switch.mjs @@ -1,459 +1,484 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { installTileCache } from './qa/tileCache.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) { - throw new Error('Supply an unreserved local preview URL'); -} +const base = localPreview(process.argv[2]); base.search = '?welcome=0&look=cinematic'; base.hash = 'v=2&lat=42.3554&lon=-71.0605&alt=6000&heading=0&pitch=-90&map=osm&l=e'; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - await installTileCache(page); - await page.evaluateOnNewDocument(() => { - localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); - const fetch = window.fetch.bind(window); - // Two Boston vessels for the Contacts session; empty air and site feeds. - const fixtures = { - '/api/vessels': () => ({ - status: 'live', - rows: ['123456789', '223456789'].map((mmsi, i) => ({ - mmsi, - name: 'Boston ' + i, - lat: 42.3554, - lon: -71.0605 + i * 0.05, - last_position_epoch: Date.now() / 1000, - })), - }), - '/api/vessels/track': () => ({ samples: [] }), - '/api/flights': () => ({ time: Date.now() / 1000, states: [] }), - '/api/military': () => ({ now: Date.now(), ac: [] }), - '/api/military-installations': () => ({ elements: [], status: 'ok' }), - }; - window.fetch = (url, options) => { - if ( - String(url).startsWith('https://earthquake.usgs.gov/earthquakes/feed/') - ) - return Promise.resolve( - Response.json({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - id: 'qa-boston', - geometry: { - type: 'Point', - coordinates: [-71.0605, 42.3554, 8], +const strict = createStrict({ origin: base.origin, layers: ['earthquakes'] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + await page.evaluateOnNewDocument(() => { + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + const fetch = window.fetch.bind(window); + // Two Boston vessels for the Contacts session; empty air and site feeds. + const fixtures = { + '/api/vessels': () => ({ + status: 'live', + rows: ['123456789', '223456789'].map((mmsi, i) => ({ + mmsi, + name: 'Boston ' + i, + lat: 42.3554, + lon: -71.0605 + i * 0.05, + last_position_epoch: Date.now() / 1000, + })), + }), + '/api/vessels/track': () => ({ samples: [] }), + '/api/flights': () => ({ time: Date.now() / 1000, states: [] }), + '/api/military': () => ({ now: Date.now(), ac: [] }), + '/api/military-installations': () => ({ elements: [], status: 'ok' }), + }; + window.fetch = (url, options) => { + if ( + String(url).startsWith( + 'https://earthquake.usgs.gov/earthquakes/feed/', + ) + ) + return Promise.resolve( + Response.json({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'qa-boston', + geometry: { + type: 'Point', + coordinates: [-71.0605, 42.3554, 8], + }, + properties: { mag: 4, time: Date.now(), place: 'QA Boston' }, }, - properties: { mag: 4, time: Date.now(), place: 'QA Boston' }, - }, - ], - }), - ); - const fixture = - fixtures[new URL(url.url ?? url, document.baseURI).pathname]; - return fixture - ? Promise.resolve(Response.json(fixture())) - : fetch(url, options); - }; - }); - await page.setViewport({ width: 1280, height: 800 }); - const errors = []; - page.on('pageerror', (error) => errors.push(error.message)); - async function ready(kind) { - await page.waitForFunction( - (kind) => - window.__godsEyeView?.engine?.kind === kind && - document.querySelector('#globe-engine')?.value === kind && - !document.querySelector('#globe-engine').disabled && - document.getElementById('loading-screen')?.classList.contains('hidden'), - { timeout: 90000 }, - kind, - ); - await page.evaluate(async () => { - await window.__godsEyeView.styleManager.initialRestorePromise; - if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) - throw new Error('Keyless preview required'); + ], + }), + ); + const fixture = + fixtures[new URL(url.url ?? url, document.baseURI).pathname]; + return fixture + ? Promise.resolve(Response.json(fixture())) + : fetch(url, options); + }; }); - assert.equal(await page.$eval('#globe-engine', (el) => el.value), kind); - } - async function verifyQuakeSurface(kind) { - const meanRed = async (enabled) => { - await page.evaluate(async (enabled) => { - const a = window.__godsEyeView; - await a.dataManager.setEnabled('earthquakes', enabled, { - origin: 'user', - }); - a.engine.render.request('qa-quake-surface'); - }, enabled); - if (enabled) - await page.waitForFunction( - () => - window.__godsEyeView.dataManager.layers - .get('earthquakes') - .module.getAnalystRecords().length === 1, - ); - await new Promise((resolve) => setTimeout(resolve, 1800)); - const bytes = await page.screenshot({ - clip: { x: 675, y: 403, width: 20, height: 20 }, + await page.setViewport({ width: 1280, height: 800 }); + const errors = []; + page.on('pageerror', (error) => errors.push(error.message)); + async function ready(kind) { + await page.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + document.querySelector('#globe-engine')?.value === kind && + !document.querySelector('#globe-engine').disabled && + document + .getElementById('loading-screen') + ?.classList.contains('hidden'), + { timeout: 90000 }, + kind, + ); + await page.evaluate(async () => { + await window.__godsEyeView.styleManager.initialRestorePromise; + if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) + throw new Error('Keyless preview required'); }); - const { channels } = await sharp(bytes).stats(); - return channels[0].mean - channels[1].mean; - }; - const off = await meanRed(false), - on = await meanRed(true); - assert.ok( - on > off + 8, - kind + - ' must draw the shallow red earthquake footprint: ' + - off + - ' -> ' + - on, - ); - console.log( - 'PASS', - kind, - 'earthquake footprint red minus green:', - off, - '->', - on, - ); - } - const stored = () => page.evaluate(() => localStorage.getItem('gev.engine')); - async function switchTo(kind) { - if ( - await page.$eval( - '#control-panel-toggle', - (el) => el.getAttribute('aria-expanded') !== 'true', - ) - ) { - await page.focus('#control-panel-toggle'); - await page.keyboard.press('Enter'); + assert.equal(await page.$eval('#globe-engine', (el) => el.value), kind); } - await page.waitForSelector('#globe-engine', { visible: true }); - await page.evaluate(() => { - window.__switchDocument = document; - window.__quakeEntry = - window.__godsEyeView.dataManager.layers.get('earthquakes'); - window.__quakeTimer = window.__quakeEntry.intervalId; - window.__renderers ??= new Map(); - window.__renderers.set( - window.__godsEyeView.engine.kind, - window.__godsEyeView.engine, + async function verifyQuakeSurface(kind) { + const meanRed = async (enabled) => { + await page.evaluate(async (enabled) => { + const a = window.__godsEyeView; + await a.dataManager.setEnabled('earthquakes', enabled, { + origin: 'user', + }); + a.engine.render.request('qa-quake-surface'); + }, enabled); + if (enabled) + await page.waitForFunction( + () => + window.__godsEyeView.dataManager.layers + .get('earthquakes') + .module.getAnalystRecords().length === 1, + ); + await new Promise((resolve) => setTimeout(resolve, 1800)); + const bytes = await page.screenshot({ + clip: { x: 675, y: 403, width: 20, height: 20 }, + }); + const { channels } = await sharp(bytes).stats(); + return channels[0].mean - channels[1].mean; + }; + const off = await meanRed(false), + on = await meanRed(true); + assert.ok( + on > off + 8, + kind + + ' must draw the shallow red earthquake footprint: ' + + off + + ' -> ' + + on, ); - window.__switchSplash = false; - window.__switchObserver = new MutationObserver(() => { - if (!document.querySelector('#loading-screen.hidden')) - window.__switchSplash = true; + await strict.passed(page, 'earthquakes', 'shallow red footprint drawn', { + off, + on, }); - window.__switchObserver.observe( - document.querySelector('#loading-screen'), - { attributes: true }, + console.log( + 'PASS', + kind, + 'earthquake footprint red minus green:', + off, + '->', + on, ); - window.__outgoingEngine = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', kind); - await ready(kind); - assert.ok(await page.evaluate(() => window.__switchDocument === document)); - assert.ok( + } + const stored = () => + page.evaluate(() => localStorage.getItem('gev.engine')); + async function switchTo(kind) { + if ( + await page.$eval( + '#control-panel-toggle', + (el) => el.getAttribute('aria-expanded') !== 'true', + ) + ) { + await page.focus('#control-panel-toggle'); + await page.keyboard.press('Enter'); + } + await page.waitForSelector('#globe-engine', { visible: true }); await page.evaluate(() => { - window.__switchObserver.disconnect(); - const { engine } = window.__godsEyeView; - const previous = window.__renderers.get(engine.kind); - return ( - !window.__switchSplash && - window.__quakeEntry === - window.__godsEyeView.dataManager.layers.get('earthquakes') && - window.__quakeTimer === window.__quakeEntry.intervalId && - (!previous || previous === engine) && - !window.__outgoingEngine.isDestroyed() && - !window.__outgoingEngine.host.canvas.isConnected && - engine.host.canvas.isConnected && - window.__godsEyeView.dataManager - .getEnabledLayerIds() - .has('earthquakes') + window.__switchDocument = document; + window.__quakeEntry = + window.__godsEyeView.dataManager.layers.get('earthquakes'); + window.__quakeTimer = window.__quakeEntry.intervalId; + window.__renderers ??= new Map(); + window.__renderers.set( + window.__godsEyeView.engine.kind, + window.__godsEyeView.engine, ); + window.__switchSplash = false; + window.__switchObserver = new MutationObserver(() => { + if (!document.querySelector('#loading-screen.hidden')) + window.__switchSplash = true; + }); + window.__switchObserver.observe( + document.querySelector('#loading-screen'), + { attributes: true }, + ); + window.__outgoingEngine = window.__godsEyeView.engine; + }); + await strict.pick(page, kind); + await ready(kind); + assert.ok( + await page.evaluate(() => window.__switchDocument === document), + ); + assert.ok( + await page.evaluate(() => { + window.__switchObserver.disconnect(); + const { engine } = window.__godsEyeView; + const previous = window.__renderers.get(engine.kind); + return ( + !window.__switchSplash && + window.__quakeEntry === + window.__godsEyeView.dataManager.layers.get('earthquakes') && + window.__quakeTimer === window.__quakeEntry.intervalId && + (!previous || previous === engine) && + !window.__outgoingEngine.isDestroyed() && + !window.__outgoingEngine.host.canvas.isConnected && + engine.host.canvas.isConnected && + window.__godsEyeView.dataManager + .getEnabledLayerIds() + .has('earthquakes') + ); + }), + ); + await strict.passed( + page, + 'earthquakes', + 'same layer entry, model and poll timer; no splash; outgoing canvas detached', + ); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + assert.equal( + await page.evaluate(async () => { + const parked = window.__outgoingEngine; + let frames = 0; + const remove = parked.frames.postRender.addEventListener(() => { + frames++; + }); + parked.render.request('qa-parked'); + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(resolve)), + ); + remove(); + return frames; + }), + 0, + ); + const url = new URL(page.url()); + assert.equal(url.searchParams.get('engine'), kind); + assert.equal(url.searchParams.get('look'), 'cinematic'); + const hash = new URLSearchParams(url.hash.slice(1)); + assert.equal(hash.get('map'), 'osm'); + assert.ok(Math.abs(Number(hash.get('lat')) - 42.3554) < 0.01); + assert.ok(Math.abs(Number(hash.get('lon')) + 71.0605) < 0.01); + assert.ok(Math.abs(Number(hash.get('alt')) - 6000) < 30); + const view = await page.evaluate(() => { + const { engine, mapStackController } = window.__godsEyeView; + const { latitude, longitude, height } = + engine.camera.pose().cartographic; + return { + latitude: latitude * (180 / Math.PI), + longitude: longitude * (180 / Math.PI), + height, + map: mapStackController.getActiveId(), + }; + }); + assert.equal(view.map, 'osm'); + assert.ok(Math.abs(view.latitude - 42.3554) < 0.01); + assert.ok(Math.abs(view.longitude + 71.0605) < 0.01); + assert.ok(Math.abs(view.height - 6000) < 30); + await strict.passed( + page, + 'earthquakes', + 'parked engine draws no frame; pose, map and layers carried over', + ); + } + await page.goto(base.href, { waitUntil: 'domcontentloaded' }); + await ready('cesium'); + await verifyQuakeSurface('cesium'); + const sharpen = () => + page.evaluate( + () => window.__godsEyeView.styleManager.getControlState().sharpen, + ); + await page.evaluate(() => { + const style = window.__godsEyeView.styleManager; + style.setStyle('retro'); + style.setStyle('normal'); + }); + assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 49 }); + await switchTo('navara'); + await verifyQuakeSurface('navara'); + assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 80 }); + assert.ok( + await page.evaluate( + () => + Math.abs(window.__godsEyeView.engine.render.sharpen().amount - 1.7) < + 1e-6, + ), + ); + assert.equal(await stored(), 'navara'); + await page.goBack({ waitUntil: 'domcontentloaded' }); + await ready('cesium'); + assert.equal((await sharpen()).intensityPct, 49); + await page.goForward(); + await ready('navara'); + assert.equal((await sharpen()).intensityPct, 80); + await page.evaluate(() => + window.__godsEyeView.styleManager.setSharpen({ + enabled: false, + intensityPct: 49, }), ); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), + await switchTo('cesium'); + await switchTo('navara'); + assert.deepEqual(await sharpen(), { enabled: false, intensityPct: 49 }); + // The address carries the running layer set once its debounced write lands. + const settledAddress = async () => { + let previous; + for (;;) { + await new Promise((resolve) => setTimeout(resolve, 1200)); + const current = await page.evaluate(() => + new URLSearchParams(location.hash.slice(1)).get('l'), + ); + if (current === previous) return current; + previous = current; + } + }; + const pick = async (kind) => { + await strict.pick(page, kind); + await ready(kind); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + }; + const visit = async (direction, kind) => { + await (direction === 'back' + ? page.goBack({ waitUntil: 'domcontentloaded' }) + : page.goForward()); + await ready(kind); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + }; + const quakes = () => + page.evaluate(() => { + const { dataManager } = window.__godsEyeView; + return { + model: window.__quakeModel === dataManager.layers.get('earthquakes'), + enabled: dataManager.getEnabledLayerIds().has('earthquakes'), + }; + }); + // Like a same-engine Back, each entry brings back its own layer set, on + // the same layer models. + await page.evaluate(() => { + window.__quakeModel = + window.__godsEyeView.dataManager.layers.get('earthquakes'); + }); + const navaraLayers = await settledAddress(); + await pick('cesium'); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('earthquakes', false, { + origin: 'user', + }), + ); + const cesiumLayers = await settledAddress(); + assert.notEqual(cesiumLayers, navaraLayers); + await visit('back', 'navara'); + assert.deepEqual(await quakes(), { model: true, enabled: true }); + await strict.passed( + page, + 'earthquakes', + 'Back restores the enabled layer on the same model', + ); + assert.equal(await settledAddress(), navaraLayers, 'Back keeps its entry'); + await visit('forward', 'cesium'); + assert.deepEqual(await quakes(), { model: true, enabled: false }); + await strict.passed( + page, + 'earthquakes', + 'Forward restores the disabled layer on the same model', ); assert.equal( - await page.evaluate(async () => { - const parked = window.__outgoingEngine; - let frames = 0; - const remove = parked.frames.postRender.addEventListener(() => { - frames++; - }); - parked.render.request('qa-parked'); - await new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(resolve)), - ); - remove(); - return frames; + await settledAddress(), + cesiumLayers, + 'Forward keeps its entry', + ); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('earthquakes', true, { + origin: 'user', }), - 0, ); - const url = new URL(page.url()); - assert.equal(url.searchParams.get('engine'), kind); - assert.equal(url.searchParams.get('look'), 'cinematic'); - const hash = new URLSearchParams(url.hash.slice(1)); - assert.equal(hash.get('map'), 'osm'); - assert.ok(Math.abs(Number(hash.get('lat')) - 42.3554) < 0.01); - assert.ok(Math.abs(Number(hash.get('lon')) + 71.0605) < 0.01); - assert.ok(Math.abs(Number(hash.get('alt')) - 6000) < 30); - const view = await page.evaluate(() => { - const { engine, mapStackController } = window.__godsEyeView; - const { latitude, longitude, height } = engine.camera.pose().cartographic; - return { - latitude: latitude * (180 / Math.PI), - longitude: longitude * (180 / Math.PI), - height, - map: mapStackController.getActiveId(), - }; - }); - assert.equal(view.map, 'osm'); - assert.ok(Math.abs(view.latitude - 42.3554) < 0.01); - assert.ok(Math.abs(view.longitude + 71.0605) < 0.01); - assert.ok(Math.abs(view.height - 6000) < 30); - } - await page.goto(base.href, { waitUntil: 'domcontentloaded' }); - await ready('cesium'); - await verifyQuakeSurface('cesium'); - const sharpen = () => - page.evaluate( - () => window.__godsEyeView.styleManager.getControlState().sharpen, + await pick('navara'); + console.log( + "PASS Back/Forward restore each entry's layers on the same models", ); - await page.evaluate(() => { - const style = window.__godsEyeView.styleManager; - style.setStyle('retro'); - style.setStyle('normal'); - }); - assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 49 }); - await switchTo('navara'); - await verifyQuakeSurface('navara'); - assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 80 }); - assert.ok( - await page.evaluate( - () => - Math.abs(window.__godsEyeView.engine.render.sharpen().amount - 1.7) < - 1e-6, - ), - ); - assert.equal(await stored(), 'navara'); - await page.goBack({ waitUntil: 'domcontentloaded' }); - await ready('cesium'); - assert.equal((await sharpen()).intensityPct, 49); - await page.goForward(); - await ready('navara'); - assert.equal((await sharpen()).intensityPct, 80); - await page.evaluate(() => - window.__godsEyeView.styleManager.setSharpen({ - enabled: false, - intensityPct: 49, - }), - ); - await switchTo('cesium'); - await switchTo('navara'); - assert.deepEqual(await sharpen(), { enabled: false, intensityPct: 49 }); - // The address carries the running layer set once its debounced write lands. - const settledAddress = async () => { - let previous; - for (;;) { - await new Promise((resolve) => setTimeout(resolve, 1200)); - const current = await page.evaluate(() => - new URLSearchParams(location.hash.slice(1)).get('l'), - ); - if (current === previous) return current; - previous = current; - } - }; - const pick = async (kind) => { - await page.select('#globe-engine', kind); - await ready(kind); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), - ); - }; - const visit = async (direction, kind) => { - await (direction === 'back' - ? page.goBack({ waitUntil: 'domcontentloaded' }) - : page.goForward()); - await ready(kind); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), - ); - }; - const quakes = () => - page.evaluate(() => { - const { dataManager } = window.__godsEyeView; - return { - model: window.__quakeModel === dataManager.layers.get('earthquakes'), - enabled: dataManager.getEnabledLayerIds().has('earthquakes'), - }; - }); - // Like a same-engine Back, each entry brings back its own layer set, on - // the same layer models. - await page.evaluate(() => { - window.__quakeModel = - window.__godsEyeView.dataManager.layers.get('earthquakes'); - }); - const navaraLayers = await settledAddress(); - await pick('cesium'); - await page.evaluate(() => - window.__godsEyeView.dataManager.setEnabled('earthquakes', false, { - origin: 'user', - }), - ); - const cesiumLayers = await settledAddress(); - assert.notEqual(cesiumLayers, navaraLayers); - await visit('back', 'navara'); - assert.deepEqual(await quakes(), { model: true, enabled: true }); - assert.equal(await settledAddress(), navaraLayers, 'Back keeps its entry'); - await visit('forward', 'cesium'); - assert.deepEqual(await quakes(), { model: true, enabled: false }); - assert.equal(await settledAddress(), cesiumLayers, 'Forward keeps its entry'); - await page.evaluate(() => - window.__godsEyeView.dataManager.setEnabled('earthquakes', true, { - origin: 'user', - }), - ); - await pick('navara'); - console.log( - "PASS Back/Forward restore each entry's layers on the same models", - ); - // Back/Forward hand the session off like the picker: the Contacts subject - // and its navigation history, its layers and the Director survive. - await page.evaluate(async () => { - const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; - await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user' }); - const outcome = await styleManager.setContextMode('contacts'); - if (!outcome.ok) throw Error(JSON.stringify(outcome)); - window.qaSession = { - director: sceneDirector, - context: dataManager.layers.get('military-awareness').module, - vessels: dataManager.layers.get('ais-live-vessels'), - }; - if (!window.qaSession.vessels.module.selectById('123456789')) - throw Error('Missing first vessel'); - if ( - !window.qaSession.context.navigateNext({ - targetLayer: 'ais-live-vessels', - }) - ) - throw Error('Cannot navigate to the second vessel'); - }); - const session = () => - page.evaluate(() => { - const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; - const { director, context, vessels } = window.qaSession; - return { - director: director === sceneDirector && !director._destroyed, - owners: - context === dataManager.layers.get('military-awareness').module && - vessels === dataManager.layers.get('ais-live-vessels'), - mode: styleManager.getContextModeState().mode, - subject: context.getContextSnapshot()?.subject?.id, - history: context._getAwarenessNavigationStateForTest().historyKeys, - enabled: [...dataManager.getEnabledLayerIds()].sort(), - }; - }); - const live = await session(); - assert.equal(live.mode, 'flights'); - assert.equal(live.subject, '223456789'); - assert.ok(live.history.includes('ais-live-vessels:123456789')); - await settledAddress(); - const historyLength = await page.evaluate(() => history.length); - await pick('cesium'); - assert.deepEqual(await session(), live, 'a pick keeps the session'); - await settledAddress(); - await visit('back', 'navara'); - assert.deepEqual(await session(), live, 'Back keeps the session'); - await visit('forward', 'cesium'); - assert.deepEqual(await session(), live, 'Forward keeps the session'); - assert.equal(await page.evaluate(() => history.length), historyLength + 1); - assert.ok( + // Back/Forward hand the session off like the picker: the Contacts subject + // and its navigation history, its layers and the Director survive. await page.evaluate(async () => { - const { dataManager, styleManager } = window.__godsEyeView; - const result = await styleManager.setContextMode('off'); - await dataManager.setEnabled('ais-live-vessels', false, { + const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; + await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user', }); - return result.ok; - }), - ); - await pick('navara'); - console.log( - 'PASS Back/Forward keep Contacts subject, history, layers and Director', - ); - await page.goto(base.href, { waitUntil: 'domcontentloaded' }); - await ready('navara'); - await switchTo('cesium'); - assert.equal(await stored(), 'cesium'); - await page.evaluate(() => { - const write = Storage.prototype.setItem; - Storage.prototype.setItem = function (key, value) { - if (key === 'gev.engine') - throw new DOMException('Blocked', 'SecurityError'); - return write.call(this, key, value); - }; - }); - await switchTo('navara'); - assert.equal(await stored(), 'cesium'); - // The real Map Source chips move into daylight inspection and back to live lighting. - await page.$eval('[data-stack-id="sentinel2"]', (el) => el.click()); - await page.waitForFunction( - () => - window.__godsEyeView.mapStackController.getActiveId() === 'sentinel2' && - window.__godsEyeView.engine.host.container.hasAttribute( - 'data-terrain-daylight', - ), - ); - // Zooming out on the satellite map must keep the same inspection sun. - assert.ok( - await page.evaluate(async () => { - const { engine } = window.__godsEyeView; - engine.camera.zoomOut(2_000_000); - await new Promise((resolve) => { - const remove = engine.frames.postRender.addEventListener(() => { - remove(); - resolve(); - }); - engine.render.request('qa-orbital-daylight'); + const outcome = await styleManager.setContextMode('contacts'); + if (!outcome.ok) throw Error(JSON.stringify(outcome)); + window.qaSession = { + director: sceneDirector, + context: dataManager.layers.get('military-awareness').module, + vessels: dataManager.layers.get('ais-live-vessels'), + }; + if (!window.qaSession.vessels.module.selectById('123456789')) + throw Error('Missing first vessel'); + if ( + !window.qaSession.context.navigateNext({ + targetLayer: 'ais-live-vessels', + }) + ) + throw Error('Cannot navigate to the second vessel'); + }); + const session = () => + page.evaluate(() => { + const { dataManager, styleManager, sceneDirector } = + window.__godsEyeView; + const { director, context, vessels } = window.qaSession; + return { + director: director === sceneDirector && !director._destroyed, + owners: + context === dataManager.layers.get('military-awareness').module && + vessels === dataManager.layers.get('ais-live-vessels'), + mode: styleManager.getContextModeState().mode, + subject: context.getContextSnapshot()?.subject?.id, + history: context._getAwarenessNavigationStateForTest().historyKeys, + enabled: [...dataManager.getEnabledLayerIds()].sort(), + }; }); - return ( - engine.camera.height() > 1_500_000 && - engine.host.container.hasAttribute('data-terrain-daylight') - ); - }), - 'the orbital frame keeps daylight on the inspected globe', - ); - await page.$eval('[data-stack-id="osm"]', (el) => el.click()); - await page.waitForFunction( - () => - window.__godsEyeView.mapStackController.getActiveId() === 'osm' && - !window.__godsEyeView.engine.host.container.hasAttribute( - 'data-terrain-daylight', - ), - ); - assert.deepEqual(errors, []); - console.log( - 'PASS Cesium default, both switches, same document, warm engine identity, parked canvas, no startup splash, share pose/map/layers, Back/Forward, remembered engine, blocked preference storage', - ); -} finally { - await browser.close(); -} + const live = await session(); + assert.equal(live.mode, 'flights'); + assert.equal(live.subject, '223456789'); + assert.ok(live.history.includes('ais-live-vessels:123456789')); + await settledAddress(); + const historyLength = await page.evaluate(() => history.length); + await pick('cesium'); + assert.deepEqual(await session(), live, 'a pick keeps the session'); + await settledAddress(); + await visit('back', 'navara'); + assert.deepEqual(await session(), live, 'Back keeps the session'); + await visit('forward', 'cesium'); + assert.deepEqual(await session(), live, 'Forward keeps the session'); + assert.equal(await page.evaluate(() => history.length), historyLength + 1); + assert.ok( + await page.evaluate(async () => { + const { dataManager, styleManager } = window.__godsEyeView; + const result = await styleManager.setContextMode('off'); + await dataManager.setEnabled('ais-live-vessels', false, { + origin: 'user', + }); + return result.ok; + }), + ); + await pick('navara'); + console.log( + 'PASS Back/Forward keep Contacts subject, history, layers and Director', + ); + await page.goto(base.href, { waitUntil: 'domcontentloaded' }); + await ready('navara'); + await switchTo('cesium'); + assert.equal(await stored(), 'cesium'); + await page.evaluate(() => { + const write = Storage.prototype.setItem; + Storage.prototype.setItem = function (key, value) { + if (key === 'gev.engine') + throw new DOMException('Blocked', 'SecurityError'); + return write.call(this, key, value); + }; + }); + await switchTo('navara'); + assert.equal(await stored(), 'cesium'); + // The real Map Source chips move into daylight inspection and back to live lighting. + await page.$eval('[data-stack-id="sentinel2"]', (el) => el.click()); + await page.waitForFunction( + () => + window.__godsEyeView.mapStackController.getActiveId() === 'sentinel2' && + window.__godsEyeView.engine.host.container.hasAttribute( + 'data-terrain-daylight', + ), + ); + // Zooming out on the satellite map must keep the same inspection sun. + assert.ok( + await page.evaluate(async () => { + const { engine } = window.__godsEyeView; + engine.camera.zoomOut(2_000_000); + await new Promise((resolve) => { + const remove = engine.frames.postRender.addEventListener(() => { + remove(); + resolve(); + }); + engine.render.request('qa-orbital-daylight'); + }); + return ( + engine.camera.height() > 1_500_000 && + engine.host.container.hasAttribute('data-terrain-daylight') + ); + }), + 'the orbital frame keeps daylight on the inspected globe', + ); + await page.$eval('[data-stack-id="osm"]', (el) => el.click()); + await page.waitForFunction( + () => + window.__godsEyeView.mapStackController.getActiveId() === 'osm' && + !window.__godsEyeView.engine.host.container.hasAttribute( + 'data-terrain-daylight', + ), + ); + assert.deepEqual(errors, []); + console.log( + 'PASS Cesium default, both switches, same document, warm engine identity, parked canvas, no startup splash, share pose/map/layers, Back/Forward, remembered engine, blocked preference storage', + ); + }, +); diff --git a/scripts/qa-fire-camera-switch.mjs b/scripts/qa-fire-camera-switch.mjs index 12ce71c5d..75b635f98 100644 --- a/scripts/qa-fire-camera-switch.mjs +++ b/scripts/qa-fire-camera-switch.mjs @@ -1,19 +1,10 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { installAlprFixture } from './fixtures/w2/alpr.mjs'; -import { firmsFixture } from './fixtures/w2/firms.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -const url = new URL(process.argv[2]), +import { installAlprFixture, cameras } from './fixtures/w2/alpr.mjs'; +import { firmsFixture, FIRMS_VALID_COUNT } from './fixtures/w2/firms.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]), fire = process.argv.includes('--fires'), id = fire ? 'local-firms' : 'alpr-cameras'; -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&' + @@ -21,13 +12,17 @@ url.hash = ? 'lat=37.77&lon=-122.42&alt=100000' : 'lat=30.262&lon=-97.7431&alt=5500') + '&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Records the fixtures produce: the valid FIRMS rows; the ALPR cameras the +// fixture tiles carry around Austin. +const COUNT = fire ? FIRMS_VALID_COUNT : cameras.length; +const strict = createStrict({ + origin: url.origin, + layers: [id], + // installAlprFixture answers the camera tiles. + fixtures: ['https://tiles.dontgetflocked.com/*'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => errors.push(e.message)); @@ -48,7 +43,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async (id) => { @@ -62,52 +57,123 @@ try { await p.waitForFunction(() => window.qaGround?.getStats().count > 0, { timeout: 30000, }); - const target = await p.waitForFunction( - (fire) => { - const a = window.__godsEyeView, - m = window.qaGround; - const points = fire - ? m - .getDetectableObjects({ maxCount: 1000 }) - .filter((v) => v.id === m.getAnalystRecords(1)[0]?.id) - : [...window.__gevContextStore.entities.values()] - .filter((v) => v.layerId === 'alpr-cameras') - .map((v) => ({ - id: v.id, - position: - v.entity.gevAlprCanvasPosition || - v.entity.gevDisplayPosition?.(), - })); - for (const row of points) { - if (!row.position) continue; - const s = a.engine.projector.worldToWindow(row.position); - if (!s || s.x < 430 || s.x > 850 || s.y < 220 || s.y > 730) continue; - const picked = a.engine.picker.pick(s); - if (fire || picked?.id === row.id || picked?.id?.id === row.id) - return { x: s.x, y: s.y, id: row.id }; - } - return false; - }, - { timeout: 30000 }, - fire, - ); - const xy = await target.jsonValue(); - await new Promise(r=>setTimeout(r,1000)); + // A pickable target in the middle of the view: the camera the engine's + // own picker returns, or the hottest fire. + const findTarget = () => + p + .waitForFunction( + (fire) => { + const a = window.__godsEyeView, + m = window.qaGround; + const points = fire + ? m + .getDetectableObjects({ maxCount: 1000 }) + .filter((v) => v.id === m.getAnalystRecords(1)[0]?.id) + : [...window.__gevContextStore.entities.values()] + .filter((v) => v.layerId === 'alpr-cameras') + .map((v) => ({ + id: v.id, + position: + v.entity.gevAlprCanvasPosition || + v.entity.gevDisplayPosition?.(), + })); + for (const row of points) { + if (!row.position) continue; + const s = a.engine.projector.worldToWindow(row.position); + if (!s || s.x < 430 || s.x > 850 || s.y < 220 || s.y > 600) + continue; + const canvas = a.engine.host.canvas; + const box = canvas.getBoundingClientRect(); + if ( + !canvas.contains( + document.elementFromPoint(box.left + s.x, box.top + s.y), + ) + ) + continue; + const picked = a.engine.picker.pick(s); + if (fire || picked?.id === row.id || picked?.id?.id === row.id) + return { x: box.left + s.x, y: box.top + s.y, id: row.id }; + } + return false; + }, + { timeout: 30000 }, + fire, + ) + .then((handle) => handle.jsonValue()); + const selected = (id) => + p.waitForFunction( + (id, fire) => + fire + ? window.qaGround.getSelectedInfo()?.frp === 999 + : window.__gevContextStore?.selectedEntityId === id, + { timeout: 15000 }, + id, + fire, + ); + // A click on empty ground (nothing picked there, nothing covering it). + const clickEmpty = async () => { + const handle = await p.waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 240; y < 600; y += 9) + for (let x = 440; x < 840; x += 9) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [6, 0], + [-6, 0], + [0, 6], + [0, -6], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ); + const at = await handle.jsonValue(); + await p.mouse.click(at.x, at.y); + await p.waitForFunction( + (fire) => + fire + ? !window.qaGround.getSelectedInfo() + : !window.__gevContextStore?.selectedEntityId, + { timeout: 15000 }, + fire, + ); + }; + const xy = await findTarget(); + await new Promise((r) => setTimeout(r, 1000)); await p.mouse.click(xy.x, xy.y); - await p.waitForFunction( - (id,fire) => fire ? window.qaGround.getSelectedInfo()?.frp === 999 : window.__gevContextStore?.selectedEntityId === id, - {}, - xy.id, fire, + await selected(xy.id); + await strict.passed( + p, + id, + fire + ? 'a click on the hottest fire selects it' + : 'a click on a camera selects it', + xy, ); await new Promise((r) => setTimeout(r, 1600)); - const expectedId = fire ? await p.evaluate(()=>window.qaGround.getSelectedInfo().id) : xy.id; + const expectedId = fire + ? await p.evaluate(() => window.qaGround.getSelectedInfo().id) + : xy.id; const before = await p.evaluate(() => ({ stamp: window.qaGround.getStats().lastUpdate, records: window.qaGround.getAnalystRecords(1000).map((v) => v.id), reads: window.qaFirmsReads, })); for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); const result = await p.evaluate( (id) => ({ same: @@ -129,9 +195,31 @@ try { assert.equal(result.reads, before.reads); assert.deepEqual(result.records, before.records); assert.equal(result.stamp, before.stamp); + await strict.check( + p, + id, + `same model, ${COUNT} records, the clicked record still selected, no source read, same receipt time`, + result.count === COUNT, + { count: result.count }, + ); + if (kind === 'cesium') { + // Click-picking on Cesium: clear the selection with a click on empty + // ground, then click the same record again. + await clickEmpty(); + const again = await findTarget(); + assert.equal(again.id, xy.id); + await p.mouse.click(again.x, again.y); + await selected(again.id); + await strict.passed( + p, + id, + fire + ? 'a click on empty ground clears the fire; a click on it selects it again' + : 'a click on empty ground clears the camera; a click on it selects it again', + again, + ); + } } assert.deepEqual(errors, []); console.log('PASS retained source, stable clicked record on each planet'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-ghostmaps-switch.mjs b/scripts/qa-ghostmaps-switch.mjs index d6542da7d..d5633ffbe 100644 --- a/scripts/qa-ghostmaps-switch.mjs +++ b/scripts/qa-ghostmaps-switch.mjs @@ -1,27 +1,15 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import { readFile } from 'node:fs/promises'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['s2-ghostmaps'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -78,7 +66,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -108,12 +96,22 @@ try { ); await p.mouse.click(x, y); }; + const overlay = () => + p.evaluate(() => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return pixels; + }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { await p.evaluate(() => { window.ghostStats.titles = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction(() => window.ghostStats.titles.some((v) => /QA zone/i.test(v)), @@ -133,6 +131,11 @@ try { streams: 1, reads: 1, }); + await strict.passed( + p, + 's2-ghostmaps', + 'same model, three features, one event stream, one KMZ read', + ); await new Promise((resolve) => setTimeout(resolve, 2200)); for (const [name, lon, lat] of [ ['QA pin', -79.38, 43.65], @@ -148,19 +151,51 @@ try { { timeout: 12000 }, name, ); + await strict.passed( + p, + 's2-ghostmaps', + `a click on ${name} draws its card text`, + ); console.log('PASS', kind, name, 'pointer hit'); } + // Not vacuous: the zone's card is up, so the overlay only clears when + // the click in the hole is handled as a click on empty ground. The + // overlay without any card is measured with a click far from every + // feature. + const shown = await overlay(); await click(-79.316, 43.61); - await new Promise((resolve) => setTimeout(resolve, 600)); - await p.evaluate(() => { - window.ghostStats.titles = []; - window.__godsEyeView.engine.render.request('qa-hole'); - }); - await new Promise((resolve) => setTimeout(resolve, 400)); - assert.equal( - await p.evaluate(() => window.ghostStats.titles.length), - 0, - kind + ' polygon hole', + await p.waitForFunction( + (shown) => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return pixels < shown - 100; + }, + { timeout: 10000 }, + shown, + ); + const hole = await overlay(); + await click(-79.45, 43.55); + await new Promise((resolve) => setTimeout(resolve, 800)); + const empty = await overlay(); + console.log( + kind, + 'overlay pixels: zone card', + shown, + 'hole', + hole, + 'empty ground', + empty, + ); + await strict.check( + p, + 's2-ghostmaps', + 'a click in the zone hole clears the selected zone card like empty ground', + shown > empty + 100 && Math.abs(hole - empty) <= 0.02 * shown, + { shown, hole, empty }, ); await click(-79.28, 43.61); await p.waitForFunction(() => @@ -168,6 +203,4 @@ try { ); } assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-ground-layer-switch.mjs b/scripts/qa-ground-layer-switch.mjs index b1f58133d..96093706b 100644 --- a/scripts/qa-ground-layer-switch.mjs +++ b/scripts/qa-ground-layer-switch.mjs @@ -1,27 +1,23 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l=4'; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: [ + 'ais-live-vessels', + 'air-quality', + 'gnss-integrity', + 'fire-perimeters', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -34,30 +30,66 @@ try { const url = new URL(input.url ?? input, document.baseURI); if (url.pathname === '/api/vessels/track') { window.qaVesselHistories = (window.qaVesselHistories ?? 0) + 1; - return Promise.resolve(Response.json({ samples: [{lat: 43.63, lon: -79.38}] })); + return Promise.resolve( + Response.json({ samples: [{ lat: 43.63, lon: -79.38 }] }), + ); } if (url.pathname === '/api/vessels') { - return Promise.resolve(Response.json({status: 'live', rows: [{ - mmsi: '123456789', name: 'Toronto QA', lat: 43.65, lon: -79.38, - last_position_epoch: Date.now() / 1000, - }]})); + return Promise.resolve( + Response.json({ + status: 'live', + rows: [ + { + mmsi: '123456789', + name: 'Toronto QA', + lat: 43.65, + lon: -79.38, + last_position_epoch: Date.now() / 1000, + }, + ], + }), + ); } if (url.pathname.startsWith('/api/fire-perimeters/inciweb')) { return Promise.resolve(Response.json({ rows: [] })); } if (url.pathname === '/api/fire-perimeters') { const square = (dx) => [ - [-79.38-dx,43.65-dx],[-79.38+dx,43.65-dx],[-79.38+dx,43.65+dx], - [-79.38-dx,43.65+dx],[-79.38-dx,43.65-dx], + [-79.38 - dx, 43.65 - dx], + [-79.38 + dx, 43.65 - dx], + [-79.38 + dx, 43.65 + dx], + [-79.38 - dx, 43.65 + dx], + [-79.38 - dx, 43.65 - dx], ]; - return Promise.resolve(Response.json({rows: [{ - stableId: 'fire:fixture', name:'QA Fire', acres:500, containedPct:40, - polygons:[[square(0.09), square(0.018).reverse()]], - }]})); + return Promise.resolve( + Response.json({ + rows: [ + { + stableId: 'fire:fixture', + name: 'QA Fire', + acres: 500, + containedPct: 40, + polygons: [[square(0.09), square(0.018).reverse()]], + }, + ], + }), + ); } if (url.pathname === '/api/gnss-integrity') { - return Promise.resolve(Response.json({ageMs: 0, rows: ['a1','a2','a3'].map(hex => - ({hex, lat: 43.65, lon: -79.38, nic: 0, nacp: 0, degraded: true, gpsLost: false}))})); + return Promise.resolve( + Response.json({ + ageMs: 0, + rows: ['a1', 'a2', 'a3'].map((hex) => ({ + hex, + lat: 43.65, + lon: -79.38, + nic: 0, + nacp: 0, + degraded: true, + gpsLost: false, + })), + }), + ); } if (url.hostname === 'api.weather.gc.ca') { const records = url.pathname.includes('aqhi-stations') @@ -93,7 +125,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -105,12 +137,78 @@ try { ); }; const capture = async (visible, layerId = 'air-quality') => { - await p.evaluate(async (show, id) => { - const d = window.__godsEyeView; - await d.dataManager.setEnabled(id, show, { origin: 'user' }); - d.engine.render.request('qa-air-quality'); - }, visible, layerId); - await new Promise((r) => setTimeout(r, 3900)); + await p.evaluate( + async (show, id) => { + const d = window.__godsEyeView; + await d.dataManager.setEnabled(id, show, { origin: 'user' }); + d.engine.render.request('qa-air-quality'); + }, + visible, + layerId, + ); + // Ready to compare: the layer's data is in (when shown), the globe's + // tiles under the view have loaded and stayed loaded, and a frame has + // been drawn since. + await p + .waitForFunction( + (show, id) => { + const d = window.__godsEyeView; + if (!d?.engine) return false; + const module = d.dataManager.layers.get(id).module; + const data = + !show || + (id === 'air-quality' + ? module.getAnalystRecords().length === 1 + : module.getStats().levels?.high === 1); + const now = performance.now(); + // The engines draw on request: keep frames coming so tiles load. + d.engine.render.request('qa-ground-settle'); + if (!data || !d.engine.surface.tilesLoaded()) { + window.__qaSettledSince = null; + return false; + } + window.__qaSettledSince ??= now; + return now - window.__qaSettledSince > 1200; + }, + { polling: 100, timeout: 30000 }, + visible, + layerId, + ) + .catch(async (error) => { + console.log( + 'not settled', + JSON.stringify( + await p.evaluate((id) => { + const d = window.__godsEyeView; + const module = d?.dataManager.layers.get(id).module; + return { + engine: d?.engine?.kind, + tiles: d?.engine?.surface.tilesLoaded(), + records: module?.getAnalystRecords().length, + stats: module?.getStats(), + }; + }, layerId), + ), + ); + throw error; + }); + // A frame drawn after the settle (the engine draws on request), then + // the animation frame that presents it. + await p.evaluate( + () => + new Promise((resolve) => { + const { engine } = window.__godsEyeView; + const remove = engine.frames.postRender.addEventListener(() => { + remove(); + requestAnimationFrame(() => resolve()); + }); + engine.render.request('qa-ground'); + }), + ); + await p.evaluate(() => (window.__qaSettledSince = null)); + return sample(); + }; + const sample = async () => { const shot = await p.screenshot(); return sharp(shot) .extract({ left: 537, top: 403, width: 22, height: 18 }) @@ -119,6 +217,19 @@ try { .toBuffer() .then((buf) => sharp(buf).stats()); }; + // The layer shown, sampled until its polygon appears in the patch (the + // engines draw a clamped polygon a few frames after its data, with no + // signal for it), for at most 20 s; the comparison stays the same. + const orangeOf = (v) => v.channels[0].mean - v.channels[2].mean; + const drawn = async (off, layerId) => { + const deadline = Date.now() + 20000; + let on = await capture(true, layerId); + while (orangeOf(on) <= orangeOf(off) + 5 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 500)); + on = await sample(); + } + return on; + }; await ready('navara'); await p.evaluate(() => { const d = window.__godsEyeView; @@ -129,19 +240,27 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); const same = await p.evaluate( () => window.__godsEyeView.dataManager.layers.get('air-quality').module === - window.qaAir && - window.__godsEyeView.dataManager.layers.get('gnss-integrity').module === window.qaGnss, + window.qaAir && + window.__godsEyeView.dataManager.layers.get('gnss-integrity') + .module === window.qaGnss, + ); + await strict.check( + p, + ['air-quality', 'gnss-integrity'], + 'same layer models after the switch', + same, ); - assert.ok(same); } - await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('gnss-integrity', false)); + await p.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('gnss-integrity', false), + ); const off = await capture(false), - on = await capture(true); + on = await drawn(off, 'air-quality'); const values = await p.evaluate(() => ({ records: window.qaAir.getAnalystRecords(), card: window.qaAir.getRowControls().list.items, @@ -162,54 +281,128 @@ try { assert.equal(values.records[0].name, 'QA Toronto'); assert.equal(values.records[0].value, 9); assert.equal(values.card.length, 1); + await strict.passed( + p, + 'air-quality', + 'AQHI 9 station footprint drawn, one card row', + { + off: orange(off), + on: orange(on), + }, + ); await capture(false); const cellOff = await capture(false, 'gnss-integrity'); - const cellOn = await capture(true, 'gnss-integrity'); - assert.ok(orange(cellOn) > orange(cellOff) + 5, kind + ' must draw GNSS high cell'); + const cellOn = await drawn(cellOff, 'gnss-integrity'); + assert.ok( + orange(cellOn) > orange(cellOff) + 5, + kind + ' must draw GNSS high cell', + ); const gnss = await p.evaluate(() => window.qaGnss.getStats()); assert.equal(gnss.levels.high, 1); - console.log('PASS', kind, 'GNSS high cell, view anchor', gnss.coverage.latitude, gnss.coverage.longitude); - await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('air-quality', true)); + await strict.passed(p, 'gnss-integrity', 'one high GNSS cell drawn', { + off: orange(cellOff), + on: orange(cellOn), + }); + console.log( + 'PASS', + kind, + 'GNSS high cell, view anchor', + gnss.coverage.latitude, + gnss.coverage.longitude, + ); + await p.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('air-quality', true), + ); } await p.evaluate(async () => { - const {dataManager} = window.__godsEyeView; + const { dataManager } = window.__godsEyeView; await dataManager.setEnabled('air-quality', false); - await dataManager.setEnabled('fire-perimeters', true, {origin: 'user'}); + await dataManager.setEnabled('fire-perimeters', true, { origin: 'user' }); window.qaFire = dataManager.layers.get('fire-perimeters').module; - if (!dataManager.isEnabled('fire-perimeters') || window.qaFire.getStats().count !== 1) - throw Error('Failed to enable fire perimeter fixture: '+JSON.stringify(window.qaFire.getStats())); + if ( + !dataManager.isEnabled('fire-perimeters') || + window.qaFire.getStats().count !== 1 + ) + throw Error( + 'Failed to enable fire perimeter fixture: ' + + JSON.stringify(window.qaFire.getStats()), + ); }); - const card = async (visible) => p.waitForFunction((want) => { - const canvas = document.getElementById('world-overlay-canvas'); - const data = canvas.getContext('2d').getImageData(0,0,canvas.width,canvas.height).data; - let pixels = 0; - for (let i=3; i 100 : pixels === 0; - }, {timeout: 15000}, visible); + const card = async (visible) => + p.waitForFunction( + (want) => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return want ? pixels > 100 : pixels === 0; + }, + { timeout: 15000 }, + visible, + ); const click = async (lon, lat) => { - const {x, y} = await p.evaluate((world) => { - const {engine} = window.__godsEyeView; - const point = engine.projector.worldToWindow(world, {}); - const box = engine.host.canvas.getBoundingClientRect(); - return { x: box.left+point.x, y: box.top+point.y }; - }, fromDegrees(lon, lat)); - await p.mouse.click(x,y); + const { x, y } = await p.evaluate( + (world) => { + const { engine } = window.__godsEyeView; + const point = engine.projector.worldToWindow(world, {}); + if (!point) throw Error('the point is not in view'); + const box = engine.host.canvas.getBoundingClientRect(); + return { x: box.left + point.x, y: box.top + point.y }; + }, + fromDegrees(lon, lat), + ); + await p.mouse.click(x, y); }; for (const kind of ['navara', 'cesium', 'navara']) { - if (await p.evaluate(() => window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine', kind); + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await strict.pick(p, kind); await ready(kind); await card(true); - assert.ok(await p.evaluate(() => window.qaFire === window.__godsEyeView.dataManager.layers.get('fire-perimeters').module)); + await strict.check( + p, + 'fire-perimeters', + 'same model; the incident card survives the switch', + await p.evaluate( + () => + window.qaFire === + window.__godsEyeView.dataManager.layers.get('fire-perimeters') + .module, + ), + ); } await click(-79.51, 43.65); await card(false); - await click(-79.38, 43.65); - await new Promise(resolve => setTimeout(resolve, 500)); - await card(false); + await strict.passed( + p, + 'fire-perimeters', + 'a click outside the perimeter clears the card', + ); await click(-79.435, 43.65); await card(true); - console.log('PASS', kind, 'select fire under GNSS, exclude hole, retain incident card'); + await strict.check( + p, + 'fire-perimeters', + 'a click on the ring selects the fire', + await p.evaluate(() => window.qaFire.getSelectedInfo?.()?.id ?? true), + ); + // Not vacuous: the fire is selected, so the card only goes when the + // click in the hole is handled as a click outside the polygon. + await click(-79.38, 43.65); + await card(false); + await strict.passed( + p, + 'fire-perimeters', + 'a click in the hole deselects the selected fire', + ); + await click(-79.435, 43.65); + await card(true); + console.log( + 'PASS', + kind, + 'select fire under GNSS, exclude hole, retain incident card', + ); } await p.evaluate(async () => { const { dataManager } = window.__godsEyeView; @@ -218,17 +411,79 @@ try { await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user' }); window.qaVesselEntry = dataManager.layers.get('ais-live-vessels'); }); - await p.waitForFunction(() => window.qaVesselEntry.module.qaDrawn()?.chevrons[0]?.show); + await p.waitForFunction( + () => window.qaVesselEntry.module.qaDrawn()?.chevrons[0]?.show, + ); await new Promise((resolve) => setTimeout(resolve, 800)); + // Click-picking on the running engine: clear the selection with a click on + // empty water, then select the vessel with a click on its chevron. + const pickVessel = async () => { + // Empty ground: a point in the middle of the view where the engine + // picks nothing and nothing covers the canvas. + const empty = await p + .waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 220; y < 560; y += 11) + for (let x = 300; x < 720; x += 11) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [10, 0], + [-10, 0], + [0, 10], + [0, -10], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ) + .then((handle) => handle.jsonValue()); + await p.mouse.click(empty.x, empty.y); + await p.waitForFunction( + () => !window.qaVesselEntry.module.getSelectedInfo(), + ); + await new Promise((resolve) => setTimeout(resolve, 600)); + await click(-79.38, 43.65); + await p.waitForFunction( + () => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789', + { timeout: 15000 }, + ); + await strict.passed( + p, + 'ais-live-vessels', + 'a click on the chevron selects the vessel', + ); + }; await click(-79.38, 43.65); - await p.waitForFunction(() => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789'); + await p.waitForFunction( + () => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789', + ); + await strict.passed( + p, + 'ais-live-vessels', + 'a click on the chevron selects the vessel', + ); + let histories = await p.evaluate(() => window.qaVesselHistories); for (const kind of ['navara', 'cesium', 'navara']) { - if (await p.evaluate(() => window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine', kind); + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await strict.pick(p, kind); await ready(kind); } const sample = await p.evaluate(() => { - const entry = window.__godsEyeView.dataManager.layers.get('ais-live-vessels'); + const entry = + window.__godsEyeView.dataManager.layers.get('ais-live-vessels'); const drawn = entry.module.qaDrawn(); return { retained: entry === window.qaVesselEntry, @@ -238,13 +493,31 @@ try { histories: window.qaVesselHistories, }; }); - assert.deepEqual(sample, { - retained: true, selected: '123456789', chevron: '123456789', - card: 'vessel:123456789', histories: 1, - }, kind); - console.log('PASS', kind, 'retained selected AIS vessel, chevron, card and single track request'); + assert.deepEqual( + sample, + { + retained: true, + selected: '123456789', + chevron: '123456789', + card: 'vessel:123456789', + histories, + }, + kind, + ); + await strict.passed( + p, + 'ais-live-vessels', + 'same entry; selection, chevron and card kept; no new track request', + ); + if (kind === 'cesium' || histories === 1) { + await pickVessel(); + histories = await p.evaluate(() => window.qaVesselHistories); + } + console.log( + 'PASS', + kind, + 'retained selected AIS vessel, chevron, card and single track request', + ); } assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-infrastructure-engine-switch.mjs b/scripts/qa-infrastructure-engine-switch.mjs index 758a531b6..e29f08d65 100644 --- a/scripts/qa-infrastructure-engine-switch.mjs +++ b/scripts/qa-infrastructure-engine-switch.mjs @@ -1,26 +1,33 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { readFileSync } from 'node:fs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=41.5340&lon=-90.5070&alt=20000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Every geometry of the bundled files (a Multi* feature counts each part), +// read from the files this checkout builds. +const geometries = (name) => + readFileSync( + new URL(`../src/data/local_data/${name}/${name}.geojsonl`, import.meta.url), + 'utf8', + ) + .split('\n') + .filter((line) => line.trim()) + .reduce((sum, line) => { + const { type, coordinates } = JSON.parse(line).geometry; + return sum + (type.startsWith('Multi') ? coordinates.length : 1); + }, 0); +const DATACENTERS = geometries('datacenters'); +const DAMS = geometries('dams'); +const strict = createStrict({ + origin: base.origin, + layers: ['local-dams', 'local-datacenters'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -39,12 +46,16 @@ try { if (detail?.layerId?.startsWith('local-')) window.qaSelections.push(detail.id); }); + window.qaCleared = 0; + window.addEventListener('gev:entity-selection-cleared', () => { + window.qaCleared++; + }); }); await p.goto(base.href); const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -77,8 +88,10 @@ try { } return records; }, ids); - assert.ok( - dataset[0].count >= 4351 && dataset[1].count >= 704, + // The bundled files are fixed: every record is loaded. + assert.deepEqual( + dataset.map((d) => d.count), + [DATACENTERS, DAMS], JSON.stringify(dataset), ); const record = await p.evaluate(() => @@ -103,13 +116,75 @@ try { { timeout: 15000 }, ); }; + // A dam in view, clicked where its record is. + const dam = await p.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('local-dams') + .module.getAnalystRecords(5000) + .find((r) => r.name === 'Arsenal Power Dam'), + ); + assert.ok(dam); + const clickDam = async () => { + const screen = await p.evaluate( + (world) => { + const { engine } = window.__godsEyeView; + const at = engine.projector.worldToWindow(world, {}); + const box = engine.host.canvas.getBoundingClientRect(); + window.qaSelections = []; + return { x: box.left + at.x, y: box.top + at.y }; + }, + fromDegrees(dam.lon, dam.lat), + ); + await p.mouse.click(screen.x, screen.y); + await p + .waitForFunction( + () => window.qaSelections.at(-1) === 'local-dams:-12841971', + { timeout: 15000 }, + ) + .catch(async (error) => { + console.log( + 'dam click', + JSON.stringify(screen), + JSON.stringify( + await p.evaluate(({ x, y }) => { + const { engine } = window.__godsEyeView; + const box = engine.host.canvas.getBoundingClientRect(); + const id = engine.picker.pick({ + x: x - box.left, + y: y - box.top, + })?.id; + return { + selections: window.qaSelections, + cleared: window.qaCleared, + pick: String(id?.id ?? id), + hit: document.elementFromPoint(x, y)?.tagName, + }; + }, screen), + ), + ); + throw error; + }); + }; + const start = fromDegrees(-90.507, 41.534, 20000); + const home = async () => { + await p.evaluate((destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.cancelFlight(); + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-home'); + }, start); + await new Promise((resolve) => setTimeout(resolve, 2000)); + }; const counts = { ...reads }; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { await p.evaluate(() => { window.qaSelections = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction( () => window.qaSelections.includes('local-datacenters:342664739'), @@ -142,16 +217,38 @@ try { assert.equal(layer.count, dataset[i].count); assert.equal(layer.name, dataset[i].name); } - if ( - kind === 'navara' && - !(await p.evaluate(() => window.qaSelections.length)) - ) - await clickSite(); assert.deepEqual( reads, counts, 'Do not reload bundled data on scene handoff', ); + await strict.passed( + p, + ids, + 'same model, every bundled record, bounded LOD stems, no file read again', + result.map((r) => r.count), + ); + if ( + kind !== 'navara' || + (await p.evaluate(() => window.qaSelections.length)) + ) + await strict.passed( + p, + 'local-datacenters', + 'the clicked datacenter stays selected across the switch', + ); + // Click-picking on this engine: a dam, then the datacenter again, each + // from the starting view (a selection flies the camera to it). + await home(); + await clickDam(); + await strict.passed(p, 'local-dams', 'a click on a dam selects it'); + await home(); + await clickSite(); + await strict.passed( + p, + 'local-datacenters', + 'a click on the datacenter selects it', + ); } assert.equal(counts.datacenters, 1); assert.equal(counts.dams, 1); @@ -159,6 +256,4 @@ try { console.log( 'PASS complete bundled datasets, bounded stems, retained selection and one read per file', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-layer-handoff.mjs b/scripts/qa-layer-handoff.mjs index 4cd412fe0..865a1ee8a 100644 --- a/scripts/qa-layer-handoff.mjs +++ b/scripts/qa-layer-handoff.mjs @@ -1,15 +1,7 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const url = localPreview(process.argv[2]); const military = process.argv.includes('--military'); const layerId = military ? 'military' : 'flights'; const aircraftId = military ? 'aaa052' : '71be26'; @@ -17,381 +9,520 @@ url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=40.2539&lon=-125.5779&alt=21130&heading=360&pitch=-35&map=sentinel2' + (military ? '&l=m&lo=f.e.1_f.u.aaa052' : '&l=f&lo=f.e.1_f.t.71be26'); -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - const errors = []; - page.on('pageerror', (e) => errors.push(e.message)); - await page.setViewport({ width: 1280, height: 720 }); - await page.evaluateOnNewDocument( - (layerId, aircraftId) => { - window.__qaAircraft = { layerId, aircraftId }; - const fetch = window.fetch.bind(window); - window.__flightPolls = 0; - const interval = window.setInterval.bind(window); - window.setInterval = function (callback, delay, ...args) { - if (typeof callback !== 'function') - return interval(callback, delay, ...args); - const id = interval( - function (...values) { - if (window.__handoff?.timer === id) window.__handoff.ticks++; - return callback.apply(this, values); - }, - delay, - ...args, - ); - return id; - }; - window.fetch = (input, options) => { - const path = new URL(input.url ?? input, document.baseURI).pathname; - const now = Math.floor(Date.now() / 1000); - if (path === '/api/flights') { - if (layerId === 'flights') window.__flightPolls++; - return Promise.resolve( - Response.json({ - time: now, - states: [ - [ - '71be26', - 'GEV123', - 'QA', - now, - now, - -125.5779, - 40.2539, - 9700, - false, - 200, - 45, - 0, - null, - 9700, - null, - false, - 0, +// The Command Dock, and the Globe Engine picker in it, is hidden in cockpit +// mode: no person can pick an engine there, so the cockpit switches below go +// through the select element; the real picker is used once the cockpit is +// left. +const HIDDEN_PICKER = 'the Globe Engine picker is hidden in cockpit mode'; +const strict = createStrict({ origin: url.origin, layers: [layerId] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', (e) => errors.push(e.message)); + await page.setViewport({ width: 1280, height: 720 }); + await page.evaluateOnNewDocument( + (layerId, aircraftId) => { + window.__qaAircraft = { layerId, aircraftId }; + const fetch = window.fetch.bind(window); + window.__flightPolls = 0; + const interval = window.setInterval.bind(window); + window.setInterval = function (callback, delay, ...args) { + if (typeof callback !== 'function') + return interval(callback, delay, ...args); + const id = interval( + function (...values) { + if (window.__handoff?.timer === id) window.__handoff.ticks++; + return callback.apply(this, values); + }, + delay, + ...args, + ); + return id; + }; + window.fetch = (input, options) => { + const path = new URL(input.url ?? input, document.baseURI).pathname; + const now = Math.floor(Date.now() / 1000); + if (path === '/api/flights') { + if (layerId === 'flights') window.__flightPolls++; + return Promise.resolve( + Response.json({ + time: now, + states: [ + [ + '71be26', + 'GEV123', + 'QA', + now, + now, + -125.5779, + 40.2539, + 9700, + false, + 200, + 45, + 0, + null, + 9700, + null, + false, + 0, + ], ], - ], - }), - ); - } - if (path === '/api/military') { - if (layerId === 'military') window.__flightPolls++; - return Promise.resolve( - Response.json({ - now: Date.now(), - ac: [ - { - hex: 'aaa052', - flight: 'GEVMIL', - lon: -125.5779, - lat: 40.2539, - alt_geom: 31825, - alt_baro: 31825, - gs: 389, - track: 45, - seen: 0, - seen_pos: 0, - dbFlags: 1, - type: 'adsb_icao', - }, - ], - }), - ); - } - if (path === '/api/flights/track') - return Promise.resolve(Response.json({ path: [] })); - if (path.startsWith('/api/flights/') || path.startsWith('/api/openai/')) - return Promise.resolve(Response.json({})); - return fetch(input, options); - }; - }, - layerId, - aircraftId, - ); - await page.goto(url.href, { waitUntil: 'domcontentloaded' }); - const ready = (kind) => - page.waitForFunction( - (engine) => - window.__godsEyeView?.engine.kind === engine && - !document.querySelector('#globe-engine')?.disabled && - document.querySelector('#loading-screen.hidden') && - !document.querySelector('.engine-switch-frame'), - { timeout: 90000 }, - kind, + }), + ); + } + if (path === '/api/military') { + if (layerId === 'military') window.__flightPolls++; + return Promise.resolve( + Response.json({ + now: Date.now(), + ac: [ + { + hex: 'aaa052', + flight: 'GEVMIL', + lon: -125.5779, + lat: 40.2539, + alt_geom: 31825, + alt_baro: 31825, + gs: 389, + track: 45, + seen: 0, + seen_pos: 0, + dbFlags: 1, + type: 'adsb_icao', + }, + ], + }), + ); + } + if (path === '/api/flights/track') + return Promise.resolve(Response.json({ path: [] })); + if (path.startsWith('/api/flights/')) + return Promise.resolve(Response.json({})); + return fetch(input, options); + }; + }, + layerId, + aircraftId, ); - await ready('navara'); - await page.evaluate( - () => window.__godsEyeView.styleManager.initialRestorePromise, - ); - await page.waitForSelector('#cockpit-entry', { visible: true }); - await page.click('#cockpit-entry'); - await page.waitForFunction( - () => window.__godsEyeView.styleManager.getCockpitState().active, - ); - await page.evaluate(() => { - const { dataManager, styleManager, tileset } = window.__godsEyeView; - if (window.__GOOGLE_MAPS_API_KEY__ || tileset) - throw Error('Keyless preview required'); - styleManager.cockpitView.setVisionMode('thermal'); - const entry = dataManager.layers.get(window.__qaAircraft.layerId); - window.__handoff = { - dataManager, - entry, - row: document.querySelector( - `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, - ), - timer: entry.intervalId, - ticks: 0, - model: entry.module, - pollCount: window.__flightPolls, - timestamp: entry.module.getStats().lastUpdate, - }; - }); - // Refuse the first Cesium canvas. The application must roll back with the - // original aircraft still in the cockpit, then allow an ordinary retry. - await page.evaluate(() => { - const original = HTMLCanvasElement.prototype.getContext; - HTMLCanvasElement.prototype.getContext = function (type, ...args) { - if (String(type).includes('webgl')) { - window.__refusedContext = true; - throw new Error('QA renderer startup failure'); - } - return original.call(this, type, ...args); - }; - window.__restoreContext = () => { - HTMLCanvasElement.prototype.getContext = original; - }; - }); - const historyLength = await page.evaluate(() => history.length); - await page.select('#globe-engine', 'cesium'); - await page.waitForFunction(() => window.__refusedContext === true); - await ready('navara'); - // A failed switch leaves the session usable on the engine that worked and - // adds no history entry. - const survived = (kind) => - page.evaluate( - (kind, length) => { + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + const ready = (kind) => + page.waitForFunction( + (engine) => + window.__godsEyeView?.engine?.kind === engine && + !document.querySelector('#globe-engine')?.disabled && + document.querySelector('#loading-screen.hidden') && + !document.querySelector('.engine-switch-frame'), + { timeout: 90000 }, + kind, + ); + await ready('navara'); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); + await page.waitForSelector('#cockpit-entry', { visible: true }); + await page.click('#cockpit-entry'); + await page.waitForFunction( + () => window.__godsEyeView.styleManager.getCockpitState().active, + ); + await page.evaluate(() => { + const { dataManager, styleManager, tileset } = window.__godsEyeView; + if (window.__GOOGLE_MAPS_API_KEY__ || tileset) + throw Error('Keyless preview required'); + styleManager.cockpitView.setVisionMode('thermal'); + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + window.__handoff = { + dataManager, + entry, + row: document.querySelector( + `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, + ), + timer: entry.intervalId, + ticks: 0, + model: entry.module, + pollCount: window.__flightPolls, + timestamp: entry.module.getStats().lastUpdate, + }; + }); + // Refuse the first Cesium canvas. The application must roll back with the + // original aircraft still in the cockpit, then allow an ordinary retry. + await page.evaluate(() => { + const original = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + if (String(type).includes('webgl')) { + window.__refusedContext = true; + throw new Error('QA renderer startup failure'); + } + return original.call(this, type, ...args); + }; + window.__restoreContext = () => { + HTMLCanvasElement.prototype.getContext = original; + }; + }); + const historyLength = await page.evaluate(() => history.length); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await page.waitForFunction(() => window.__refusedContext === true); + await ready('navara'); + // A failed switch leaves the session usable on the engine that worked and + // adds no history entry. + const survived = (kind) => + page.evaluate( + (kind, length) => { + const { dataManager, engine, styleManager } = window.__godsEyeView; + return ( + engine.kind === kind && + new URL(location.href).searchParams.get('engine') === kind && + history.length === length && + dataManager === window.__handoff.dataManager && + dataManager.layers.get(window.__qaAircraft.layerId) === + window.__handoff.entry && + styleManager.getCockpitState().active && + styleManager.getCockpitState().visionMode === 'thermal' && + styleManager.getAircraftTrackingTarget()?.id === + window.__qaAircraft.aircraftId + ); + }, + kind, + historyLength, + ); + await page.evaluate(() => window.__restoreContext()); + await strict.check( + page, + layerId, + 'cold Cesium failure rolls back to Navara: same session, cockpit, aircraft, no history entry', + await survived('navara'), + ); + console.log( + 'PASS cold Cesium failure rolled back to Navara with the same flight, cockpit and history', + ); + + for (const kind of ['cesium', 'navara']) { + await strict.selectEngine(page, kind, HIDDEN_PICKER); + await ready(kind); + const result = await page.evaluate(() => { const { dataManager, engine, styleManager } = window.__godsEyeView; + const before = window.__handoff; + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + const cockpit = styleManager.getCockpitState(); + return { + sameOwner: + before.dataManager === dataManager && + before.entry === entry && + before.model === entry.module, + sameRow: + Boolean(before.row) && + before.row === + document.querySelector( + `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, + ), + sameTimer: + before.timer === entry.intervalId && entry.intervalId != null, + sourcePolls: window.__flightPolls - before.pollCount, + timerTicks: before.ticks, + stamp: entry.module.getStats().lastUpdate, + initialStamp: before.timestamp, + id: cockpit.subject?.id, + active: cockpit.active, + mode: cockpit.visionMode, + hud: !document.querySelector('#cockpit-hud').hidden, + drawn: entry.module + .qaDrawn() + .some((row) => row.id === window.__qaAircraft.aircraftId), + height: engine.camera.height(), + }; + }); + assert.ok( + result.sameOwner && + result.sameTimer && + result.sameRow && + result.active && + result.hud && + result.drawn, + JSON.stringify(result), + ); + assert.equal(result.id, aircraftId); + assert.equal(result.mode, 'thermal'); + assert.ok( + result.sourcePolls <= result.timerTicks, + 'switch must not add a source request', + ); + if (result.timerTicks === 0) { + assert.equal(result.stamp, result.initialStamp); + } + await strict.passed( + page, + layerId, + 'same manager entry, model, row, poll timer, cockpit, thermal mode, HUD and drawn aircraft', + result, + ); + // Polling goes on after the switch: the same timer ticks and each tick + // reads the source once (no tick may be skipped, none added). + const polls = await page.evaluate(async (layerId) => { + const entry = window.__godsEyeView.dataManager.layers.get(layerId); + const every = + Number(entry.module.refreshInterval) || + Number(entry.module.updateInterval); + const start = { + polls: window.__flightPolls, + ticks: window.__handoff.ticks, + stamp: entry.module.getStats().lastUpdate, + }; + const deadline = Date.now() + 2.5 * every + 5000; + while ( + window.__handoff.ticks - start.ticks < 2 && + Date.now() < deadline + ) + await new Promise((resolve) => setTimeout(resolve, 250)); + // Let the last tick's request land. + await new Promise((resolve) => setTimeout(resolve, 1500)); + return { + every, + ticks: window.__handoff.ticks - start.ticks, + polls: window.__flightPolls - start.polls, + updated: entry.module.getStats().lastUpdate !== start.stamp, + sameTimer: entry.intervalId === window.__handoff.timer, + }; + }, layerId); + await strict.check( + page, + layerId, + 'polling continues after the switch: two timer ticks, one source read each, fresh data', + polls.every > 0 && + polls.ticks >= 2 && + polls.polls === polls.ticks && + polls.updated && + polls.sameTimer, + polls, + ); + await page.waitForFunction( + () => + Math.abs(window.__godsEyeView.engine.camera.height() - 9700) < 1000, + ); + console.log( + `PASS ${kind} cockpit, aircraft, HUD, manager and timer (${result.sourcePolls} polls since entry)`, + ); + } + // The other direction, warm: the parked Navara renderer fails to resume on + // the way back from Cesium. The session rolls back to Cesium, then an + // ordinary retry starts Navara cold. + await page.evaluate(() => { + window.__parkedNavara = window.__godsEyeView.engine; + }); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await ready('cesium'); + const lengthOnCesium = await page.evaluate(() => { + const request = window.__parkedNavara.render.request; + window.__parkedNavara.render.request = function (...args) { + window.__parkedNavara.render.request = request; + window.__refusedResume = true; + throw new Error('QA parked renderer failure'); + }; + return history.length; + }); + await strict.selectEngine(page, 'navara', HIDDEN_PICKER); + await page.waitForFunction(() => window.__refusedResume === true); + await ready('cesium'); + assert.ok( + (await page.evaluate( + (length) => history.length === length, + lengthOnCesium, + )) && + (await page.evaluate(() => { + const { styleManager } = window.__godsEyeView; + return ( + styleManager.getCockpitState().active && + styleManager.getAircraftTrackingTarget()?.id === + window.__qaAircraft.aircraftId && + new URL(location.href).searchParams.get('engine') === 'cesium' + ); + })), + 'a failed warm Navara resume must leave Cesium usable without a history entry', + ); + await strict.passed( + page, + layerId, + 'warm Navara failure rolls back to Cesium: same aircraft and cockpit, no history entry', + ); + console.log( + 'PASS warm Navara failure rolled back to Cesium with the same flight and cockpit', + ); + await page.evaluate(() => { + window.__parkedCesium = window.__godsEyeView.engine; + }); + await strict.selectEngine(page, 'navara', HIDDEN_PICKER); + await ready('navara'); + assert.ok( + await page.evaluate(() => { + const { engine, styleManager } = window.__godsEyeView; return ( - engine.kind === kind && - new URL(location.href).searchParams.get('engine') === kind && - history.length === length && - dataManager === window.__handoff.dataManager && - dataManager.layers.get(window.__qaAircraft.layerId) === - window.__handoff.entry && + engine !== window.__parkedNavara && styleManager.getCockpitState().active && - styleManager.getCockpitState().visionMode === 'thermal' && styleManager.getAircraftTrackingTarget()?.id === window.__qaAircraft.aircraftId ); - }, - kind, - historyLength, + }), + 'the retry starts a new Navara renderer on the same session', ); - await page.evaluate(() => window.__restoreContext()); - assert.ok( - await survived('navara'), - 'a failed scene must leave the live session usable', - ); - console.log( - 'PASS cold Cesium failure rolled back to Navara with the same flight, cockpit and history', - ); + console.log('PASS retry after the warm failure started Navara cold'); - for (const kind of ['cesium', 'navara']) { - await page.select('#globe-engine', kind); - await ready(kind); - const result = await page.evaluate(() => { - const { dataManager, engine, styleManager } = window.__godsEyeView; - const before = window.__handoff; - const entry = dataManager.layers.get(window.__qaAircraft.layerId); - const cockpit = styleManager.getCockpitState(); - return { - sameOwner: - before.dataManager === dataManager && - before.entry === entry && - before.model === entry.module, - sameRow: - Boolean(before.row) && - before.row === - document.querySelector( - `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, - ), - sameTimer: - before.timer === entry.intervalId && entry.intervalId != null, - sourcePolls: window.__flightPolls - before.pollCount, - timerTicks: before.ticks, - stamp: entry.module.getStats().lastUpdate, - initialStamp: before.timestamp, - id: cockpit.subject?.id, - active: cockpit.active, - mode: cockpit.visionMode, - hud: !document.querySelector('#cockpit-hud').hidden, - drawn: entry.module - .qaDrawn() - .some((row) => row.id === window.__qaAircraft.aircraftId), - height: engine.camera.height(), - }; + // Both engines fail: the warm Cesium resume and the warm Navara recovery. + // No engine runs, but the error screen offers the engine picker, and a + // retry starts Cesium cold on the same session without a reload. + const lengthBeforeDoubleFailure = await page.evaluate(() => { + window.__failResume = true; + for (const engine of [ + window.__godsEyeView.engine, + window.__parkedCesium, + ]) { + const request = engine.render.request; + engine.render.request = function (reason, ...args) { + if (reason === 'engine-resume' && window.__failResume) { + window.__refusedResumes = (window.__refusedResumes ?? 0) + 1; + throw new Error('QA parked renderer failure'); + } + return request.call(this, reason, ...args); + }; + } + window.__startDocument = document; + return history.length; }); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await page + .waitForSelector('#loading-screen .engine-switch-retry button', { + visible: true, + timeout: 90000, + }) + .catch(async (error) => { + console.log( + 'no retry picker', + JSON.stringify( + await page.evaluate(() => ({ + refused: window.__refusedResumes, + engine: window.__godsEyeView?.engine?.kind ?? null, + retry: !!document.querySelector('.engine-switch-retry'), + loading: document.querySelector('#loading-screen')?.className, + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + address: location.search, + })), + ), + ); + throw error; + }); assert.ok( - result.sameOwner && - result.sameTimer && - result.sameRow && - result.active && - result.hud && - result.drawn, - JSON.stringify(result), + await page.evaluate( + (length) => + window.__refusedResumes === 2 && + // No globe runs (the page's debug object may stay, without one). + !window.__godsEyeView?.engine && + history.length === length && + !document + .querySelector('#loading-screen') + .classList.contains('hidden') && + /QA parked renderer failure/.test( + document.querySelector('#loading-screen .loader-status') + .textContent, + ), + lengthBeforeDoubleFailure, + ), + 'a failed recovery shows its error and the engine picker ' + + JSON.stringify( + await page.evaluate(() => ({ + refused: window.__refusedResumes, + app: !!window.__godsEyeView, + engine: window.__godsEyeView?.engine?.kind ?? null, + history: history.length, + loading: document.querySelector('#loading-screen')?.className, + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + })), + ), ); - assert.equal(result.id, aircraftId); - assert.equal(result.mode, 'thermal'); + await page.evaluate(() => { + window.__failResume = false; + }); + await page.click( + '#loading-screen .engine-switch-retry [data-engine="cesium"]', + ); + await ready('cesium'); assert.ok( - result.sourcePolls <= result.timerTicks, - 'switch must not add a source request', - ); - if (result.timerTicks === 0) { - assert.equal(result.stamp, result.initialStamp); - } - await page.waitForFunction( - () => Math.abs(window.__godsEyeView.engine.camera.height() - 9700) < 1000, - ); - console.log( - `PASS ${kind} cockpit, aircraft, HUD, manager and timer (${result.sourcePolls} polls since entry)`, - ); - } - // The other direction, warm: the parked Navara renderer fails to resume on - // the way back from Cesium. The session rolls back to Cesium, then an - // ordinary retry starts Navara cold. - await page.evaluate(() => { - window.__parkedNavara = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', 'cesium'); - await ready('cesium'); - const lengthOnCesium = await page.evaluate(() => { - const request = window.__parkedNavara.render.request; - window.__parkedNavara.render.request = function (...args) { - window.__parkedNavara.render.request = request; - window.__refusedResume = true; - throw new Error('QA parked renderer failure'); - }; - return history.length; - }); - await page.select('#globe-engine', 'navara'); - await page.waitForFunction(() => window.__refusedResume === true); - await ready('cesium'); - assert.ok( - (await page.evaluate( - (length) => history.length === length, - lengthOnCesium, - )) && - (await page.evaluate(() => { - const { styleManager } = window.__godsEyeView; + await page.evaluate(() => { + const { dataManager, styleManager } = window.__godsEyeView; return ( + window.__startDocument === document && + !document.querySelector('.engine-switch-retry') && + dataManager === window.__handoff.dataManager && + dataManager.layers.get(window.__qaAircraft.layerId) === + window.__handoff.entry && styleManager.getCockpitState().active && styleManager.getAircraftTrackingTarget()?.id === window.__qaAircraft.aircraftId && new URL(location.href).searchParams.get('engine') === 'cesium' ); - })), - 'a failed warm Navara resume must leave Cesium usable without a history entry', - ); - console.log( - 'PASS warm Navara failure rolled back to Cesium with the same flight and cockpit', - ); - await page.evaluate(() => { - window.__parkedCesium = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', 'navara'); - await ready('navara'); - assert.ok( + }), + 'the retry restores the same session on Cesium', + ); + await strict.passed( + page, + layerId, + 'after both engines fail, the retry restores the same session', + ); + console.log( + 'PASS failed recovery offered the picker; the retry restored the session on Cesium', + ); + await page.evaluate(() => + window.__godsEyeView.styleManager.controlCockpit('exit'), + ); + await page.waitForFunction( + () => !window.__godsEyeView.styleManager.getCockpitState().active, + ); + // Out of the cockpit the dock shows the picker again: a switch with + // real input keeps the same entry and timer, the aircraft drawn, and the + // source polled on its timer. await page.evaluate(() => { - const { engine, styleManager } = window.__godsEyeView; - return ( - engine !== window.__parkedNavara && - styleManager.getCockpitState().active && - styleManager.getAircraftTrackingTarget()?.id === - window.__qaAircraft.aircraftId - ); - }), - 'the retry starts a new Navara renderer on the same session', - ); - console.log('PASS retry after the warm failure started Navara cold'); - - // Both engines fail: the warm Cesium resume and the warm Navara recovery. - // No engine runs, but the error screen offers the engine picker, and a - // retry starts Cesium cold on the same session without a reload. - const lengthBeforeDoubleFailure = await page.evaluate(() => { - window.__failResume = true; - for (const engine of [window.__godsEyeView.engine, window.__parkedCesium]) { - const request = engine.render.request; - engine.render.request = function (reason, ...args) { - if (reason === 'engine-resume' && window.__failResume) { - window.__refusedResumes = (window.__refusedResumes ?? 0) + 1; - throw new Error('QA parked renderer failure'); - } - return request.call(this, reason, ...args); + window.__handoff.ticks = 0; + window.__handoff.pollCount = window.__flightPolls; + window.__handoff.timer = window.__godsEyeView.dataManager.layers.get( + window.__qaAircraft.layerId, + ).intervalId; + }); + await strict.pick(page, 'navara'); + await ready('navara'); + const after = await page.evaluate(async () => { + const { dataManager } = window.__godsEyeView; + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + const every = + Number(entry.module.refreshInterval) || + Number(entry.module.updateInterval); + const deadline = Date.now() + 2.5 * every + 5000; + while (window.__handoff.ticks < 1 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise((resolve) => setTimeout(resolve, 1500)); + return { + same: + dataManager === window.__handoff.dataManager && + entry === window.__handoff.entry, + timer: entry.intervalId === window.__handoff.timer, + drawn: entry.module + .qaDrawn() + .some((row) => row.id === window.__qaAircraft.aircraftId), + ticks: window.__handoff.ticks, + polls: window.__flightPolls - window.__handoff.pollCount, }; - } - window.__startDocument = document; - return history.length; - }); - await page.select('#globe-engine', 'cesium'); - await page.waitForSelector('#loading-screen .engine-switch-retry button', { - visible: true, - timeout: 90000, - }); - assert.ok( - await page.evaluate( - (length) => - window.__refusedResumes === 2 && - !window.__godsEyeView && - history.length === length && - !document - .querySelector('#loading-screen') - .classList.contains('hidden') && - /QA parked renderer failure/.test( - document.querySelector('#loading-screen .loader-status').textContent, - ), - lengthBeforeDoubleFailure, - ), - 'a failed recovery shows its error and the engine picker', - ); - await page.evaluate(() => { - window.__failResume = false; - }); - await page.click( - '#loading-screen .engine-switch-retry [data-engine="cesium"]', - ); - await ready('cesium'); - assert.ok( - await page.evaluate(() => { - const { dataManager, styleManager } = window.__godsEyeView; - return ( - window.__startDocument === document && - !document.querySelector('.engine-switch-retry') && - dataManager === window.__handoff.dataManager && - dataManager.layers.get(window.__qaAircraft.layerId) === - window.__handoff.entry && - styleManager.getCockpitState().active && - styleManager.getAircraftTrackingTarget()?.id === - window.__qaAircraft.aircraftId && - new URL(location.href).searchParams.get('engine') === 'cesium' - ); - }), - 'the retry restores the same session on Cesium', - ); - console.log( - 'PASS failed recovery offered the picker; the retry restored the session on Cesium', - ); - await page.evaluate(() => - window.__godsEyeView.styleManager.controlCockpit('exit'), - ); - await page.waitForFunction( - () => !window.__godsEyeView.styleManager.getCockpitState().active, - ); - assert.deepEqual(errors, []); -} finally { - await browser.close(); -} + }); + await strict.check( + page, + layerId, + 'picker switch keeps the entry, timer and drawn aircraft; polling continues', + after.same && + after.timer && + after.drawn && + after.ticks >= 1 && + after.polls === after.ticks, + after, + ); + assert.deepEqual(errors, []); + }, +); diff --git a/scripts/qa-local-adsb-engine-switch.mjs b/scripts/qa-local-adsb-engine-switch.mjs index 329049e09..e5dbc1f9d 100644 --- a/scripts/qa-local-adsb-engine-switch.mjs +++ b/scripts/qa-local-adsb-engine-switch.mjs @@ -1,25 +1,13 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=41.5340&lon=-90.5070&alt=20000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['local-adsb'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -32,6 +20,8 @@ try { const url = new URL(input?.url ?? input, location.href); if (url.pathname === '/api/local-receivers/aircraft') { const at = Date.now(); + // Each poll reports the aircraft a little further north. + const n = (window.qaAdsbPolls = (window.qaAdsbPolls ?? 0) + 1); return Promise.resolve( Response.json({ configured: true, @@ -40,14 +30,14 @@ try { { icao: 'abc123', callsign: 'LOCAL1', - lat: 41.534, + lat: 41.534 + 0.0002 * n, lon: -90.507, altitudeFt: 11000, groundSpeedKt: 180, trackDeg: 5, lastPositionAt: at, lastMessageAt: at, - messageCount: 12, + messageCount: 12 + n, band: '1090', source: 'decoder', }, @@ -66,7 +56,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -112,7 +102,7 @@ try { await p.evaluate(() => { window.qaSelections = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction(() => window.qaSelections.includes('local-adsb:abc123'), @@ -133,11 +123,34 @@ try { ]; }); assert.deepEqual(result, [true, true, true, true, 'abc123', 1, 0, 0]); + await strict.passed( + p, + 'local-adsb', + 'same decoder, receiver and feeds, one feed start, selection republished', + ); + // Updates continue on the new engine: further polls move the aircraft. + const before = await p.evaluate(() => ({ + polls: window.qaAdsbPolls, + lat: window.qaAdsb.mod.getAnalystRecords()[0]?.lat, + stamp: window.qaAdsb.mod.getStats().lastUpdate, + })); + await p.waitForFunction( + (before) => + window.qaAdsbPolls >= before.polls + 2 && + window.qaAdsb.mod.getAnalystRecords()[0]?.lat > before.lat && + window.qaAdsb.mod.getStats().lastUpdate > before.stamp, + { timeout: 15000 }, + before, + ); + await strict.passed( + p, + 'local-adsb', + 'receiver polls continue and move the selected aircraft', + before, + ); } assert.deepEqual(errors, []); console.log( 'PASS retained local decoder, receiver, marker and selection in both scenes', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-map-policy.mjs b/scripts/qa-map-policy.mjs index 240e003f5..9fa1600d0 100644 --- a/scripts/qa-map-policy.mjs +++ b/scripts/qa-map-policy.mjs @@ -13,3 +13,105 @@ export async function blockPaidQaMaps(page, { allowPaidMaps = false } = {}) { }); return session; } + +/** + * Free public tile hosts the strict acceptance lets a page ask for. The page + * never reaches them: the strict harness answers them from a disk cache + * (scripts/qa/tileCache.mjs), and Node fetches a missing tile once, politely. + * OpenStreetMap raster tiles, the keyless re:earth terrain, the EOX + * Sentinel-2 cloudless mosaic and Esri World Imagery (the keyless Cesium + * base imagery and the Navara startup imagery). + */ +export const STRICT_TILE_HOSTS = Object.freeze([ + 'tile.openstreetmap.org', + 'terrain.reearth.land', + 'tiles.maps.eox.at', + 'services.arcgisonline.com', +]); + +/** + * Requests every keyless page makes on its own, answered by the strict + * harness exactly as the keyless preview would (or with an empty stylesheet + * for Google Fonts), so they reach neither the preview's paid routes nor + * Google. Any other request under a paid route or host stays blocked. + * Keys are URLPattern strings relative to the preview origin or absolute. + */ +export const KEYLESS_PROBES = Object.freeze([ + { + pattern: '/api/openai/hud-summary/status', + method: 'GET', + status: 200, + contentType: 'application/json', + body: '{"configured":false}', + }, + { pattern: '/api/google/tiles-token', method: 'GET', status: 204 }, + { pattern: '/api/realtime/debug-log', method: 'POST', status: 204 }, + { + pattern: 'https://fonts.googleapis.com/css2*', + method: 'GET', + status: 200, + contentType: 'text/css', + body: '/* Google Fonts withheld by the strict QA policy */', + }, +]); + +/** A probe's absolute URLPattern string. */ +export const probePattern = (origin, probe) => + probe.pattern.startsWith('/') + ? new URL(origin).origin + probe.pattern + : probe.pattern; + +/** Paid or keyed services: blocked first, whatever else a scenario allows. */ +export const PAID_HOST_PATTERNS = Object.freeze([ + '*://*.googleapis.com/*', + '*://googleapis.com/*', + '*://*.google.com/*', + '*://google.com/*', + '*://maps.google.*/*', + '*://*.gstatic.com/*', + '*://*.cesium.com/*', + '*://cesium.com/*', +]); + +/** Preview routes that spend a key (OpenAI, Realtime, Google proxies). */ +export const PAID_PREVIEW_PATHS = Object.freeze([ + '/api/openai/*', + '/api/realtime/*', + '/api/google/*', +]); + +/** + * The ordered Network.setBlockedURLs patterns of the strict policy (first + * match wins): the keyless probes the harness answers are let through to + * it, paid hosts and paid preview routes are blocked, then the + * preview origin, the scenario's fixture URLs and the free tile hosts are + * allowed, then everything else is blocked. + * + * @param {string} origin The preview origin, e.g. http://127.0.0.1:4208 + * @param {{fixtures?: string[], tileHosts?: readonly string[]}} [options] + * `fixtures` are URLPattern strings for requests the scenario answers. + */ +export function strictBlockPatterns( + origin, + { fixtures = [], tileHosts = STRICT_TILE_HOSTS } = {}, +) { + const preview = new URL(origin).origin; + return [ + ...KEYLESS_PROBES.map((probe) => ({ + urlPattern: probePattern(origin, probe), + block: false, + })), + ...PAID_HOST_PATTERNS.map((urlPattern) => ({ urlPattern, block: true })), + ...PAID_PREVIEW_PATHS.map((path) => ({ + urlPattern: preview + path, + block: true, + })), + { urlPattern: preview + '/*', block: false }, + ...fixtures.map((urlPattern) => ({ urlPattern, block: false })), + ...tileHosts.map((host) => ({ + urlPattern: `https://${host}/*`, + block: false, + })), + { urlPattern: '*://*:*/*', block: true }, + ]; +} diff --git a/scripts/qa-map-policy.test.mjs b/scripts/qa-map-policy.test.mjs index 647a7cca3..cc90402f1 100644 --- a/scripts/qa-map-policy.test.mjs +++ b/scripts/qa-map-policy.test.mjs @@ -1,6 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { + blockPaidQaMaps, + KEYLESS_PROBES, + probePattern, + strictBlockPatterns, +} from './qa-map-policy.mjs'; test('routine QA blocks direct Google tiles, hosted Google assets and token fallback', async () => { const calls = []; @@ -25,3 +30,54 @@ test('a necessary 3D-specific check must explicitly opt out of the request block { allowPaidMaps: true }, ); }); + +test('the strict policy blocks paid hosts and routes first and everything unlisted last', () => { + const patterns = strictBlockPatterns('http://127.0.0.1:4208', { + fixtures: ['https://tiles.dontgetflocked.com/*'], + }); + const decide = (url) => + patterns.find(({ urlPattern }) => new URLPattern(urlPattern).test(url)) + ?.block; + for (const url of [ + 'https://tile.googleapis.com/v1/3dtiles/root.json', + 'https://maps.googleapis.com/maps/api/js', + 'https://fonts.gstatic.com/s/inter.woff2', + 'https://api.cesium.com/v1/assets/1/endpoint', + 'https://assets.ion.cesium.com/google-credit.png', + 'http://127.0.0.1:4208/api/openai/hud-summary', + 'http://127.0.0.1:4208/api/realtime/token', + 'http://127.0.0.1:4208/api/google/nearby-places?lat=1', + 'https://example.com/anything', + 'wss://example.com/socket', + ]) + assert.equal(decide(url), true, url); + for (const url of [ + 'http://127.0.0.1:4208/assets/index.js', + 'http://127.0.0.1:4208/api/flights', + 'https://tiles.dontgetflocked.com/cameras-us-hourly/1/2/3.mvt', + 'https://tile.openstreetmap.org/1/2/3.png', + 'https://terrain.reearth.land/cesium-mesh/layer.json', + 'https://tiles.maps.eox.at/wmts/1.0.0/s2/1/2/3.jpg', + ]) + assert.equal(decide(url), false, url); +}); + +test('the keyless probes are let through only to their exact routes', () => { + const patterns = strictBlockPatterns('http://127.0.0.1:4208'); + const decide = (url) => + patterns.find(({ urlPattern }) => new URLPattern(urlPattern).test(url)) + ?.block; + for (const probe of KEYLESS_PROBES) { + const url = probePattern('http://127.0.0.1:4208', probe).replace( + /\*$/, + '?family=Inter', + ); + assert.equal(decide(url), false, url); + } + assert.equal( + decide('http://127.0.0.1:4208/api/openai/hud-summary'), + true, + 'the paid summary itself stays blocked', + ); + assert.equal(decide('https://fonts.googleapis.com/icon?family=x'), true); +}); diff --git a/scripts/qa-nepal-event-switch.mjs b/scripts/qa-nepal-event-switch.mjs index 5ae7adf64..bdcdb2aa9 100644 --- a/scripts/qa-nepal-event-switch.mjs +++ b/scripts/qa-nepal-event-switch.mjs @@ -1,26 +1,45 @@ import sharp from 'sharp'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=28.2110&lon=85.3380&alt=9000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// The media panel embeds YouTube players with their thumbnails: a plain +// thumbnail and an inert player page stand in for them. +const thumbnail = await sharp({ + create: { width: 480, height: 360, channels: 3, background: '#40506a' }, +}) + .jpeg() + .toBuffer(); +const strict = createStrict({ + origin: base.origin, + layers: ['bhote-koshi-2026'], + routes: [ + { + pattern: 'https://i.ytimg.com/*', + respond: () => ({ contentType: 'image/jpeg', body: thumbnail }), + }, + { + // The player API script: an inert stub (the app keeps its own controls). + pattern: 'https://www.youtube.com/iframe_api*', + respond: () => ({ + contentType: 'text/javascript', + body: '/* QA: YouTube iframe API withheld */', + }), + }, + { + pattern: 'https://www.youtube-nocookie.com/*', + respond: () => ({ + contentType: 'text/html', + body: 'QA player', + }), + }, + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -45,7 +64,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -59,7 +78,8 @@ try { await ready('navara'); await p.evaluate(async () => { const { tileset, dataManager, sceneDirector } = window.__godsEyeView; - if (!sceneDirector?.listScenes().length) throw Error('No scenes available on Navara'); + if (!sceneDirector?.listScenes().length) + throw Error('No scenes available on Navara'); if (tileset || window.__GOOGLE_MAPS_API_KEY__) throw Error('Keyless preview required'); const mod = dataManager.layers.get('bhote-koshi-2026').module; @@ -82,7 +102,7 @@ try { }; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } const state = await p.evaluate(() => { @@ -107,7 +127,10 @@ try { ); window.qaEvent.setParams({ split: 0.75 }); }); - assert.ok(await p.evaluate(()=>Boolean(window.qaPlayer)), 'persistent embedded media root'); + assert.ok( + await p.evaluate(() => Boolean(window.qaPlayer)), + 'persistent embedded media root', + ); const before = await sample(); await p.evaluate(() => window.qaEvent.setParams({ split: 0.27 })); const after = await sample(); @@ -115,6 +138,12 @@ try { before.reduce((n, v, i) => n + Math.abs(v - after[i]), 0) / before.length; console.log(kind, state.state, 'imagery difference', diff); assert.ok(diff > 6, 'divider changes the actual imagery in the viewport'); + await strict.passed( + p, + 'bhote-koshi-2026', + 'same model and media root; paused at 0.52 with split 0.27; the divider changes the imagery', + { diff }, + ); assert.equal( await p.evaluate( () => @@ -130,14 +159,25 @@ try { 1, JSON.stringify(assets), ); + await strict.passed( + p, + 'bhote-koshi-2026', + 'event file and image pair read once over three engine visits', + ); await p.evaluate(() => window.qaEvent.setParams({ autoPlay: true })); await p.waitForFunction( () => window.qaEvent.getPlaybackState().progress > 0.53, ); - await p.select('#globe-engine', 'cesium'); + await strict.pick(p, 'cesium'); await ready('cesium'); const live = await p.evaluate(() => window.qaEvent.getPlaybackState()); - assert.ok(live.playing && live.progress > 0.53); + await strict.check( + p, + 'bhote-koshi-2026', + 'playback keeps playing past 0.53 through the switch', + live.playing && live.progress > 0.53, + live, + ); await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('bhote-koshi-2026', false, { origin: 'user', @@ -147,6 +187,4 @@ try { console.log( 'PASS event timeline, paused and playing handoff, image pair, divider, player host and single asset acquisition', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-nepal-locator-switch.mjs b/scripts/qa-nepal-locator-switch.mjs index 6f3209fb2..30416b2e8 100644 --- a/scripts/qa-nepal-locator-switch.mjs +++ b/scripts/qa-nepal-locator-switch.mjs @@ -1,27 +1,18 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; import sharp from 'sharp'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=28.3326&lon=85.4848&alt=18000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: ['bhote-koshi-locator'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -36,7 +27,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -60,7 +51,7 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } const { same, stats, params, at } = await p.evaluate( @@ -114,9 +105,12 @@ try { added, at, ); - assert.ok( + await strict.check( + p, + 'bhote-koshi-locator', + 'same model; the flood path adds more than 50 cyan pixels to the same scene without it', added > 50, - 'visible river differs from the same scene without a path', + { added }, ); await p.evaluate(() => window.qaLocator.setParams({ presentation: 'bhote-koshi-path-overview' }), @@ -130,7 +124,21 @@ try { preview: true, }); }); - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 2500)); + // The last of three quick presentation changes is the one that stays. + const last = await p.evaluate(() => ({ + params: window.qaLocator.getParams(), + stats: window.qaLocator.getStats(), + })); + await strict.check( + p, + 'bhote-koshi-locator', + 'the last of three quick presentation changes is the one applied', + last.params.presentation === 'bhote-koshi-trigger-record' && + last.stats.count === 1 && + !last.stats.error, + { presentation: last.params.presentation, count: last.stats.count }, + ); await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('bhote-koshi-locator', false, { origin: 'user', @@ -140,6 +148,4 @@ try { console.log( 'PASS Nepal locator flood path and presentation retained, trigger and places update; both engines', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-radio-engine-switch.mjs b/scripts/qa-radio-engine-switch.mjs index 9c035014d..427de7712 100644 --- a/scripts/qa-radio-engine-switch.mjs +++ b/scripts/qa-radio-engine-switch.mjs @@ -1,168 +1,91 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; import { stations, installRadioFixture } from './fixtures/w2/radio.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]); -const soak = process.argv.includes('--soak'); -const requested = process.argv.find((arg) => arg.startsWith('--swaps=')); -const swaps = requested ? Number(requested.slice(8)) : soak ? 50 : 2; -if (requested && (!soak || !Number.isInteger(swaps) || swaps % 2 || swaps < 4 || swaps > 1000)) - throw Error('Use --soak --swaps=N with an even count from 4 to 1000'); - -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Local keyless preview required'); +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]); +// The long soak and switch timings live in qa-strict-soak.mjs. url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.27&lon=-97.74&alt=12000&pitch=-90&map=sentinel2&l='; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - const errors = []; - page.on('pageerror', (e) => errors.push(e.message)); - await installRadioFixture(page); - if (soak) await page.evaluateOnNewDocument(() => { - const fetch = window.fetch.bind(window); - window.qaQuakeReads = 0; - window.fetch = (input, init) => { - const url = new URL(input?.url ?? input, location.href); - if (url.hostname !== 'earthquake.usgs.gov') return fetch(input, init); - window.qaQuakeReads++; - return Promise.resolve(Response.json({ - type: 'FeatureCollection', - features: [{ type: 'Feature', id: 'qa-austin', properties: { mag: 4.1, time: Date.now(), place: 'Austin' }, - geometry: { type: 'Point', coordinates: [-97.74, 30.27, 8] } }], - })); - }; - }); - await page.goto(url.href); - await page.waitForFunction( - () => - window.__godsEyeView?.engine?.kind === 'navara' && - document.querySelector('#loading-screen.hidden'), - ); - await page.evaluate(async (id) => { - const { dataManager, styleManager, tileset } = window.__godsEyeView; - await styleManager.initialRestorePromise; - if (window.__GOOGLE_MAPS_API_KEY__ || tileset) - throw Error('Expected keyless preview'); - window.qaLayer = dataManager.layers.get('radio').module; - await dataManager.setEnabled('radio', true, { origin: 'user' }); - }, stations[0].id); - await page.waitForFunction(() => window.qaLayer?.getStats().count === 750); - await page.evaluate((id) => { - window.qaLayer.selectStation(id, { - autoplay: true, - focus: false, - origin: 'user', - }); - window.qaLayer.setRadioVolume(0.36); - }, stations[10].id); - await page.waitForFunction( - () => - window.__qaRadio.plays.length === 1 && - window.qaLayer.getUIState().audioState === 'playing', - ); - const first = await page.evaluate(() => { - window.qaAudio = window.qaPlayer; - window.qaSnapshot = window.qaLayer.getAcceptedCatalogSnapshot(); - return window.__qaRadio.reads; - }); - const client = soak ? await page.createCDPSession() : null; - if (soak) await page.evaluate(async () => { - const { dataManager, engine } = window.__godsEyeView; - await dataManager.setEnabled('earthquakes', true, { origin: 'user' }); - window.qaModels = new Map([...dataManager.layers].map(([id, entry]) => [id, entry.module])); - window.qaEngines = new Map([[engine.kind, engine]]); - }); - const measurements = [], memory = []; - let contextListeners; - for (let i = 1; i <= swaps; i++) { - const kind = i % 2 ? 'cesium' : 'navara'; - const start = await page.evaluate(() => performance.now()); - await switchGlobe(page, kind); - const elapsed = await page.evaluate((start) => performance.now() - start, start); - measurements.push({ kind, ms: Math.round(elapsed) }); - if (soak) { - assert.ok(await page.evaluate(() => { - const { dataManager, engine } = window.__godsEyeView; - const previous = window.qaEngines.get(engine.kind); - window.qaEngines.set(engine.kind, engine); - return (!previous || previous === engine) && window.qaEngines.size === 2 && - [...window.qaModels].every(([id, model]) => dataManager.layers.get(id)?.module === model) && - window.qaModels.size === 33 && window.qaQuakeReads >= 1 && - dataManager.getEnabledLayerIds().has('earthquakes'); - })); - if ([2, 10, 50, 100, 250, swaps].includes(i)) { - const { result } = await client.send('Runtime.evaluate', { - includeCommandLineAPI: true, - returnByValue: true, - expression: `(() => { - const events = getEventListeners(window.__godsEyeView.engine.host.canvas); - return [events.webglcontextlost.length, events.webglcontextrestored.length]; - })()`, - }); - assert.ok( - Array.isArray(result.value) && result.value.length === 2 && - result.value.every((count) => Number.isInteger(count) && count > 0), - 'the active canvas must expose both WebGL event subscriptions', - ); - contextListeners ??= result.value; - assert.deepEqual(result.value, contextListeners, 'warm Navara must release old context listeners'); - await client.send('HeapProfiler.collectGarbage'); - const heap = await client.send('Runtime.getHeapUsage'); - const dom = await client.send('Memory.getDOMCounters'); - memory.push({ swap: i, contextListeners, jsMiB: Math.round(heap.usedSize / 1048576), ...dom }); - } - } - const state = await page.evaluate(() => ({ - same: - window.qaLayer === - window.__godsEyeView.dataManager.layers.get('radio').module, - player: window.qaAudio === window.qaPlayer, - snapshot: - window.qaSnapshot === window.qaLayer.getAcceptedCatalogSnapshot(), - plays: window.__qaRadio.plays.length, - reads: window.__qaRadio.reads, - state: window.qaLayer.getUIState(), - count: window.qaLayer.getStats().count, - })); - console.log( - kind, - state.count, - state.state.selected?.id, - state.state.audioState, - state.plays, - state.reads, +const strict = createStrict({ origin: url.origin, layers: ['radio'] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', (e) => errors.push(e.message)); + await installRadioFixture(page); + await page.goto(url.href); + await page.waitForFunction( + () => + window.__godsEyeView?.engine?.kind === 'navara' && + document.querySelector('#loading-screen.hidden'), ); - assert.ok(state.same && state.player && state.snapshot); - assert.equal(state.plays, 1); - assert.equal(state.reads, first); - assert.equal(state.count, 750); - assert.equal(state.state.audioState, 'playing'); - assert.equal(state.state.selected.id, stations[10].id); - assert.equal(state.state.volume, 0.36); - } - if (soak) { - const warm = {}; - for (const kind of ['navara', 'cesium']) { - const ms = measurements.slice(2).filter(x => x.kind === kind).map(x => x.ms).sort((a, b) => a - b); - warm[kind] = { p50: ms[Math.ceil(ms.length * 0.50) - 1], p95: ms[Math.ceil(ms.length * 0.95) - 1] }; + await page.evaluate(async (id) => { + const { dataManager, styleManager, tileset } = window.__godsEyeView; + await styleManager.initialRestorePromise; + if (window.__GOOGLE_MAPS_API_KEY__ || tileset) + throw Error('Expected keyless preview'); + window.qaLayer = dataManager.layers.get('radio').module; + await dataManager.setEnabled('radio', true, { origin: 'user' }); + }, stations[0].id); + await page.waitForFunction(() => window.qaLayer?.getStats().count === 750); + await page.evaluate((id) => { + window.qaLayer.selectStation(id, { + autoplay: true, + focus: false, + origin: 'user', + }); + window.qaLayer.setRadioVolume(0.36); + }, stations[10].id); + await page.waitForFunction( + () => + window.__qaRadio.plays.length === 1 && + window.qaLayer.getUIState().audioState === 'playing', + ); + const first = await page.evaluate(() => { + window.qaAudio = window.qaPlayer; + window.qaSnapshot = window.qaLayer.getAcceptedCatalogSnapshot(); + return window.__qaRadio.reads; + }); + for (const kind of ['cesium', 'navara']) { + await strict.switchTo(page, kind); + const state = await page.evaluate(() => ({ + same: + window.qaLayer === + window.__godsEyeView.dataManager.layers.get('radio').module, + player: window.qaAudio === window.qaPlayer, + snapshot: + window.qaSnapshot === window.qaLayer.getAcceptedCatalogSnapshot(), + plays: window.__qaRadio.plays.length, + reads: window.__qaRadio.reads, + state: window.qaLayer.getUIState(), + count: window.qaLayer.getStats().count, + })); + console.log( + kind, + state.count, + state.state.selected?.id, + state.state.audioState, + state.plays, + state.reads, + ); + assert.ok(state.same && state.player && state.snapshot); + assert.equal(state.plays, 1); + assert.equal(state.reads, first); + assert.equal(state.count, 750); + assert.equal(state.state.audioState, 'playing'); + assert.equal(state.state.selected.id, stations[10].id); + assert.equal(state.state.volume, 0.36); + // The fixture's HTMLMediaElement.prototype.play counts play() calls: the + // switch must not start the stream again. + await strict.passed( + page, + 'radio', + 'same model, player element and catalog snapshot; 750 stations, one catalog read, no second play(), same station playing at volume 0.36', + ); } - console.log('SOAK', JSON.stringify({ swaps, first: measurements.slice(0, 2), warm, memory })); - } - assert.deepEqual(errors, []); - console.log( - 'PASS one live radio player, one station directory and the same selection', - ); -} finally { - await browser.close(); -} + assert.deepEqual(errors, []); + console.log( + 'PASS one live radio player, one station directory and the same selection', + ); + }, +); diff --git a/scripts/qa-space-engine-switch.mjs b/scripts/qa-space-engine-switch.mjs index 4a25c910b..a7a6248c2 100644 --- a/scripts/qa-space-engine-switch.mjs +++ b/scripts/qa-space-engine-switch.mjs @@ -1,30 +1,21 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; import { launchFixture } from '../src/bench/launchFixture.js'; import { satelliteFixture, SATELLITE_FIXTURE_EPOCH, } from '../src/bench/satelliteFixture.js'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]), +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]), missions = process.argv.includes('--missions'); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=5&lon=5&alt=18000000&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const layer = missions ? 'rocket-launches' : 'satellites'; +const strict = createStrict({ + origin: url.origin, + layers: [missions ? 'rocket-launches' : 'satellites'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); const errors = []; p.on('pageerror', (e) => { errors.push(e.message); @@ -68,7 +59,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async (missions) => { @@ -127,8 +118,28 @@ try { before.mission, before.replay?.active, ); + // Each source read exactly once: the seven TLE groups of the dense + // catalog; in Space Missions the launch list and every TLE group the + // mission view asks for. + const groupPaths = Object.keys(groups).map((g) => '/api/celestrak/' + g); + const readsOnce = async () => { + const reads = await p.evaluate(() => ({ ...window.spaceReads })); + await strict.check( + p, + layer, + missions + ? 'the launch list and each TLE group read exactly once' + : 'each of the seven TLE groups read exactly once', + Object.values(reads).every((n) => n === 1) && + (missions + ? reads['/api/launches'] === 1 + : Object.keys(reads).sort().join() === [...groupPaths].sort().join()), + reads, + ); + }; + await readsOnce(); for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); const state = await p.evaluate(() => { const { dataManager, engine, styleManager } = window.__godsEyeView; return { @@ -174,6 +185,14 @@ try { assert.ok(state.follow && state.ring > 0); assert.equal(state.sat.lastUpdate, before.stamp); } + await strict.passed( + p, + layer, + missions + ? 'same models; Space Missions mode, the same mission paused at the same progress, speed 4' + : 'same models; 6 839 objects, the ISS tracked with its ring, same update stamp', + ); + await readsOnce(); } if (missions) { await p.evaluate(() => window.la.toggleReplayPause()); @@ -183,9 +202,12 @@ try { {}, before.replay.phaseProgress, ); - await switchGlobe(p, 'cesium'); + await strict.switchTo(p, 'cesium'); const moving = await p.evaluate(() => window.la.qaDrawn()); - assert.ok( + await strict.check( + p, + layer, + 'a running replay keeps advancing through the switch', moving.replay?.active && !moving.paused && moving.replay.phaseProgress > before.replay.phaseProgress, @@ -203,6 +225,4 @@ try { } assert.deepEqual(errors, []); console.log('PASS retained space data and active camera across both planets'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-street-level-engine-switch.mjs b/scripts/qa-street-level-engine-switch.mjs index fd7650ca8..46b63d71b 100644 --- a/scripts/qa-street-level-engine-switch.mjs +++ b/scripts/qa-street-level-engine-switch.mjs @@ -1,6 +1,5 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { serveFixtures, newFixture } from './qa-street-level.mjs'; import { KV_LINE, KV_SEQUENCE_ID } from './fixtures/street-level/kartaView.mjs'; import { @@ -10,162 +9,205 @@ import { } from './fixtures/street-level/panoramax.mjs'; const panorama = process.argv.includes('--panoramax'); const provider = panorama ? 'panoramax' : 'kartaview'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw Error('Keyless local preview required'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0'; const at = panorama ? PNX_PARK : { lat: 38.5831, lon: KV_LINE.lon, height: 850 }; base.hash = `v=2&lat=${at.lat}&lon=${at.lon}&alt=${at.height}&heading=0&pitch=-90&roll=0&map=sentinel2&l=`; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// serveFixtures (qa-street-level.mjs) answers these provider hosts. +const strict = createStrict({ + origin: base.origin, + layers: ['street-level'], + fixtures: [ + '*://*.mapillary.com/*', + 'https://api.kartaview.org/*', + 'https://cdn.kartaview.org/*', + '*://*.openstreetcam.org/*', + 'https://api.panoramax.xyz/*', + 'https://panoramax.openstreetmap.fr/*', + ], }); -try { - const p = await browser.newPage(); - await blockPaidQaMaps(p); - await p.setViewport({ width: 1280, height: 900 }); - const errors = [], - fixture = newFixture({ configured: true }); - p.on('pageerror', (e) => { - errors.push(e.message); - console.error(e.message.split('?')[0]); - }); - p.on('console', (m) => { - if (/shader|registerMeshPicker/i.test(m.text()) && m.type() === 'error') - errors.push(m.text()); - }); - await serveFixtures(p, fixture); - let photoRequests = 0; - p.on('request', (req) => { - if ( - /\.jpg(?:$|\?)/i.test(new URL(req.url()).pathname) && - /openstreetcam|kartaview|panoramax/.test(new URL(req.url()).hostname) - ) - photoRequests++; - }); - await p.goto(base.href); - const ready = async (k) => { - await p.waitForFunction( - (kind) => - window.__godsEyeView?.engine?.kind === kind && - !document.querySelector('#globe-engine')?.disabled && - !document.querySelector('.engine-switch-frame') && - document.querySelector('#loading-screen.hidden'), - { timeout: 90000 }, - k, - ); - await p.evaluate( - () => window.__godsEyeView.styleManager.initialRestorePromise, - ); - }; - await ready('navara'); - await p.evaluate(async (provider) => { - const { dataManager, tileset } = window.__godsEyeView; - if (tileset || window.__GOOGLE_MAPS_API_KEY__) - throw Error('Expected keyless preview'); - window.qaStreet = dataManager.layers.get('street-level').module; - await dataManager.setEnabled('street-level', true, { origin: 'user' }); - window.qaStreet.setProviderEnabled(provider, true); - }, provider); - const until = async (fn) => { - try { - await p.waitForFunction(fn, { timeout: 20000 }); - } catch (e) { - console.log(await p.evaluate(() => window.qaStreet.getUIState())); - throw e; +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const p = await browser.newPage(); + await p.setViewport({ width: 1280, height: 900 }); + const errors = [], + fixture = newFixture({ configured: true }); + p.on('pageerror', (e) => { + errors.push(e.message); + console.error(e.message.split('?')[0]); + }); + p.on('console', (m) => { + if (/shader|registerMeshPicker/i.test(m.text()) && m.type() === 'error') + errors.push(m.text()); + }); + await serveFixtures(p, fixture); + let photoRequests = 0; + p.on('request', (req) => { + if ( + /\.jpg(?:$|\?)/i.test(new URL(req.url()).pathname) && + /openstreetcam|kartaview|panoramax/.test(new URL(req.url()).hostname) + ) + photoRequests++; + }); + await p.goto(base.href); + const ready = async (k) => { + await p.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { timeout: 90000 }, + k, + ); + await p.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); + }; + await ready('navara'); + await p.evaluate(async (provider) => { + const { dataManager, tileset } = window.__godsEyeView; + if (tileset || window.__GOOGLE_MAPS_API_KEY__) + throw Error('Expected keyless preview'); + window.qaStreet = dataManager.layers.get('street-level').module; + await dataManager.setEnabled('street-level', true, { origin: 'user' }); + window.qaStreet.setProviderEnabled(provider, true); + }, provider); + const until = async (fn) => { + try { + await p.waitForFunction(fn, { timeout: 20000 }); + } catch (e) { + console.log(await p.evaluate(() => window.qaStreet.getUIState())); + throw e; + } + }; + await until(() => window.qaStreet.getUIState().coverage.count > 0); + // The fixture drive passes under the middle of the canvas. Click its actual picked line. + const pickLine = async () => { + const xy = await p.evaluate( + (prefix) => { + const { engine } = window.__godsEyeView, + canvas = engine.host.canvas, + rect = canvas.getBoundingClientRect(); + for (let dy = -30; dy <= 30; dy += 3) + for (let dx = -40; dx <= 40; dx++) { + const at = { x: rect.width / 2 + dx, y: rect.height / 2 + dy }, + pick = engine.picker.pick(at), + id = pick?.id?.id ?? pick?.id; + if (typeof id === 'string' && id.startsWith(prefix)) + return { x: at.x + rect.x, y: at.y + rect.y, id }; + } + return null; + }, + panorama ? 'pnx:seq:' : 'kv:line:', + ); + assert.ok(xy, 'native ground line was pickable'); + console.log('pick', xy); + await p.mouse.click(xy.x, xy.y); + await until( + () => + window.qaStreet.getUIState().street.open && + window.qaStreet.getUIState().sequence.images > 0 && + (document.querySelector('#sl-viewer img')?.naturalWidth > 0 || + document.querySelector('#sl-viewer canvas:not([hidden])')?.width > + 0), + ); + await strict.passed( + p, + 'street-level', + 'a click on the coverage line opens its sequence and photo', + xy, + ); + return xy; + }; + const xy = await pickLine(); + if (panorama) { + const expected = panoramaxPictures().filter( + (p) => p.sequenceId === xy.id.slice('pnx:seq:'.length), + ).length; + await p.waitForFunction( + (n) => window.qaStreet.getUIState().sequence.images === n, + { timeout: 20000 }, + expected, + ); } - }; - await until(() => window.qaStreet.getUIState().coverage.count > 0); - // The fixture drive passes under the middle of the canvas. Click its actual picked line. - const xy = await p.evaluate( - (prefix) => { - const { engine } = window.__godsEyeView, - canvas = engine.host.canvas, - rect = canvas.getBoundingClientRect(); - for (let dy = -30; dy <= 30; dy += 3) - for (let dx = -40; dx <= 40; dx++) { - const at = { x: rect.width / 2 + dx, y: rect.height / 2 + dy }, - pick = engine.picker.pick(at); - if (typeof pick?.id === 'string' && pick.id.startsWith(prefix)) - return { x: at.x + rect.x, y: at.y + rect.y, id: pick.id }; - } - return null; - }, - panorama ? 'pnx:seq:' : 'kv:line:', - ); - assert.ok(xy, 'native ground line was pickable'); - console.log('pick', xy); - await p.mouse.click(xy.x, xy.y); - await until( - () => - window.qaStreet.getUIState().street.open && - window.qaStreet.getUIState().sequence.images > 0 && - (document.querySelector('#sl-viewer img')?.naturalWidth > 0 || - document.querySelector('#sl-viewer canvas:not([hidden])')?.width > 0), - ); - if (panorama) { - const expected = panoramaxPictures().filter( - (p) => p.sequenceId === xy.id.slice('pnx:seq:'.length), - ).length; - await p.waitForFunction( - (n) => window.qaStreet.getUIState().sequence.images === n, - { timeout: 20000 }, - expected, + const first = await p.evaluate(() => { + window.streetPhoto = + document.querySelector('#sl-viewer canvas:not([hidden])') ?? + document.querySelector('#sl-viewer img'); + return window.qaStreet.getUIState(); + }); + assert.ok( + (panorama ? PNX_SEQUENCES : [KV_SEQUENCE_ID]).includes( + first.street.sequenceId, + ), ); - } - const first = await p.evaluate(() => { - window.streetPhoto = - document.querySelector('#sl-viewer canvas:not([hidden])') ?? - document.querySelector('#sl-viewer img'); - return window.qaStreet.getUIState(); - }); - assert.ok( - (panorama ? PNX_SEQUENCES : [KV_SEQUENCE_ID]).includes( - first.street.sequenceId, - ), - ); - const reads = photoRequests; - for (const kind of ['cesium', 'navara']) { - await p.select('#globe-engine', kind); - await ready(kind); - const state = await p.evaluate(() => ({ - model: - window.__godsEyeView.dataManager.layers.get('street-level').module === - window.qaStreet, - samePhoto: - document.querySelector('#sl-viewer')?.contains(window.streetPhoto) && - window.streetPhoto.isConnected, - ui: window.qaStreet.getUIState(), - })); + const reads = photoRequests; + for (const kind of ['cesium', 'navara']) { + await strict.pick(p, kind); + await ready(kind); + // The new globe reloads coverage right after it attaches: compare once + // that load is over. + await until( + () => + !window.qaStreet.getUIState().coverage.loading && + window.qaStreet.getUIState().coverage.count > 0, + ); + const state = await p.evaluate(() => ({ + model: + window.__godsEyeView.dataManager.layers.get('street-level').module === + window.qaStreet, + samePhoto: + document.querySelector('#sl-viewer')?.contains(window.streetPhoto) && + window.streetPhoto.isConnected, + ui: window.qaStreet.getUIState(), + })); + console.log( + kind, + state.model, + state.samePhoto, + state.ui.street.imageId, + state.ui.sequence.images, + state.ui.coverage.count, + ); + assert.equal(state.model, true); + assert.equal(state.samePhoto, true); + assert.ok(state.ui.street.open); + assert.equal(state.ui.street.imageId, first.street.imageId); + assert.equal(state.ui.sequence.images, first.sequence.images); + assert.deepEqual(state.ui.filter, first.filter); + assert.ok(state.ui.coverage.count > 0); + await strict.passed( + p, + 'street-level', + 'same model, the same mounted photo, image and sequence; coverage kept', + ); + } + assert.equal(photoRequests, reads); + await strict.passed( + p, + 'street-level', + 'no photo requested again by the switches', + ); + // Click-picking on Cesium: close the viewer, switch, click the line again. + await p.evaluate(() => window.qaStreet.closeViewer()); + await until(() => !window.qaStreet.getUIState().street.open); + await strict.pick(p, 'cesium'); + await ready('cesium'); + await until( + () => + !window.qaStreet.getUIState().coverage.loading && + window.qaStreet.getUIState().coverage.count > 0, + ); + await new Promise((resolve) => setTimeout(resolve, 1000)); + await pickLine(); + assert.deepEqual(errors, []); + assert.deepEqual(fixture.unknown, []); console.log( - kind, - state.model, - state.samePhoto, - state.ui.street.imageId, - state.ui.sequence.images, - state.ui.coverage.count, + 'PASS native line pick, same mounted photo and sequence across two switches', ); - assert.equal(state.model, true); - assert.equal(state.samePhoto, true); - assert.ok(state.ui.street.open); - assert.equal(state.ui.street.imageId, first.street.imageId); - assert.equal(state.ui.sequence.images, first.sequence.images); - assert.deepEqual(state.ui.filter, first.filter); - assert.ok(state.ui.coverage.count > 0); - } - assert.equal(photoRequests, reads); - assert.deepEqual(errors, []); - assert.deepEqual(fixture.unknown, []); - console.log( - 'PASS native line pick, same mounted photo and sequence across two switches', - ); -} finally { - await browser.close(); -} + }, +); diff --git a/scripts/qa-strict-cold-chunks.mjs b/scripts/qa-strict-cold-chunks.mjs new file mode 100644 index 000000000..37f79e157 --- /dev/null +++ b/scripts/qa-strict-cold-chunks.mjs @@ -0,0 +1,95 @@ +// Contract: a cold start loads only the engine it starts. From the network +// log of a fresh page: +// - a cold Navara start requests nothing of the Cesium library (/cesium/…: +// Cesium.js, its workers and assets); +// - a cold Cesium start requests no chunk that carries three or Navara code +// (the FORBIDDEN_MARKERS of src/tooling/mainChunk.js) and no Navara wasm. +// Control: after a pick to the other engine, its code is requested, so the +// detection sees what it is looking for. +// +// node scripts/qa-strict-cold-chunks.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { forbiddenMarkersIn } from '../src/tooling/mainChunk.js'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { installSessionFixtures, openApp } from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +const contents = new Map(); +// The preview's own file: three or Navara markers in it? +const markersOf = async (url) => { + if (!contents.has(url)) + contents.set( + url, + fetch(url) + .then((r) => r.text()) + .then(forbiddenMarkersIn), + ); + return contents.get(url); +}; +const cesiumCode = (url) => new URL(url).pathname.startsWith('/cesium/'); +const navaraCode = async (url) => { + const { pathname } = new URL(url); + if (/navara.*\.wasm$/.test(pathname)) return ['navara wasm']; + if (!pathname.endsWith('.js') || pathname.startsWith('/cesium/')) return []; + return markersOf(url); +}; +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + for (const start of ['navara', 'cesium']) { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const urls = []; + p.on('request', (request) => { + if (request.url().startsWith(base.origin)) urls.push(request.url()); + }); + await installSessionFixtures(p); + await openApp(p, base.origin, start); + // Let lazy loads that follow the start (workers, idle prefetch) happen. + await new Promise((resolve) => setTimeout(resolve, 5000)); + const before = [...urls]; + const other = start === 'navara' ? 'cesium' : 'navara'; + await strict.pick(p, other); + await waitForEngine(p, other); + const after = urls.slice(before.length); + const otherCode = async (list) => { + const found = []; + for (const url of list) + if ( + other === 'cesium' + ? cesiumCode(url) + : (await navaraCode(url)).length > 0 + ) + found.push(new URL(url).pathname); + return [...new Set(found)]; + }; + const row = { + start, + otherEngine: other, + loadedAtColdStart: await otherCode(before), + loadedAfterPick: (await otherCode(after)).length, + }; + results.push(row); + console.log( + row.loadedAtColdStart.length ? 'FAIL' : 'PASS', + JSON.stringify(row), + ); + await p.close(); + } + for (const row of results) + assert.ok( + row.loadedAfterPick > 0, + `control: picking ${row.otherEngine} loads its code`, + ); + assert.deepEqual( + results.filter((row) => row.loadedAtColdStart.length), + [], + 'a cold start loaded the other engine', + ); +}); diff --git a/scripts/qa-strict-context-loss.mjs b/scripts/qa-strict-context-loss.mjs new file mode 100644 index 000000000..0bf428d8c --- /dev/null +++ b/scripts/qa-strict-context-loss.mjs @@ -0,0 +1,235 @@ +// Contract: a WebGL context loss during a switch (WEBGL_lose_context, the +// context coming back 1 s later with restoreContext(), as after a GPU +// reset). After each case the page runs one engine on the same session, the +// history matches the outcome (one entry for a completed switch, none for a +// rollback), the running engine draws a non-blank frame, and a later pick +// back to the engine that lost its context draws a non-blank frame too. +// +// - outgoing: the running engine's context is lost by the same input that +// starts the switch (cold Navara→Cesium, then warm Cesium→Navara); +// - incoming: the parked target engine's context is lost by the input that +// asks for it (warm, both directions). +// +// node scripts/qa-strict-context-loss.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const other = (kind) => (kind === 'navara' ? 'cesium' : 'navara'); +const strict = createStrict({ origin: base.origin, layers: [] }); +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + let p = null; + const errors = []; + // A fresh page on `from`; warm visits the other engine first. + const fresh = async (from, warm) => { + await p?.close().catch(() => {}); + p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + await openApp(p, base.origin, from); + await startSession(p, ['earthquakes', 'flights']); + if (!warm) return; + for (const kind of [other(from), from]) { + await strict.pick(p, kind); + await waitForEngine(p, kind); + await rememberEngine(p); + } + }; + + // The running engine draws a non-blank frame within 15 s. + const draws = (kind) => + p.evaluate( + (kind) => + Promise.race([ + window.__qaFirstFrame(kind).then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 15000)), + ]), + kind, + ); + const settle = async () => { + await p.waitForFunction( + () => + (window.__godsEyeView && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden')) || + document.querySelector('.engine-switch-retry'), + { polling: 100, timeout: 90000 }, + ); + await new Promise((resolve) => setTimeout(resolve, 1500)); + }; + const intact = (state) => + state.running && + state.manager && + state.models && + state.enabled && + state.address === state.engine && + state.picker === state.engine; + + for (const [loser, from, temperature] of [ + ['outgoing', 'navara', 'cold'], + ['outgoing', 'cesium', 'warm'], + ['incoming', 'navara', 'warm'], + ['incoming', 'cesium', 'warm'], + ]) { + const to = other(from); + const label = `${temperature} ${from}→${to}, ${loser} context lost`; + // Cold cases, and any case after a failed one, start on a fresh page. + if (!p || temperature === 'cold' || results.at(-1)?.failed.length) + await fresh(from, temperature === 'warm'); + else if ((await sessionState(p)).engine !== from) { + await strict.pick(p, from); + await waitForEngine(p, from); + await rememberEngine(p); + } + const before = await sessionState(p); + // The input that starts the switch also loses the context; it comes + // back a second later. + await p.evaluate( + (loser, from, to) => { + const select = document.querySelector('#globe-engine'); + select.addEventListener( + 'change', + () => { + const engine = + loser === 'outgoing' + ? window.__godsEyeView.engine + : window.__qaEngines.get(to); + const canvas = engine.host.canvas; + const context = + canvas.getContext('webgl2') ?? canvas.getContext('webgl'); + const extension = context.getExtension('WEBGL_lose_context'); + window.__qaLoss = { + kind: engine.kind, + lost: false, + restored: false, + }; + canvas.addEventListener( + 'webglcontextrestored', + () => (window.__qaLoss.restoredEvent = true), + { once: true }, + ); + extension.loseContext(); + window.__qaLoss.lost = context.isContextLost(); + setTimeout(() => { + extension.restoreContext(); + window.__qaLoss.restored = true; + }, 1000); + }, + { once: true, capture: true }, + ); + }, + loser, + from, + to, + ); + try { + await strict.pick(p, to, { settle: false }); + await settle(); + await p.waitForFunction(() => window.__qaLoss?.restored, { + polling: 100, + timeout: 10000, + }); + await settle(); + } catch (error) { + results.push({ + case: label, + failed: [ + 'the switch could not be made: ' + + String(error.message).slice(0, 200), + ], + }); + console.log('FAIL', JSON.stringify(results.at(-1))); + continue; + } + const after = await sessionState(p); + const loss = await p.evaluate(() => window.__qaLoss); + const outcome = after.engine === to ? 'switched' : 'rolled back'; + if (after.running) await rememberEngine(p); + const drawn = after.running ? await draws(after.engine) : false; + console.log( + 'after', + label, + JSON.stringify({ after, loss, drawn }), + JSON.stringify( + await p.evaluate(() => ({ + toggle: document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded'), + retry: !!document.querySelector('.engine-switch-retry'), + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + loading: document.querySelector('#loading-screen')?.className, + })), + ), + ); + // Back to the engine that lost its context, with real input. + const back = after.engine === loss.kind ? other(loss.kind) : loss.kind; + let backDrawn = null, + backState = null; + let backError = null; + if (after.running) { + try { + await strict.pick(p, back); + await waitForEngine(p, back, { timeout: 60000 }); + } catch (error) { + backError = String(error.message).slice(0, 300); + } + backState = await sessionState(p); + if (backState.running) await rememberEngine(p); + backDrawn = backState.engine === back ? await draws(back) : false; + } + const row = { + case: label, + lostKind: loss.kind, + lost: loss.lost, + restoredEvent: Boolean(loss.restoredEvent), + outcome, + sessionIntact: intact(after), + history: after.history - before.history, + drawn, + back, + backEngine: backState?.engine ?? null, + backSessionIntact: backState ? intact(backState) : false, + backDrawn, + backError, + pageErrors: errors.splice(0), + }; + row.failed = [ + row.lost || 'the context was not lost', + row.sessionIntact || 'the session is not intact', + row.history === (outcome === 'switched' ? 1 : 0) || + 'the history does not match the outcome', + row.drawn || 'no non-blank frame from the running engine', + row.backEngine === back || `the pick back to ${back} did not run it`, + row.backSessionIntact || 'the session is not intact after the pick back', + row.backDrawn || `no non-blank frame from ${back} after the pick back`, + row.backError === null || 'the pick back failed: ' + row.backError, + row.pageErrors.length === 0 || 'uncaught page errors', + ].filter((v) => v !== true); + results.push(row); + console.log(row.failed.length ? 'FAIL' : 'PASS', JSON.stringify(row)); + } + assert.deepEqual( + results.filter((row) => row.failed.length), + [], + 'context-loss cases that broke the contract', + ); +}); diff --git a/scripts/qa-strict-failure-matrix.mjs b/scripts/qa-strict-failure-matrix.mjs new file mode 100644 index 000000000..c3d2a5eec --- /dev/null +++ b/scripts/qa-strict-failure-matrix.mjs @@ -0,0 +1,283 @@ +// Contract: failure injection in both directions, cold and warm, and a slow +// first frame. Each case starts on a fresh page with a session (earthquakes +// and flights, one aircraft selected) and switches with real input on the +// picker. +// +// - A failed switch rolls back to the engine that was running, with the +// same session (manager, every layer model, enabled layers, selection), +// the address still naming it and no history entry; an ordinary pick +// afterwards succeeds and adds exactly one entry. +// cold: the incoming engine cannot get a WebGL context; +// warm: the parked incoming engine fails to resume. +// - A first frame slower than 10 s (animation frames held for 12 s) does not +// roll back: the switch completes, the cover goes after the 10 s wait, one +// history entry, the session intact, and the frame arrives. +// +// node scripts/qa-strict-failure-matrix.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + assertSession, + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + SESSION_HASH, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const other = (kind) => (kind === 'navara' ? 'cesium' : 'navara'); +const strict = createStrict({ origin: base.origin, layers: [] }); +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + // A fresh page on `start` with the session; `warm` visits the other + // engine first so it is parked. + const open = async (start, warm) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.errors = []; + p.on('pageerror', (e) => p.errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + // Flights on with the first fixture aircraft tracked (the address's + // layer options), so the session carries a selection. + await openApp( + p, + base.origin, + start, + SESSION_HASH.replace(/&l=$/, '&l=f&lo=f.e.1_f.t.a00000'), + ); + await startSession(p, ['earthquakes', 'flights']); + await p.waitForFunction( + () => + window.__godsEyeView.styleManager.getAircraftTrackingTarget()?.id === + 'a00000', + { timeout: 30000 }, + ); + if (warm) { + await strict.pick(p, other(start)); + await waitForEngine(p, other(start)); + await rememberEngine(p); + await strict.pick(p, start); + await waitForEngine(p, start); + await rememberEngine(p); + // The warm-up's Navara→Cesium switch drops the tracked aircraft (a + // defect of its own, reported); track it + // again so this case tests the failure path with a selection. + await p.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('flights') + .module.trackById('a00000', { origin: 'user' }), + ); + await p.waitForFunction( + () => + window.__godsEyeView.styleManager.getAircraftTrackingTarget()?.id === + 'a00000', + { timeout: 30000 }, + ); + } + return p; + }; + const selection = (p) => + p.evaluate( + () => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id ?? + null, + ); + + for (const [start, temperature] of [ + ['navara', 'cold'], + ['cesium', 'cold'], + ['cesium', 'warm'], + ['navara', 'warm'], + ]) { + const target = other(start); + const label = `${temperature} ${start}→${target} failure`; + const p = await open(start, temperature === 'warm'); + const before = await sessionState(p); + const selected = await selection(p); + if (temperature === 'cold') + await p.evaluate(() => { + const original = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + // The running globe already holds its context; only a new + // globe asks for one. + if (/webgl/i.test(String(type))) { + window.__qaRefused = (window.__qaRefused ?? 0) + 1; + throw new Error('QA: no WebGL context for the new globe'); + } + return original.call(this, type, ...args); + }; + window.__qaRestore = () => { + HTMLCanvasElement.prototype.getContext = original; + }; + }); + else + await p.evaluate((target) => { + const parked = window.__qaEngines.get(target); + const request = parked.render.request; + parked.render.request = function (reason, ...args) { + if (reason === 'engine-resume' && !window.__qaRestored) { + window.__qaRefused = (window.__qaRefused ?? 0) + 1; + throw new Error('QA: the parked globe failed to resume'); + } + return request.call(this, reason, ...args); + }; + window.__qaRestore = () => { + window.__qaRestored = true; + }; + }, target); + await strict.pick(p, target); + await waitForEngine(p, start); + const refused = await p.evaluate(() => window.__qaRefused ?? 0); + const rolled = await assertSession(p, start, label + ': rollback'); + // The tracked aircraft, given up to 10 s to be re-acquired. + const kept = await p + .waitForFunction( + (id) => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id === + id, + { timeout: 10000, polling: 100 }, + selected, + ) + .then(() => selected) + .catch(() => selection(p)); + await p.evaluate(() => window.__qaRestore()); + // An ordinary pick afterwards. + await strict.pick(p, target); + await waitForEngine(p, target); + const retried = await assertSession(p, target, label + ': retry'); + const afterRetry = await p + .waitForFunction( + (id) => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id === + id, + { timeout: 10000, polling: 100 }, + selected, + ) + .then(() => selected) + .catch(() => selection(p)); + const row = { + case: label, + refused, + rollbackTarget: rolled.engine, + historyAfterFailure: rolled.history - before.history, + selectionKept: kept === selected, + selectionAfterRollback: kept, + historyAfterRetry: retried.history - before.history, + selectionAfterRetry: afterRetry, + errors: p.errors.length, + }; + row.failed = [ + refused > 0 || 'the injected failure did not happen', + row.rollbackTarget === start || 'rolled back elsewhere', + row.historyAfterFailure === 0 || 'the failed switch changed the history', + selected === 'a00000' || 'no tracked aircraft before the failure', + row.selectionKept || 'the tracked aircraft is lost after the rollback', + row.historyAfterRetry === 1 || 'the retry did not add one entry', + row.selectionAfterRetry === selected || + 'the tracked aircraft is lost after the retry', + ].filter((v) => v !== true); + results.push(row); + console.log(row.failed.length ? 'FAIL' : 'PASS', JSON.stringify(row)); + await p.close(); + } + + // Slow first frame: animation frames held for 12 s from the person's + // input (a stalled GPU or compositor), cold and warm. + for (const [start, temperature] of [ + ['cesium', 'cold'], + ['navara', 'warm'], + ]) { + const target = other(start); + const label = `${temperature} ${start}→${target} first frame after 12 s`; + const p = await open(start, temperature === 'warm'); + const before = await sessionState(p); + await p.evaluate((target) => { + window.__qaSlow = { frame: window.__qaFirstFrame(target) }; + window.__qaSlow.frame.then((t) => (window.__qaSlow.drawn = t)); + const select = document.querySelector('#globe-engine'); + // From the change on, every animation frame waits until 12 s after it. + select.addEventListener( + 'change', + () => { + const until = performance.now() + 12000; + window.__qaSlow.at = performance.now(); + const raf = window.requestAnimationFrame.bind(window); + window.requestAnimationFrame = (callback) => + raf((time) => { + const wait = until - performance.now(); + if (wait <= 0) return callback(time); + setTimeout(() => raf(callback), wait); + }); + window.__qaSlow.release = () => { + window.requestAnimationFrame = raf; + }; + }, + { once: true, capture: true }, + ); + new MutationObserver((_, observer) => { + if ( + window.__qaSlow.at && + !window.__qaSlow.coverGone && + !document.querySelector('.engine-switch-frame') && + !document.querySelector('#globe-engine')?.disabled + ) { + window.__qaSlow.coverGone = performance.now(); + observer.disconnect(); + } + }).observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['disabled'], + }); + }, target); + await strict.pick(p, target, { settle: false }); + await p.waitForFunction( + (target) => + window.__godsEyeView?.engine?.kind === target && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame'), + { polling: 200, timeout: 60000 }, + target, + ); + const done = await assertSession(p, target, label); + const timing = await p.evaluate(async () => { + const frame = await window.__qaSlow.frame; + return { + coverGoneMs: Math.round(window.__qaSlow.coverGone - window.__qaSlow.at), + firstFrameMs: Math.round(frame.presented - window.__qaSlow.at), + }; + }); + const row = { + case: label, + engine: done.engine, + history: done.history - before.history, + ...timing, + errors: p.errors.length, + }; + results.push(row); + assert.equal(row.history, 1, 'one history entry, no rollback'); + assert.ok( + row.coverGoneMs >= 9500 && row.coverGoneMs < 12500, + 'the cover goes after the 10 s first-frame wait ' + JSON.stringify(row), + ); + assert.ok(row.firstFrameMs >= 12000, 'the frame arrives after the stall'); + console.log('PASS', JSON.stringify(row)); + await p.close(); + } + assert.deepEqual( + results.filter((row) => row.failed?.length), + [], + 'failure-injection cases that broke the contract', + ); +}); diff --git a/scripts/qa-strict-harness.mjs b/scripts/qa-strict-harness.mjs new file mode 100644 index 000000000..ddb9dd711 --- /dev/null +++ b/scripts/qa-strict-harness.mjs @@ -0,0 +1,736 @@ +// The strict acceptance harness shared by every browser scenario that +// qa-all-layer-switches.mjs runs. +// +// - Network: a default-deny request policy on every page, frame and worker +// of the browser, installed before the target runs (CDP auto-attach with +// waitForDebuggerOnStart). Paid hosts and paid preview routes are blocked, +// then only the preview origin, the scenario's declared fixture URLs and +// the free tile hosts (answered from a disk cache) are allowed. A blocked +// request, a fixture or tile URL answered by the real network, or a socket +// to another origin is a violation and fails the scenario with its URL. +// - Coverage: the runner passes the layer ids (--layers=a,b). The scenario +// records each assertion it made per layer id and engine; finish() fails +// when a declared layer has no assertion on either engine, and writes the +// manifest (--evidence=FILE) the runner cross-checks against the catalog. +// - Engine picks: pickEngine() switches with real input on the picker (a +// pointer click on its label, then keyboard type-ahead), never page.select. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import os from 'node:os'; +import { + KEYLESS_PROBES, + probePattern, + strictBlockPatterns, + STRICT_TILE_HOSTS, +} from './qa-map-policy.mjs'; +import { createMissLoader, tileCacheFile } from './qa/tileCache.mjs'; + +export const ENGINES = Object.freeze(['cesium', 'navara']); +const RESERVED_PORTS = ['', '4173', '4185', '4186', '4187', '4188', '4189']; +const RESERVED = [...RESERVED_PORTS, '4194', '4196']; + +/** The values of --layers=a,b and --evidence=FILE (null when absent). */ +export function strictArguments(argv = process.argv) { + const value = (name) => + argv.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3); + const layers = value('layers'); + return { + layers: layers ? layers.split(',').filter(Boolean) : null, + evidence: value('evidence') ?? process.env.QA_STRICT_EVIDENCE ?? null, + }; +} + +/** Refuse anything but an unreserved local preview. */ +export function localPreview(href) { + const url = new URL(href); + if ( + !['localhost', '127.0.0.1'].includes(url.hostname) || + RESERVED.includes(url.port) + ) + throw Error('Supply an unreserved local keyless preview URL'); + return url; +} + +/** The URL with no remote IP: fulfilled by a fixture or a cache, or a data URL. */ +const live = (response) => Boolean(response.remoteIPAddress); + +/** + * Follow every request of every target of `browser` under the strict policy. + * Returns the ledger the policy fills in. + */ +export async function installStrictNetwork( + browser, + { origin, fixtures = [], routes = [], tileHosts = STRICT_TILE_HOSTS, ledger }, +) { + const preview = new URL(origin); + fixtures = [...fixtures, ...routes.map((route) => route.pattern)]; + const urlPatterns = strictBlockPatterns(origin, { fixtures, tileHosts }); + const answers = routes.map((route) => ({ + ...route, + test: new URLPattern(route.pattern), + })); + const fixturePatterns = fixtures.map((p) => new URLPattern(p)); + const isFixture = (url) => fixturePatterns.some((p) => p.test(url)); + const root = await browser.target().createCDPSession(); + const connection = root.connection(); + const probes = KEYLESS_PROBES.map((probe) => ({ + ...probe, + test: new URLPattern(probePattern(origin, probe)), + })); + const misses = createMissLoader(); + const cacheDir = path.join(os.tmpdir(), 'gev-qa-tiles'); + fs.mkdirSync(cacheDir, { recursive: true }); + const tiles = ledger.tileCache; + // One CDP Fetch handler per target: the keyless probes and the tile hosts + // (from the disk cache, filled on a miss). Nothing else is paused, so + // workers' own requests are never held. + const intercept = async (session, type) => { + session.on('Fetch.requestPaused', async ({ requestId, request }) => { + const fulfil = (status, headers, body) => + session.send('Fetch.fulfillRequest', { + requestId, + responseCode: status, + responseHeaders: headers, + ...(body == null + ? {} + : { body: Buffer.from(body).toString('base64') }), + }); + try { + const route = answers.find((r) => r.test.test(request.url)); + if (route) { + ledger.routes[route.pattern] = + (ledger.routes[route.pattern] ?? 0) + 1; + const answer = await route.respond(request); + return await fulfil( + answer.status ?? 200, + [ + ...(answer.contentType + ? [{ name: 'Content-Type', value: answer.contentType }] + : []), + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + answer.body ?? '', + ); + } + const probe = probes.find((p) => p.test.test(request.url)); + if (probe) { + if (request.method !== probe.method && request.method !== 'HEAD') { + violation('probe-method', request.url, { method: request.method }); + return await session.send('Fetch.failRequest', { + requestId, + errorReason: 'BlockedByClient', + }); + } + ledger.probes[probe.pattern] = + (ledger.probes[probe.pattern] ?? 0) + 1; + return await fulfil( + probe.status, + [ + ...(probe.contentType + ? [{ name: 'Content-Type', value: probe.contentType }] + : []), + { name: 'Cache-Control', value: 'no-store' }, + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + probe.body ?? null, + ); + } + const accept = Object.entries(request.headers ?? {}).find( + ([name]) => name.toLowerCase() === 'accept', + )?.[1]; + const file = + request.method === 'GET' + ? tileCacheFile(request.url, accept, cacheDir, tileHosts) + : null; + if (!file) { + violation('unhandled-intercept', request.url, { target: type }); + return await session.send('Fetch.failRequest', { + requestId, + errorReason: 'BlockedByClient', + }); + } + let entry; + if (fs.existsSync(`${file}.json`)) { + entry = { + ...JSON.parse(fs.readFileSync(`${file}.json`, 'utf8')), + body: fs.readFileSync(file), + }; + tiles.hits++; + } else { + entry = await misses.load(request.url, accept); + if (entry.status === 200) { + fs.writeFileSync(file, entry.body); + fs.writeFileSync( + `${file}.json`, + JSON.stringify({ status: 200, contentType: entry.contentType }), + ); + tiles.fetched++; + } else tiles.failed++; + const host = new URL(request.url).host; + tiles.fetchedByHost[host] = (tiles.fetchedByHost[host] ?? 0) + 1; + } + await fulfil( + entry.status, + [ + ...(entry.contentType + ? [{ name: 'Content-Type', value: entry.contentType }] + : []), + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + entry.body, + ); + } catch (error) { + if ( + /Invalid InterceptionId|closed|detached/i.test(String(error?.message)) + ) + return void tiles.cancelled++; + tiles.failed++; + await session + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await session + .send('Fetch.enable', { + patterns: [ + ...probes.map((probe) => ({ + urlPattern: probePattern(origin, probe).replace(/\*$/, '') + '*', + })), + ...tileHosts.map((host) => ({ urlPattern: `https://${host}/*` })), + ...routes.map((route) => ({ urlPattern: route.pattern })), + ], + }) + .catch((error) => + ledger.setupErrors.push( + type + ' Fetch.enable: ' + String(error?.message).slice(0, 120), + ), + ); + }; + const violation = (kind, url, detail = {}) => { + ledger.violations.push({ kind, url: String(url).slice(0, 300), ...detail }); + }; + const setup = async (session, info) => { + const type = info.type; + const urls = new Map(); + session.on('Network.requestWillBeSent', (event) => { + const { url } = event.request; + urls.set(event.requestId, url); + if (!/^https?:/.test(url)) return; + ledger.requests++; + const address = new URL(url); + if (address.origin === preview.origin) { + ledger.preview++; + if (address.pathname.startsWith('/api/')) { + const route = address.pathname.split('/').slice(0, 4).join('/'); + ledger.previewApi[route] = (ledger.previewApi[route] ?? 0) + 1; + } + } else { + ledger.external[address.host] = + (ledger.external[address.host] ?? 0) + 1; + } + }); + session.on('Network.loadingFailed', (event) => { + if (!event.blockedReason) return; + violation('blocked', urls.get(event.requestId) ?? '?', { + reason: event.blockedReason, + target: type, + resource: event.type, + }); + }); + session.on('Network.responseReceived', ({ response, type: resource }) => { + if (!/^https?:/.test(response.url)) return; + const address = new URL(response.url); + if (probes.some((p) => p.test.test(response.url)) && live(response)) + violation('probe-live', response.url, { target: type }); + if (address.origin === preview.origin) return; + if (!live(response)) { + if (isFixture(response.url)) ledger.fixtureResponses++; + return; + } + // A fixture or tile URL answered by the real network, or any other + // external host that slipped past the block (a frame navigation). + violation('live', response.url, { + remote: response.remoteIPAddress, + target: type, + resource, + }); + }); + session.on('Network.webSocketCreated', ({ url }) => { + const address = new URL(url); + if (address.host !== preview.host) + violation('socket', url, { target: type }); + }); + session.on('Target.attachedToTarget', (event) => + attach(event.sessionId, event.targetInfo, event.waitingForDebugger), + ); + const steps = [ + session.send('Network.enable'), + session.send('Network.setBlockedURLs', { urlPatterns }), + session + .send('Target.setAutoAttach', { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + }) + .catch(() => {}), + ]; + // Workers have no Fetch domain; the page's interception answers their + // requests, and a live answer would still show up as a violation. + if (type === 'page' || type === 'iframe') + steps.push(intercept(session, type)); + await Promise.all(steps); + }; + const attach = async (sessionId, info, waiting) => { + const session = connection.session(sessionId); + if (!session) return; + try { + await setup(session, info); + } catch (error) { + // A target that closes while it attaches has nothing left to load. + if (!/closed|detached|not found/i.test(String(error?.message))) + ledger.setupErrors.push(String(error?.message ?? error).slice(0, 200)); + } finally { + if (waiting) + await session.send('Runtime.runIfWaitingForDebugger').catch(() => {}); + } + }; + root.on('Target.attachedToTarget', (event) => + attach(event.sessionId, event.targetInfo, event.waitingForDebugger), + ); + await root.send('Target.setAutoAttach', { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + filter: [ + { type: 'page' }, + { type: 'iframe' }, + { type: 'worker' }, + { type: 'shared_worker' }, + { type: 'service_worker' }, + { type: 'tab', exclude: true }, + { type: 'browser', exclude: true }, + ], + }); + return ledger; +} + +/** A new, empty network ledger. */ +export function networkLedger() { + return { + requests: 0, + preview: 0, + previewApi: {}, + external: {}, + fixtureResponses: 0, + probes: {}, + routes: {}, + tileCache: { + hits: 0, + fetched: 0, + failed: 0, + cancelled: 0, + fetchedByHost: {}, + }, + violations: [], + setupErrors: [], + }; +} + +/** + * Switch to `kind` with real input on the Globe Engine picker: a pointer + * click on its label (which focuses the select without opening its menu) + * and keyboard type-ahead. Opens the control panel with a pointer click if + * it is collapsed and collapses it again once the switch has settled + * (`settle: false` returns as soon as the picker reports the change; + * `afterInput` runs right after the key, before anything is read back). + */ +export async function pickEngine( + page, + kind, + { settle = true, afterInput = null } = {}, +) { + assert.ok(ENGINES.includes(kind), kind); + const expanded = () => + page.$eval( + '#control-panel-toggle', + (el) => el.getAttribute('aria-expanded') === 'true', + ); + const opened = !(await expanded()); + if (opened) { + await page.click('#control-panel-toggle').catch(async (error) => { + const state = await page.evaluate(() => { + const el = document.querySelector('#control-panel-toggle'); + const box = el?.getBoundingClientRect(); + const style = el && getComputedStyle(el); + const hit = + box && + document.elementFromPoint( + box.x + box.width / 2, + box.y + box.height / 2, + ); + return { + box: box && [box.x, box.y, box.width, box.height], + display: style?.display, + visibility: style?.visibility, + hidden: el?.closest('[hidden]')?.id ?? null, + hit: hit?.id || hit?.className || null, + body: document.body.className, + }; + }); + throw Error( + `cannot click the control panel toggle (${error.message}): ` + + JSON.stringify(state), + ); + }); + await page.waitForFunction( + () => + document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded') === 'true', + ); + } + await page.waitForSelector('#globe-engine:not([disabled])', { + visible: true, + timeout: 90000, + }); + const from = await page.$eval('#globe-engine', (el) => el.value); + assert.notEqual(from, kind, `the picker already shows ${kind}`); + // Where a pointer can reach the picker: the label's own text (focuses the + // select without opening its menu), else the select itself. + const target = await page.$eval('#globe-engine', (el) => { + const box = el.closest('label').getBoundingClientRect(); + const select = el.getBoundingClientRect(); + // at: the change event (the person's input reached the page); + // idleAt: the picker enabled again after the switch. + window.__qaPick = { busy: false, changed: false, at: null, idleAt: null }; + el.addEventListener( + 'change', + () => { + window.__qaPick.changed = true; + window.__qaPick.at = performance.now(); + }, + { once: true }, + ); + window.__qaPickObserver?.disconnect(); + window.__qaPickObserver = new MutationObserver(() => { + const disabled = document.querySelector('#globe-engine')?.disabled; + if (disabled) window.__qaPick.busy = true; + else if (window.__qaPick.busy && window.__qaPick.idleAt == null) + window.__qaPick.idleAt = performance.now(); + }); + window.__qaPickObserver.observe(document.body, { + attributes: true, + attributeFilter: ['disabled'], + subtree: true, + }); + const describe = (hit) => + hit + ? `${hit.tagName.toLowerCase()}#${hit.id}.${String(hit.className).slice(0, 60)}` + : 'nothing'; + const label = { + x: box.left + Math.max(4, (select.left - box.left) / 2), + y: box.top + box.height / 2, + }; + const onLabel = document.elementFromPoint(label.x, label.y); + if ( + onLabel === el.closest('label') || + (onLabel?.closest('label') === el.closest('label') && onLabel !== el) + ) + return { ...label, via: 'label' }; + const middle = { + x: select.left + select.width / 2, + y: select.top + select.height / 2, + }; + const onSelect = document.elementFromPoint(middle.x, middle.y); + if (onSelect === el) + return { ...middle, via: 'select', labelCoveredBy: describe(onLabel) }; + return { covered: describe(onLabel) + ' / ' + describe(onSelect) }; + }); + assert.ok(!target.covered, 'the picker is covered by ' + target.covered); + await page.mouse.click(target.x, target.y); + assert.equal( + await page.evaluate(() => document.activeElement?.id), + 'globe-engine', + 'the click focuses the picker', + ); + await page.keyboard.type(kind[0]); + // Whatever the person does next, right after the key (no round trip to + // the page in between). + if (afterInput) await afterInput(); + await page.waitForFunction(() => window.__qaPick?.changed, { + timeout: 5000, + polling: 50, + }); + // Headless Chrome opens no menu for a click on the select: the key + // events above reached the page, so there is nothing to dismiss. + const how = { + opened, + clicked: target.via, + ...(target.labelCoveredBy ? { labelCoveredBy: target.labelCoveredBy } : {}), + }; + if (!settle) return how; + await page.waitForFunction( + () => + (window.__qaPick.busy && + !document.querySelector('#globe-engine')?.disabled && + document.querySelector('#loading-screen.hidden') && + !document.querySelector('.engine-switch-frame')) || + document.querySelector('.engine-switch-retry'), + { timeout: 120000, polling: 50 }, + ); + await page.evaluate(() => window.__qaPickObserver?.disconnect()); + Object.assign( + how, + await page.evaluate(() => ({ + at: window.__qaPick.at, + idleAt: window.__qaPick.idleAt, + })), + ); + if (opened && (await expanded().catch(() => false))) { + await page.click('#control-panel-toggle'); + await page.waitForFunction( + () => + document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded') !== 'true', + ); + } + return how; +} + +/** Wait until `kind` runs, its picker is idle and its view is restored. */ +export async function waitForEngine(page, kind, { timeout = 120000 } = {}) { + await page.waitForFunction( + (expected) => + window.__godsEyeView?.engine?.kind === expected && + document.querySelector('#globe-engine')?.value === expected && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { timeout }, + kind, + ); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); +} + +/** + * The strict context of one scenario run. + * + * @param {object} options + * @param {string} options.origin The preview origin. + * @param {string[]} options.layers The layer ids this scenario covers (the + * runner's --layers wins when given; both must agree). + * @param {string[]} [options.fixtures] URLPattern strings for external + * requests the scenario answers itself. + * @param {{pattern: string, respond: (request: object) => ({status?: number, + * contentType?: string, body?: string|Buffer}|Promise)}[]} [options.routes] + * External URLs the harness answers for the scenario (the pattern is both + * a URLPattern and a CDP Fetch glob, e.g. `https://i.ytimg.com/*`). + * @param {readonly string[]} [options.tileHosts] Hosts the disk cache + * answers (default STRICT_TILE_HOSTS). + * @param {string} [options.name] Scenario name for the manifest. + */ +export function createStrict({ + origin, + layers, + fixtures = [], + routes = [], + tileHosts = STRICT_TILE_HOSTS, + name = path.basename(process.argv[1] ?? 'scenario', '.mjs'), +}) { + const args = strictArguments(); + if (args.layers) + assert.deepEqual( + [...args.layers].sort(), + [...layers].sort(), + `${name} covers ${layers.join(',')}, the runner passed ${args.layers.join(',')}`, + ); + const declared = args.layers ?? layers; + const ledger = networkLedger(); + const assertions = []; + const picks = []; + const timings = []; + const extra = {}; + let finished = false; + + const record = (ids, engine, claim, detail) => { + for (const id of [ids].flat()) { + assert.ok(declared.includes(id), `${id} is not declared by ${name}`); + assert.ok(ENGINES.includes(engine), `unknown engine ${engine}`); + assertions.push({ + layer: id, + engine, + claim, + ...(detail === undefined ? {} : { detail }), + }); + } + }; + const engineOf = (page) => + page.evaluate(() => window.__godsEyeView?.engine?.kind ?? null); + + const strict = { + name, + layers: declared, + ledger, + extra, + timings, + /** Launch Chrome with the policy on every target. */ + async launch(options = {}) { + const browser = await puppeteer.launch({ headless: true, ...options }); + await installStrictNetwork(browser, { + origin, + fixtures, + routes, + tileHosts, + ledger, + }); + return browser; + }, + /** Launch, run `body(browser)`, close, then finish(). */ + async run(options, body) { + const browser = await strict.launch(options); + let failed = null; + try { + await body(browser); + } catch (error) { + failed = error; + } finally { + await browser.close().catch(() => {}); + } + await strict.finish({ failed }); + if (failed) throw failed; + }, + /** Record an assertion that already passed. */ + record, + /** Record a passed assertion on the engine the page runs now. */ + async passed(page, ids, claim, detail) { + const engine = await engineOf(page); + record(ids, engine, claim, detail); + }, + /** Assert `value` and record it on the engine the page runs now. */ + async check(page, ids, claim, value, detail) { + assert.ok( + value, + claim + (detail === undefined ? '' : ' ' + JSON.stringify(detail)), + ); + await strict.passed(page, ids, claim, detail); + }, + /** Switch with real input and log the pick. */ + async pick(page, kind, options) { + const result = await pickEngine(page, kind, options); + picks.push({ to: kind, via: 'pointer+keyboard', ...result }); + return result; + }, + /** + * Switch through the select element without input, only where no + * person can reach the picker (`reason` says why); logged as such. + */ + async selectEngine(page, kind, reason) { + assert.ok(reason, 'say why the picker cannot be used'); + await page.select('#globe-engine', kind); + picks.push({ to: kind, via: 'page.select', reason }); + }, + /** + * Switch with real input in the same document and wait until `kind` + * runs (the picker counterpart of qa-switch-globe.mjs). + */ + async switchTo(page, kind) { + const origin = await page.evaluate(() => performance.timeOrigin); + await strict.pick(page, kind); + await waitForEngine(page, kind); + assert.equal( + await page.evaluate(() => performance.timeOrigin), + origin, + 'same document after switching globes', + ); + }, + /** Write the manifest; fail on violations or missing coverage. */ + async finish({ failed = null } = {}) { + if (finished) return; + finished = true; + const coverage = Object.fromEntries( + declared.map((id) => [ + id, + Object.fromEntries( + ENGINES.map((engine) => [ + engine, + assertions.filter((a) => a.layer === id && a.engine === engine) + .length, + ]), + ), + ]), + ); + const missing = Object.entries(coverage).flatMap(([id, byEngine]) => + ENGINES.filter((engine) => !byEngine[engine]).map( + (engine) => `${id} on ${engine}`, + ), + ); + const network = { + requests: ledger.requests, + preview: ledger.preview, + previewApi: ledger.previewApi, + external: ledger.external, + fixtureResponses: ledger.fixtureResponses, + probes: ledger.probes, + routes: ledger.routes, + tileCache: ledger.tileCache, + violations: ledger.violations, + setupErrors: ledger.setupErrors, + fixtures, + tileHosts: [...tileHosts], + }; + const manifest = { + scenario: name, + argv: process.argv.slice(2), + status: failed ? 'fail' : 'pass', + error: failed ? String(failed?.stack ?? failed).slice(0, 2000) : null, + layers: declared, + coverage, + missing, + picks, + assertions, + network, + timings, + ...extra, + }; + if (args.evidence) { + fs.mkdirSync(path.dirname(args.evidence), { recursive: true }); + fs.writeFileSync( + args.evidence, + JSON.stringify(manifest, null, 2) + '\n', + ); + } + console.log( + 'STRICT', + JSON.stringify({ + scenario: name, + assertions: assertions.length, + coverage, + picks: picks.length, + requests: ledger.requests, + violations: ledger.violations.length, + }), + ); + for (const v of ledger.violations.slice(0, 20)) + console.log('NETWORK VIOLATION', JSON.stringify(v)); + if (failed) return; + assert.equal( + ledger.violations.length, + 0, + 'network policy violations: ' + + JSON.stringify(ledger.violations.slice(0, 5)), + ); + assert.deepEqual(missing, [], 'layers without an assertion'); + assert.ok( + picks.some((pick) => pick.via === 'pointer+keyboard'), + 'switch at least once with real input on the picker', + ); + }, + }; + return strict; +} diff --git a/scripts/qa-strict-hidden-switch.mjs b/scripts/qa-strict-hidden-switch.mjs new file mode 100644 index 000000000..be0ff338d --- /dev/null +++ b/scripts/qa-strict-hidden-switch.mjs @@ -0,0 +1,142 @@ +// Contract: a switch picked just before the tab is hidden finishes in the +// hidden tab without waiting for a frame (a hidden tab draws nothing) and +// without a rollback, keeps the session and adds one history entry; once the +// tab is shown again the new engine draws a non-blank first frame and the +// layers' timers poll again. Both directions, cold and then warm. +// +// node scripts/qa-strict-hidden-switch.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +import { + assertSession, + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const errors = []; + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + await openApp(p, base.origin, 'navara'); + await startSession(p, ['earthquakes', 'flights']); + const cdp = await p.createCDPSession(); + // Another tab of the same window: bringing it to the front hides ours. + const other = await b.newPage(); + await other.goto('about:blank'); + await p.bringToFront(); + const visibility = () => p.evaluate(() => document.visibilityState); + assert.equal(await visibility(), 'visible'); + const results = []; + for (const [kind, temperature] of [ + ['cesium', 'cold'], + ['navara', 'warm'], + ['cesium', 'warm'], + ]) { + const before = await sessionState(p); + await p.evaluate((kind) => { + window.__qaHidden = { frame: window.__qaFirstFrame(kind), shown: [] }; + window.__qaHidden.frame.then((t) => (window.__qaHidden.drawn = t)); + // When the tab is hidden, as the page sees it: the time and whether + // the switch (busy picker) still runs. + document.addEventListener('visibilitychange', () => + window.__qaHidden.shown.push({ + state: document.visibilityState, + at: performance.now(), + busy: !!document.querySelector('#globe-engine')?.disabled, + }), + ); + }, kind); + // The person picks, then switches tabs before the globe changes: the + // other tab comes to the front right after the key. A slow CPU (20x) + // keeps the switch running past the tab change. + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 20 }); + await strict.pick(p, kind, { + settle: false, + afterInput: () => other.bringToFront(), + }); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 1 }); + // rAF does not run in a hidden tab: poll on a timer. + await p.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { polling: 250, timeout: 60000 }, + kind, + ); + const hidden = await p.evaluate(() => ({ + state: document.visibilityState, + hide: window.__qaHidden.shown.find((e) => e.state === 'hidden') ?? null, + pick: { at: window.__qaPick.at, idleAt: window.__qaPick.idleAt }, + drawn: window.__qaHidden.drawn ?? null, + })); + await assertSession( + p, + kind, + `${temperature} switch to ${kind} while hidden`, + ); + const after = await sessionState(p); + // Shown again: the first frame arrives, the timers poll again. + const reads = await p.evaluate(() => ({ ...window.__qaReads })); + await p.bringToFront(); + assert.equal(await visibility(), 'visible'); + // A non-blank frame of the new engine once the tab shows again. + const shown = await p.evaluate(async (kind) => { + const at = performance.now(); + const frame = await Promise.race([ + window.__qaFirstFrame(kind), + new Promise((resolve) => setTimeout(() => resolve(null), 10000)), + ]); + return frame && Math.round(frame.presented - at); + }, kind); + await p.waitForFunction( + (reads) => window.__qaReads.flights > reads.flights, + { polling: 250, timeout: 30000 }, + reads, + ); + await rememberEngine(p); + const row = { + kind, + temperature, + hiddenWhileBusy: Boolean(hidden.hide?.busy), + settledWhileHidden: hidden.state === 'hidden', + // From the tab being hidden to the picker being idle again. + settleAfterHideMs: + hidden.hide && hidden.pick.idleAt + ? Math.round(hidden.pick.idleAt - hidden.hide.at) + : null, + // Observed: an engine may draw while hidden (Navara renders a requested + // frame without an animation frame); not part of the contract. + frameWhileHidden: hidden.drawn !== null, + firstFrameAfterShownMs: shown, + history: after.history - before.history, + }; + results.push(row); + assert.ok(row.hiddenWhileBusy, 'the tab is hidden while the switch runs'); + assert.ok(row.settledWhileHidden, 'the switch settles in the hidden tab'); + // A hidden tab draws nothing: no first-frame wait (10 s) there. + assert.ok( + row.settleAfterHideMs !== null && row.settleAfterHideMs < 9000, + JSON.stringify(row), + ); + assert.equal(row.history, 1, 'one history entry'); + assert.ok( + row.firstFrameAfterShownMs !== null, + 'a non-blank frame within 10 s of the tab showing', + ); + console.log('PASS', JSON.stringify(row)); + } + strict.extra.contracts = results; + assert.deepEqual(errors, []); +}); diff --git a/scripts/qa-strict-rapid-switch.mjs b/scripts/qa-strict-rapid-switch.mjs new file mode 100644 index 000000000..8bebe7356 --- /dev/null +++ b/scripts/qa-strict-rapid-switch.mjs @@ -0,0 +1,162 @@ +// Contract: rapid switches. Back-to-back picks with real input, a second +// pointer click on the picker while it is busy, a history visit superseded +// by the opposite visit before its engine is ready, and a pick superseded by +// Back before its engine is ready. After each: the expected engine runs on +// the same session, the history has exactly the expected entries, both +// engines are the warm ones (no third engine), one engine canvas is shown. +// +// node scripts/qa-strict-rapid-switch.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + assertSession, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const errors = []; + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await openApp(p, base.origin, 'navara'); + await startSession(p, ['earthquakes', 'flights']); + const engines = () => + p.evaluate(() => { + const shown = [...window.__qaEngines.values()].filter( + (engine) => engine.host.canvas.isConnected, + ).length; + return { + warm: + window.__qaEngines.get(window.__godsEyeView.engine.kind) === + window.__godsEyeView.engine, + known: window.__qaEngines.size, + shown, + }; + }); + const contract = (name, value, detail) => { + assert.ok(value, name + ' ' + JSON.stringify(detail ?? '')); + strict.extra.contracts ??= []; + strict.extra.contracts.push({ name, pass: true, detail }); + console.log('PASS', name); + }; + + // 1. Back-to-back picks; a second click on the busy picker does nothing. + const start = (await sessionState(p)).history; + for (const [i, kind] of ['cesium', 'navara', 'cesium', 'navara'].entries()) { + await strict.pick(p, kind, { settle: false }); + // The picker is disabled while the engine changes: a second pointer + // click on it (the person trying the other engine) changes nothing. + const box = await p.$eval('#globe-engine', (el) => { + const r = el.getBoundingClientRect(); + return { + x: r.x + r.width / 2, + y: r.y + r.height / 2, + disabled: el.disabled, + }; + }); + await p.mouse.click(box.x, box.y); + await waitForEngine(p, kind); + await rememberEngine(p); + const state = await assertSession(p, kind, `pick ${i + 1} to ${kind}`); + assert.equal(state.history, start + i + 1, 'one entry per pick'); + const shown = await engines(); + assert.ok(shown.warm && shown.shown === 1, JSON.stringify(shown)); + } + contract( + 'four back-to-back picks: each adds one history entry, keeps the session, reuses the warm engines and shows one canvas; a click on the busy picker changes nothing', + true, + { history: (await sessionState(p)).history - start }, + ); + + // A switch in progress, seen from the page: the engine object the page + // exposes is the starting engine until the switch is over. + const progress = () => + p.evaluate(() => { + window.__qaSteps = []; + const mark = (step) => + window.__qaSteps.push({ + step, + engine: window.__godsEyeView?.engine?.kind ?? null, + busy: !!document.querySelector('#globe-engine')?.disabled, + }); + window.__qaMark = mark; + }); + + // 2. A history visit superseded before its engine is ready: Back (to the + // Cesium entry) and, from the page's popstate handling of that Back, at + // once Forward (to the Navara entry), while Back's switch runs. + const length = (await sessionState(p)).history; + await progress(); + await p.evaluate(() => { + window.addEventListener( + 'popstate', + () => { + window.__qaMark('back'); + history.forward(); + }, + { once: true }, + ); + history.back(); + }); + await waitForEngine(p, 'navara'); + await new Promise((resolve) => setTimeout(resolve, 1500)); + await assertSession(p, 'navara', 'Forward supersedes Back'); + const visited = await sessionState(p); + const shown = await engines(); + const backSteps = await p.evaluate(() => window.__qaSteps); + contract( + 'Back then Forward before the Cesium entry is ready: Navara runs on the same session, no entry added, one canvas shown', + visited.history === length && shown.warm && shown.shown === 1, + { steps: backSteps, history: visited.history - length, ...shown }, + ); + + // 3. A pick superseded by Back before its engine is ready: the person + // picks Cesium and presses Back while it starts (Back is issued from the + // page's own change handling of the pick, so it lands mid-switch). The + // pick adds no entry and Back's entry (Cesium, the entry before) wins. + const before = await sessionState(p); + await progress(); + await p.evaluate(() => { + window.addEventListener( + 'change', + (event) => { + if (event.target?.id !== 'globe-engine') return; + window.__qaMark('pick'); + history.back(); + }, + { once: true }, + ); + }); + await strict.pick(p, 'cesium', { settle: false }); + await waitForEngine(p, 'cesium', { timeout: 30000 }); + await new Promise((resolve) => setTimeout(resolve, 1500)); + await assertSession(p, 'cesium', 'Back supersedes the pick'); + const after = await sessionState(p); + const backShown = await engines(); + const pickSteps = await p.evaluate(() => window.__qaSteps); + contract( + 'a pick superseded by Back before ready: Back’s Cesium entry runs on the same session, the pick adds no entry, one canvas shown', + after.history === before.history && + backShown.warm && + backShown.shown === 1 && + pickSteps[0]?.busy, + { steps: pickSteps, history: after.history - before.history, ...backShown }, + ); + // The page still takes an ordinary pick afterwards. + await strict.pick(p, 'navara'); + await waitForEngine(p, 'navara'); + await assertSession(p, 'navara', 'a pick after the races'); + assert.deepEqual(errors, []); +}); diff --git a/scripts/qa-strict-session.mjs b/scripts/qa-strict-session.mjs new file mode 100644 index 000000000..aa1f9aadc --- /dev/null +++ b/scripts/qa-strict-session.mjs @@ -0,0 +1,390 @@ +// Shared pieces of the strict contract scenarios (qa-strict-*.mjs): a fixed +// layer session, its identity checks, the first-frame probe and the WebGL +// object counters. Everything runs on the existing test hooks +// (window.__godsEyeView); nothing here changes the application. + +import assert from 'node:assert/strict'; +import { waitForEngine } from './qa-strict-harness.mjs'; + +/** Austin, 12 km up, OpenStreetMap (answered by the tile disk cache). */ +export const SESSION_HASH = + 'v=2&lat=30.27&lon=-97.74&alt=12000&heading=0&pitch=-90&map=osm&l='; + +/** + * In-page fixtures for the session layers: one Austin earthquake, `flights` + * aircraft in a grid around Austin, `military` aircraft, no vessels. + * The counters on window.__qaReads show every source read. + */ +export async function installSessionFixtures( + page, + { flights = 1, military = 1 } = {}, +) { + await page.evaluateOnNewDocument( + (flights, military) => { + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + window.__qaReads = { quakes: 0, flights: 0, military: 0 }; + const fetch = window.fetch.bind(window); + const side = Math.ceil(Math.sqrt(flights)); + window.fetch = (input, options) => { + const url = new URL(input?.url ?? input, document.baseURI); + const now = Math.floor(Date.now() / 1000); + if (url.hostname === 'earthquake.usgs.gov') { + window.__qaReads.quakes++; + return Promise.resolve( + Response.json({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'qa-austin', + properties: { mag: 4.1, time: Date.now(), place: 'Austin' }, + geometry: { type: 'Point', coordinates: [-97.74, 30.27, 8] }, + }, + ], + }), + ); + } + if (url.pathname === '/api/flights') { + window.__qaReads.flights++; + const states = []; + for (let i = 0; i < flights; i++) + states.push([ + (0xa00000 + i).toString(16), + 'QA' + i, + 'QA', + now, + now, + -99.5 + ((i % side) / side) * 3.5, + 28.5 + (Math.floor(i / side) / side) * 3.5, + 3000 + (i % 40) * 250, + false, + 180 + (i % 50), + (i * 37) % 360, + 0, + null, + 3000 + (i % 40) * 250, + null, + false, + 0, + ]); + return Promise.resolve(Response.json({ time: now, states })); + } + if (url.pathname === '/api/military') { + window.__qaReads.military++; + return Promise.resolve( + Response.json({ + now: Date.now(), + ac: Array.from({ length: military }, (_, i) => ({ + hex: (0xae0000 + i).toString(16), + flight: 'QAMIL' + i, + lon: -97.9 + i * 0.01, + lat: 30.1 + i * 0.01, + alt_geom: 20000, + alt_baro: 20000, + gs: 300, + track: 90, + seen: 0, + seen_pos: 0, + dbFlags: 1, + type: 'adsb_icao', + })), + }), + ); + } + if (url.pathname.startsWith('/api/flights/')) + return Promise.resolve(Response.json({ path: [] })); + if (url.pathname === '/api/vessels') + return Promise.resolve(Response.json({ status: 'live', rows: [] })); + return fetch(input, options); + }; + }, + flights, + military, + ); +} + +/** Open the app on `engine` with the session hash and wait until it runs. */ +export async function openApp(page, origin, engine, hash = SESSION_HASH) { + const url = new URL(origin); + url.search = `?engine=${engine}&welcome=0`; + url.hash = hash; + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + await waitForEngine(page, engine); + await page.evaluate(() => { + if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) + throw Error('Keyless preview required'); + window.__qaEngines = new Map([ + [window.__godsEyeView.engine.kind, window.__godsEyeView.engine], + ]); + }); +} + +/** Enable `layers` and remember the session's identity. */ +export async function startSession(page, layers) { + await page.evaluate(async (layers) => { + const { dataManager } = window.__godsEyeView; + for (const id of layers) + await dataManager.setEnabled(id, true, { origin: 'user' }); + window.__qaSession = { + manager: dataManager, + models: new Map( + [...dataManager.layers].map(([id, entry]) => [id, entry.module]), + ), + enabled: [...dataManager.getEnabledLayerIds()].sort(), + }; + }, layers); +} + +/** + * The session as the page has it now against the one startSession saw: + * the same manager, every layer model, the same enabled layers. + */ +export function sessionState(page) { + return page.evaluate(() => { + const app = window.__godsEyeView; + const session = window.__qaSession; + if (!app) return { running: false }; + const { dataManager } = app; + return { + running: true, + engine: app.engine.kind, + manager: dataManager === session.manager, + models: + session.models.size === dataManager.layers.size && + [...session.models].every( + ([id, model]) => dataManager.layers.get(id)?.module === model, + ), + enabled: + JSON.stringify([...dataManager.getEnabledLayerIds()].sort()) === + JSON.stringify(session.enabled), + address: new URL(location.href).searchParams.get('engine'), + picker: document.querySelector('#globe-engine')?.value ?? null, + history: history.length, + }; + }); +} + +/** Assert the session is intact on `engine`. */ +export async function assertSession(page, engine, message) { + const state = await sessionState(page); + assert.deepEqual( + { + running: state.running, + engine: state.engine, + manager: state.manager, + models: state.models, + enabled: state.enabled, + address: state.address, + picker: state.picker, + }, + { + running: true, + engine, + manager: true, + models: true, + enabled: true, + address: engine, + picker: engine, + }, + message, + ); + return state; +} + +/** Remember the running engine object under its kind (for warm switches). */ +export function rememberEngine(page) { + return page.evaluate(() => { + const { engine } = window.__godsEyeView; + window.__qaEngines.set(engine.kind, engine); + return engine.kind; + }); +} + +/** + * Install (before navigation) the first-frame probe: + * window.__qaFirstFrame(kind) resolves with the performance.now() times of + * the first frame of the `kind` engine that is drawn into its connected + * canvas with a non-blank picture (postRender, the canvas read back in the + * same task: more than 1% of a 48x48 sample differs from its first pixel), + * and of the animation frame after it (presented). + */ +export async function installFirstFrameProbe(page) { + await page.evaluateOnNewDocument(() => { + window.__qaFirstFrame = (kind) => + new Promise((resolve) => { + const probe = document.createElement('canvas'); + probe.width = probe.height = 48; + const context = probe.getContext('2d', { willReadFrequently: true }); + const nonBlank = (canvas) => { + try { + context.clearRect(0, 0, 48, 48); + context.drawImage(canvas, 0, 0, 48, 48); + const d = context.getImageData(0, 0, 48, 48).data; + let differing = 0; + for (let i = 4; i < d.length; i += 4) + if ( + Math.abs(d[i] - d[0]) + + Math.abs(d[i + 1] - d[1]) + + Math.abs(d[i + 2] - d[2]) > + 24 + ) + differing++; + return differing > 23; + } catch { + return false; + } + }; + let attached = null; + let remove = () => {}; + const attach = (engine) => { + attached = engine; + remove = engine.frames.postRender.addEventListener(() => { + if (!engine.host.canvas.isConnected) return; + if (!nonBlank(engine.host.canvas)) return; + const drawn = performance.now(); + remove(); + clearInterval(timer); + requestAnimationFrame(() => + resolve({ drawn, presented: performance.now() }), + ); + }); + }; + const known = window.__qaEngines?.get(kind); + if (known) attach(known); + // A cold engine: attach as soon as the page exposes it. + const timer = setInterval(() => { + const engine = window.__godsEyeView?.engine; + if (!attached && engine?.kind === kind) attach(engine); + }, 2); + }); + }); +} + +/** + * Count live WebGL objects per context (before navigation): every + * create/delete pair of buffers, textures, programs, shaders, + * framebuffers, renderbuffers and vertex arrays (WebGL2 and the OES + * extension), per context, on window.__qaGl. + */ +export async function installWebGlCounters(page) { + await page.evaluateOnNewDocument(() => { + const KINDS = { + Buffer: 'buffers', + Texture: 'textures', + Program: 'programs', + Shader: 'shaders', + Framebuffer: 'framebuffers', + Renderbuffer: 'renderbuffers', + VertexArray: 'vaos', + }; + const entries = new WeakMap(); + const live = new WeakSet(); + const contexts = []; + window.__qaGl = { contexts }; + const fresh = () => + Object.fromEntries(Object.values(KINDS).map((key) => [key, 0])); + const getContext = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + const context = getContext.call(this, type, ...args); + if (context && /webgl/i.test(type) && !entries.has(context)) { + const entry = { + canvas: this, + context, + type, + live: fresh(), + created: fresh(), + }; + entries.set(context, entry); + contexts.push(entry); + } + return context; + }; + const wrap = (target, create, destroy, key, owner = (self) => self) => { + const make = target[create]; + const drop = target[destroy]; + if (typeof make !== 'function') return; + target[create] = function (...args) { + const object = make.apply(this, args); + const entry = entries.get(owner(this)); + if (object && entry) { + entry.live[key]++; + entry.created[key]++; + live.add(object); + } + return object; + }; + target[destroy] = function (object, ...args) { + const entry = entries.get(owner(this)); + if (object && entry && live.has(object)) { + live.delete(object); + entry.live[key]--; + } + return drop.call(this, object, ...args); + }; + }; + for (const proto of [ + WebGLRenderingContext.prototype, + globalThis.WebGL2RenderingContext?.prototype, + ].filter(Boolean)) + for (const [name, key] of Object.entries(KINDS)) + wrap(proto, 'create' + name, 'delete' + name, key); + // WebGL1 vertex arrays come from an extension object. + const getExtension = WebGLRenderingContext.prototype.getExtension; + const vaoOwners = new WeakMap(); + WebGLRenderingContext.prototype.getExtension = function (name, ...args) { + const extension = getExtension.call(this, name, ...args); + if ( + extension && + name === 'OES_vertex_array_object' && + !vaoOwners.has(extension) + ) { + vaoOwners.set(extension, this); + wrap( + extension, + 'createVertexArrayOES', + 'deleteVertexArrayOES', + 'vaos', + (self) => vaoOwners.get(self), + ); + } + return extension; + }; + window.__qaGlCounts = () => { + const engines = window.__qaEngines ?? new Map(); + return contexts.map((entry) => { + let owner = 'other'; + for (const [kind, engine] of engines) + if (engine.host?.canvas === entry.canvas) owner = kind; + return { + owner, + connected: entry.canvas.isConnected, + lost: entry.context.isContextLost(), + total: Object.values(entry.live).reduce((a, b) => a + b, 0), + ...entry.live, + }; + }); + }; + }); +} + +/** Least-squares slope of y over x. */ +export function slope(points) { + const n = points.length; + if (n < 2) return 0; + const mx = points.reduce((s, p) => s + p[0], 0) / n; + const my = points.reduce((s, p) => s + p[1], 0) / n; + let num = 0, + den = 0; + for (const [x, y] of points) { + num += (x - mx) * (y - my); + den += (x - mx) ** 2; + } + return den ? num / den : 0; +} + +/** The q-quantile (nearest rank) of numbers. */ +export function quantile(values, q) { + const sorted = [...values].sort((a, b) => a - b); + if (!sorted.length) return null; + return sorted[Math.max(0, Math.ceil(sorted.length * q) - 1)]; +} diff --git a/scripts/qa-strict-session.test.mjs b/scripts/qa-strict-session.test.mjs new file mode 100644 index 000000000..1522bbe53 --- /dev/null +++ b/scripts/qa-strict-session.test.mjs @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { quantile, slope } from './qa-strict-session.mjs'; +import { strictArguments } from './qa-strict-harness.mjs'; + +test('slope is the least-squares growth per step', () => { + assert.equal(slope([[0, 5]]), 0); + assert.equal( + slope([ + [20, 100], + [50, 103], + [100, 108], + ]).toFixed(3), + '0.100', + ); + assert.equal( + slope([ + [1, 7], + [2, 7], + [3, 7], + ]), + 0, + ); +}); + +test('quantile takes the nearest rank', () => { + assert.equal(quantile([], 0.5), null); + assert.equal(quantile([5, 1, 3, 2, 4], 0.5), 3); + assert.equal( + quantile( + [...Array(100)].map((_, i) => i + 1), + 0.95, + ), + 95, + ); +}); + +test('the runner hands each scenario its layers and its manifest path', () => { + assert.deepEqual( + strictArguments(['node', 'x', '--layers=a,b', '--evidence=/tmp/e.json']), + { layers: ['a', 'b'], evidence: '/tmp/e.json' }, + ); + assert.deepEqual(strictArguments(['node', 'x', '--layers=']), { + layers: null, + evidence: process.env.QA_STRICT_EVIDENCE ?? null, + }); +}); diff --git a/scripts/qa-strict-soak.mjs b/scripts/qa-strict-soak.mjs new file mode 100644 index 000000000..d5d3ffbd9 --- /dev/null +++ b/scripts/qa-strict-soak.mjs @@ -0,0 +1,378 @@ +// Soak and switch timing under a mixed workload: radio playing (750 +// stations), earthquakes, 10 000 flights, 200 military aircraft, wind with +// its GPU particles, and the traffic simulation, all from fixtures. +// +// Each run starts a fresh browser on one engine and switches with real +// input on the picker. Run r starts on Navara when r is odd and on Cesium +// when it is even, so the first switch of each run is a cold one in either +// direction. +// +// "First frame" of a switch: the first postRender of the incoming engine +// whose picture, read back from its canvas in that callback, is not blank, +// then the next animation frame (presented). Timed in the page from the +// picker's change event, so no CDP round trip is included. +// +// Checked on every switch: the same session (manager and every layer +// model), the radio still playing the same station with one play() call. +// With --assert (the long run), after the warm-up (switch 20): live WebGL +// objects per engine context grow by at most 0.1 per switch (least-squares +// slope over the checkpoints) and by at most 64 overall; no new WebGL +// context; JS heap slope at most 10 KiB per switch; no new document; the +// webglcontextlost/restored listener counts on both engine canvases stay as +// they were (each canvas keeps its webglcontextlost listener). +// +// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50 +// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + installFirstFrameProbe, + installSessionFixtures, + installWebGlCounters, + openApp, + quantile, + rememberEngine, + slope, + startSession, +} from './qa-strict-session.mjs'; +import { installRadioFixture, stations } from './fixtures/w2/radio.mjs'; +import { trafficFixtureResponse } from './traffic-fixtures.mjs'; + +const base = localPreview(process.argv[2]); +const option = (name, fallback) => { + const arg = process.argv.find((a) => a.startsWith(`--${name}=`)); + return arg ? Number(arg.slice(name.length + 3)) : fallback; +}; +const runs = option('runs', 6); +const swaps = option('swaps', 50); +const strictMode = process.argv.includes('--assert'); +assert.ok( + Number.isInteger(swaps) && swaps >= 4 && swaps % 2 === 0, + 'even --swaps', +); +const WARM_UP = 20; +const CHECKPOINTS = [ + 2, 10, 20, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, +]; +const HASH = + 'v=2&lat=30.2670&lon=-97.7440&alt=3000&heading=0&pitch=-90&map=osm&l='; +const LAYERS = [ + 'earthquakes', + 'flights', + 'military', + 'wind', + 'traffic', + 'radio', +]; + +const strict = createStrict({ + origin: base.origin, + layers: [], + // Road tiles of the traffic simulation, from scripts/traffic-fixtures.mjs. + fixtures: ['https://tiles.openfreemap.org/*'], +}); + +/** Wind and traffic fixtures on their own CDP Fetch patterns. */ +async function installWorkloadFixtures(page) { + const session = await page.createCDPSession(); + const grid = (u) => { + const kind = u.searchParams.get('overlay'); + const count = 36 * 19; + const values = new Float32Array(count * (kind ? 3 : 2)); + values.fill(8, 0, count); + values.fill(6, count, count * 2); + if (kind) values.fill(40, count * 2); + return Buffer.from(values.buffer); + }; + session.on('Fetch.requestPaused', async ({ requestId, request }) => { + const u = new URL(request.url); + let answer; + try { + if (u.pathname.startsWith('/api/wind/')) { + const model = u.searchParams.get('model'); + answer = u.pathname.endsWith('/manifest') + ? { + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + model, + overlay: 'none', + grid: { nx: 36, ny: 19, lo1: 0, la1: 90, dx: 10, dy: 10 }, + gridUrl: '/api/wind/grid/' + model + '-qa.bin' + u.search, + cycle: { + validIso: '2026-11-02T12:00:00Z', + runIso: '2026-11-02T06:00:00Z', + }, + }), + } + : { + status: 200, + contentType: 'application/octet-stream', + body: grid(u), + }; + } else if (u.pathname === '/api/tomtom/status') + answer = { + status: 200, + contentType: 'application/json', + body: '{"hasKey":false}', + }; + else answer = trafficFixtureResponse({ url: () => request.url }); + await session.send('Fetch.fulfillRequest', { + requestId, + responseCode: answer.status, + responseHeaders: [ + { name: 'Content-Type', value: answer.contentType }, + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + body: Buffer.from(answer.body).toString('base64'), + }); + } catch { + await session + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await session.send('Fetch.enable', { + patterns: [ + { urlPattern: `${base.origin}/api/wind/*` }, + { urlPattern: `${base.origin}/api/tomtom/*` }, + { urlPattern: 'https://tiles.openfreemap.org/*' }, + ], + }); +} + +const report = { runs: [], swaps, strict: strictMode }; +strict.extra.soak = report; + +for (let run = 1; run <= runs; run++) { + const start = run % 2 ? 'navara' : 'cesium'; + const browser = await strict.launch({ + args: ['--use-angle=metal', '--enable-gpu'], + }); + const row = { run, start, swaps: [], checkpoints: [], errors: [] }; + report.runs.push(row); + try { + const p = await browser.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.on('pageerror', (e) => row.errors.push(e.message.slice(0, 300))); + await installRadioFixture(p); + await installSessionFixtures(p, { flights: 10000, military: 200 }); + await installWebGlCounters(p); + await installFirstFrameProbe(p); + await installWorkloadFixtures(p); + const cdp = await p.createCDPSession(); + await openApp(p, base.origin, start, HASH); + await startSession(p, LAYERS); + await p.waitForFunction( + () => { + const layers = window.__godsEyeView.dataManager.layers; + const stat = (id) => layers.get(id).module.getStats(); + return ( + stat('radio').count === 750 && + stat('flights').count > 0 && + stat('military').count > 0 && + stat('traffic').count >= 10 && + layers.get('wind').module.getDiagnostics().gpu?.ready + ); + }, + { timeout: 90000, polling: 500 }, + ); + await p.evaluate((id) => { + const radio = window.__godsEyeView.dataManager.layers.get('radio').module; + radio.selectStation(id, { autoplay: true, focus: false, origin: 'user' }); + window.__qaRadio.station = id; + }, stations[10].id); + await p.waitForFunction( + () => + window.__qaRadio.plays.length === 1 && + window.__godsEyeView.dataManager.layers.get('radio').module.getUIState() + .audioState === 'playing', + ); + row.workload = await p.evaluate(() => { + const layers = window.__godsEyeView.dataManager.layers; + return Object.fromEntries( + ['flights', 'military', 'traffic', 'radio', 'earthquakes'].map((id) => [ + id, + layers.get(id).module.getStats().count, + ]), + ); + }); + const listeners = async () => { + const { result } = await cdp.send('Runtime.evaluate', { + includeCommandLineAPI: true, + returnByValue: true, + expression: `(() => Object.fromEntries([...window.__qaEngines].map(([kind, engine]) => { + const events = getEventListeners(engine.host.canvas); + return [kind, [events.webglcontextlost?.length ?? 0, events.webglcontextrestored?.length ?? 0]]; + })))()`, + }); + return result.value; + }; + const seen = new Set([start]); + let kind = start; + for (let i = 1; i <= swaps; i++) { + kind = kind === 'navara' ? 'cesium' : 'navara'; + const cold = !seen.has(kind); + seen.add(kind); + await p.evaluate((kind) => { + window.__qaFrame = window.__qaFirstFrame(kind); + }, kind); + const pick = await strict.pick(p, kind); + await waitForEngine(p, kind); + const frame = await p.evaluate(() => window.__qaFrame); + await rememberEngine(p); + const state = await p.evaluate(() => { + const { dataManager } = window.__godsEyeView; + const session = window.__qaSession; + const radio = dataManager.layers.get('radio').module; + return { + session: + dataManager === session.manager && + [...session.models].every( + ([id, model]) => dataManager.layers.get(id)?.module === model, + ), + radio: + window.__qaRadio.plays.length === 1 && + radio.getUIState().audioState === 'playing' && + radio.getUIState().selected?.id === window.__qaRadio.station, + }; + }); + assert.ok( + state.session && state.radio, + `swap ${i}: ${JSON.stringify(state)}`, + ); + row.swaps.push({ + i, + to: kind, + cold, + firstFrameMs: Math.round((frame.presented - pick.at) * 10) / 10, + drawnMs: Math.round((frame.drawn - pick.at) * 10) / 10, + settledMs: + pick.idleAt == null + ? null + : Math.round((pick.idleAt - pick.at) * 10) / 10, + }); + if (CHECKPOINTS.includes(i) || i === swaps) { + await cdp.send('HeapProfiler.collectGarbage'); + const heap = await cdp.send('Runtime.getHeapUsage'); + const dom = await cdp.send('Memory.getDOMCounters'); + row.checkpoints.push({ + swap: i, + engine: kind, + heapMiB: Math.round((heap.usedSize / 1048576) * 100) / 100, + ...dom, + listeners: await listeners(), + webgl: await p.evaluate(() => window.__qaGlCounts()), + }); + } + } + } finally { + await browser.close().catch(() => {}); + } + const warm = (to) => + row.swaps.filter((s) => !s.cold && s.to === to).map((s) => s.firstFrameMs); + row.summary = Object.fromEntries( + ['navara', 'cesium'].map((to) => [ + to, + { + cold: + row.swaps.find((s) => s.cold && s.to === to)?.firstFrameMs ?? null, + warmP50: quantile(warm(to), 0.5), + warmP95: quantile(warm(to), 0.95), + }, + ]), + ); + console.log( + 'RUN', + JSON.stringify({ run, start, ...row.summary, workload: row.workload }), + ); +} + +// Pooled over the runs, per direction. +const pooled = (to, cold) => + report.runs.flatMap((r) => + r.swaps + .filter((s) => s.to === to && s.cold === cold) + .map((s) => s.firstFrameMs), + ); +report.summary = Object.fromEntries( + ['navara', 'cesium'].map((to) => [ + to === 'navara' ? 'cesium→navara' : 'navara→cesium', + { + cold: pooled(to, true), + warmP50: quantile(pooled(to, false), 0.5), + warmP95: quantile(pooled(to, false), 0.95), + warmSwitches: pooled(to, false).length, + }, + ]), +); +console.log('SOAK', JSON.stringify(report.summary)); +strict.timings.push( + ...report.runs.flatMap((r) => r.swaps.map((s) => ({ run: r.run, ...s }))), +); + +if (strictMode) { + const checks = []; + for (const row of report.runs) { + const late = row.checkpoints.filter((c) => c.swap >= WARM_UP); + assert.ok(late.length >= 3, 'enough checkpoints after the warm-up'); + const first = late[0]; + const last = late.at(-1); + for (const owner of ['navara', 'cesium']) { + const totals = late.map((c) => [ + c.swap, + c.webgl + .filter((g) => g.owner === owner && !g.lost) + .reduce((s, g) => s + g.total, 0), + ]); + checks.push({ + check: `${owner} live WebGL objects`, + slopePerSwap: Math.round(slope(totals) * 1000) / 1000, + from: totals[0][1], + to: totals.at(-1)[1], + pass: slope(totals) <= 0.1 && totals.at(-1)[1] - totals[0][1] <= 64, + }); + } + checks.push({ + check: 'WebGL contexts', + from: first.webgl.length, + to: last.webgl.length, + pass: last.webgl.length === first.webgl.length, + }); + const heap = late.map((c) => [c.swap, c.heapMiB * 1024]); + checks.push({ + check: 'JS heap KiB per switch', + slopePerSwap: Math.round(slope(heap) * 100) / 100, + fromMiB: first.heapMiB, + toMiB: last.heapMiB, + pass: slope(heap) <= 10, + }); + checks.push({ + check: 'documents', + from: first.documents, + to: last.documents, + pass: last.documents <= first.documents, + }); + checks.push({ + check: 'context-loss listeners on both canvases', + from: first.listeners, + to: last.listeners, + pass: + JSON.stringify(last.listeners) === JSON.stringify(first.listeners) && + Object.keys(last.listeners).length === 2 && + Object.values(last.listeners).every(([lost]) => lost > 0), + }); + } + report.checks = checks; + for (const check of checks) + console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check)); + await strict.finish({ + failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null, + }); + if (checks.some((c) => !c.pass)) process.exitCode = 1; +} else await strict.finish(); diff --git a/scripts/qa-submarine-cables-engine-switch.mjs b/scripts/qa-submarine-cables-engine-switch.mjs index 005684ae0..ceb4079ab 100644 --- a/scripts/qa-submarine-cables-engine-switch.mjs +++ b/scripts/qa-submarine-cables-engine-switch.mjs @@ -1,26 +1,34 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { readFileSync } from 'node:fs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=55.3241&lon=10.7995&alt=25000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const CABLES = 'telegeography-submarine-cables'; +// Cables plus landing points in the bundled TeleGeography files. +const FEATURES = ['cable-geo', 'landing-point-geo'] + .map( + (name) => + JSON.parse( + readFileSync( + new URL( + `../src/data/local_data/telegeography_submarine_cables/${name}.json`, + import.meta.url, + ), + 'utf8', + ), + ).features.length, + ) + .reduce((a, b) => a + b, 0); +const strict = createStrict({ + origin: base.origin, + layers: ['telegeography-submarine-cables'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -40,7 +48,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -63,7 +71,7 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await p.waitForFunction( @@ -79,10 +87,31 @@ try { }); console.log(kind, check); assert.equal(check.same, true); - assert.equal(check.count, 2629); + assert.equal(check.count, FEATURES); assert.ok(check.referenceLabelCount <= 160); assert.equal(check.error, null); - if (kind === 'navara') { + await strict.passed( + p, + CABLES, + `same model, all ${FEATURES} cables and landing points of the bundled files, at most 160 labels`, + { count: check.count, labels: check.referenceLabelCount }, + ); + { + // Click-picking on this engine, from the starting view: a click on a + // landing point flies the camera to it. + await p.evaluate( + (destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.cancelFlight(); + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-home'); + }, + fromDegrees(10.7995, 55.3241, 25000), + ); + await new Promise((resolve) => setTimeout(resolve, 2500)); const screen = await p.evaluate( (world) => { const { engine } = window.__godsEyeView; @@ -103,13 +132,22 @@ try { }, { timeout: 20000 }, ); + await strict.passed( + p, + CABLES, + 'a click on a landing point flies the camera to it', + ); } } - assert.equal(reads.length, 2, JSON.stringify(reads)); + await strict.check( + p, + CABLES, + 'each bundled file read once', + reads.length === 2, + reads, + ); assert.deepEqual(errors, []); console.log( 'PASS full bundled cable dataset, bounded labels, landing-point fly-to and one read per file', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-traffic-engine-switch.mjs b/scripts/qa-traffic-engine-switch.mjs index 8927b35be..f98fb4598 100644 --- a/scripts/qa-traffic-engine-switch.mjs +++ b/scripts/qa-traffic-engine-switch.mjs @@ -1,27 +1,29 @@ import { trafficFixtureResponse } from './traffic-fixtures.mjs'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +import { STRICT_TILE_HOSTS } from './qa-map-policy.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=30.2670&lon=-97.7440&alt=1800&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// The scenario answers the Sentinel-2 host itself (a fixed dark ground) and +// the OpenFreeMap road tiles from its fixtures; the disk cache keeps the +// other tile hosts. +// The fixed fixture gives 1 093 vehicles and 35 congestion corridors with +// flow and 618 simulated vehicles without it; the bands allow 5%. +const BAND = process.argv.includes('--simulation') + ? { vehicles: [587, 649], corridors: [0, 0] } + : { vehicles: [1038, 1148], corridors: [33, 37] }; +const strict = createStrict({ + origin: base.origin, + layers: ['traffic'], + fixtures: ['https://tiles.maps.eox.at/*', 'https://tiles.openfreemap.org/*'], + tileHosts: STRICT_TILE_HOSTS.filter((host) => host !== 'tiles.maps.eox.at'), }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -79,7 +81,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -101,9 +103,16 @@ try { }); const waitTraffic = async () => { try { - await p.waitForFunction(() => window.qaTraffic.getStats().count >= 10, { - timeout: 40000, - }); + // Vehicles, and with flow the congestion corridors, which the layer + // publishes again once the next globe has attached. + await p.waitForFunction( + (keyless) => { + const stats = window.qaTraffic.getStats(); + return stats.count >= 10 && (keyless || stats.heatLines > 0); + }, + { timeout: 40000 }, + keyless, + ); } catch (e) { console.log( 'no traffic', @@ -190,7 +199,7 @@ try { ); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await waitTraffic(); @@ -228,9 +237,22 @@ try { assert.ok(check.heatLines > 0, 'congestion corridors are published'); if (keyless) assert.equal(check.flowBuckets.sim, check.count); else assert.ok(check.flowBuckets.slow + check.flowBuckets.jam > 0); + await strict.check( + p, + 'traffic', + keyless + ? `same module, parameters and vehicle ids; ${BAND.vehicles.join('-')} simulated vehicles, no flow` + : `same module, parameters and vehicle ids; ${BAND.vehicles.join('-')} vehicles with flow, ${BAND.corridors.join('-')} congestion corridors`, + check.count >= BAND.vehicles[0] && + check.count <= BAND.vehicles[1] && + check.heatLines >= BAND.corridors[0] && + check.heatLines <= BAND.corridors[1], + { count: check.count, corridors: check.heatLines }, + ); } if (keyless) assert.ok(![...reads.keys()].some((u) => u.includes('/tomtom/flow/'))); + // Compare the same view with the layer shown and hidden, vehicle by // vehicle. const raw = async () => @@ -333,6 +355,16 @@ try { `the population after the round trip (${count}) stays within 80-125% of the ${population} before it`, ); assert.ok(aboveGround >= 12, 'at least 12 vehicles above ground in view'); + await strict.check( + p, + 'traffic', + 'after the round trip the population stays at 80-125% and at least 90% of the vehicles above ground in view are drawn where the layer places them', + visible.drawn >= 0.9 * aboveGround && + [settledCount, finalCount].every( + (count) => count >= 0.8 * population && count <= 1.25 * population, + ), + { population, settledCount, finalCount, aboveGround, drawn: visible.drawn }, + ); assert.ok( visible.drawn >= 0.9 * aboveGround, 'at least 90% of vehicles above the ground are drawn where the layer places them', @@ -345,6 +377,4 @@ try { console.log( 'PASS kept the traffic module, parameters, flow and vehicle identities across both globes; vehicles in view are drawn where the layer places them', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-urban-engine-switch.mjs b/scripts/qa-urban-engine-switch.mjs index 5f94d2ef0..597ef41c7 100644 --- a/scripts/qa-urban-engine-switch.mjs +++ b/scripts/qa-urban-engine-switch.mjs @@ -1,29 +1,38 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; +import { fromDegrees } from '../src/geo/cartesian3.js'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { installTransitFixture } from './fixtures/w2/transit.mjs'; -import { installCombinedFixture } from './fixtures/w2/combined.mjs'; -const url = new URL(process.argv[2]), +import { + combinedPayloads, + installCombinedFixture, +} from './fixtures/w2/combined.mjs'; +const url = localPreview(process.argv[2]), bus = process.argv.includes('--transit'), id = bus ? 'transit' : 'bikeshare'; -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +// Boston, where the fixtures put the city's stations and vehicles. Transit +// starts outside every feed region (central Vermont) and is approached. +const CITY = { lat: 42.3554, lon: -71.0605, alt: 2700 }; +const start = bus ? { lat: 44.2, lon: -72.6, alt: 2700 } : CITY; url.search = '?engine=navara&welcome=0'; -url.hash = - 'v=2&lat=42.3554&lon=-71.0605&alt=2700&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +url.hash = `v=2&lat=${start.lat}&lon=${start.lon}&alt=${start.alt}&pitch=-90&heading=0&map=sentinel2&l=`; +// Every station of the bikeshare fixture reports 7 bikes and 13 docks; the +// transit fixture has 12 fresh vehicles (and one stale, dropped). +const STATIONS = combinedPayloads.bikes.length; +const VEHICLES = 12; +const strict = createStrict({ + origin: url.origin, + layers: [id], + // installCombinedFixture answers these hosts too. + fixtures: bus + ? [] + : [ + 'https://earthquake.usgs.gov/*', + 'https://tiles.dontgetflocked.com/*', + '*://*.earthdata.nasa.gov/*', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => { @@ -48,35 +57,141 @@ try { window.qaUrban = dataManager.layers.get(id).module; await dataManager.setEnabled(id, true, { origin: 'user' }); }, id); + const vehicleReads = () => + fx.requests.filter((path) => path.startsWith('/api/transit/vehicles/')) + .length; + if (bus) { + // Outside every feed region the layer waits and reads no feed. + await new Promise((resolve) => setTimeout(resolve, 3000)); + const outside = await p.evaluate(() => window.qaUrban.getStats()); + await strict.check( + p, + 'transit', + 'outside every feed region: no vehicles, no feed read, asks to move to a covered region', + outside.count === 0 && + outside.status === 'zoom-in' && + vehicleReads() === 0, + { stats: outside, reads: vehicleReads() }, + ); + // Approach the city: the MBTA feed turns on and its vehicles arrive. + await p.evaluate( + (destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-approach'); + }, + fromDegrees(CITY.lon, CITY.lat, CITY.alt), + ); + await p.waitForFunction( + (n) => + window.qaUrban.getStats().count === n && + window.qaUrban.getStats().feeds?.includes('mbta'), + { timeout: 30000 }, + VEHICLES, + ); + await strict.check( + p, + 'transit', + `approaching Boston from outside turns on the MBTA feed: ${VEHICLES} vehicles`, + vehicleReads() > 0, + { reads: vehicleReads() }, + ); + } await p.waitForFunction(() => window.qaUrban.getStats().count > 0, { timeout: 30000, }); - const handle = await p.waitForFunction( - () => { - const a = window.__godsEyeView; - for (const r of window.qaUrban.getDetectableObjects({ - maxCount: 500, - seed: 0, - })) { - const s = a.engine.projector.worldToWindow(r.position); - if (!s || s.x < 420 || s.x > 860 || s.y < 205 || s.y > 670) continue; - const pk = a.engine.picker.pick(s); - if (pk?.id === r.sourceId || pk?.id?.id === r.sourceId) - return { x: s.x, y: s.y, id: r.sourceId }; - } - return false; - }, - { timeout: 30000 }, - ); - const target = await handle.jsonValue(); + // A station or vehicle the engine's own picker returns, uncovered. + const findTarget = (key = null) => + p + .waitForFunction( + (key) => { + const a = window.__godsEyeView; + const canvas = a.engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (const r of window.qaUrban.getDetectableObjects({ + maxCount: 500, + seed: 0, + })) { + if (key && r.sourceId !== key) continue; + const s = a.engine.projector.worldToWindow(r.position); + if (!s || s.x < 420 || s.x > 860 || s.y < 205 || s.y > 600) + continue; + if ( + !canvas.contains( + document.elementFromPoint(box.left + s.x, box.top + s.y), + ) + ) + continue; + const pk = a.engine.picker.pick(s); + if (pk?.id === r.sourceId || pk?.id?.id === r.sourceId) + return { x: box.left + s.x, y: box.top + s.y, id: r.sourceId }; + } + return false; + }, + { timeout: 30000 }, + key, + ) + .then((handle) => handle.jsonValue()); + const selected = (key) => + p.waitForFunction( + (key) => + window.qaUrban + .getDetectableObjects({ maxCount: 500, seed: 0 }) + .some((r) => r.sourceId === key && r.skipLabel), + { timeout: 20000 }, + key, + ); + // A click on empty ground (nothing picked around it, nothing covering it). + const clickEmpty = async () => { + const handle = await p.waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 230; y < 600; y += 9) + for (let x = 440; x < 840; x += 9) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [8, 0], + [-8, 0], + [0, 8], + [0, -8], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ); + const at = await handle.jsonValue(); + await p.mouse.click(at.x, at.y); + await p.waitForFunction( + () => + !window.qaUrban + .getDetectableObjects({ maxCount: 500, seed: 0 }) + .some((r) => r.skipLabel), + { timeout: 15000 }, + ); + }; + const target = await findTarget(); await p.mouse.click(target.x, target.y); - await p.waitForFunction( - (key) => - window.qaUrban - .getDetectableObjects({ maxCount: 500, seed: 0 }) - .some((r) => r.sourceId === key && r.skipLabel), - {}, - target.id, + await selected(target.id); + await strict.passed( + p, + id, + bus ? 'a click on a vehicle selects it' : 'a click on a station selects it', + target, ); const reads = () => fx.requests.filter((path) => @@ -88,20 +203,43 @@ try { await p.waitForFunction( () => window.qaUrban.getDetectableObjects({ maxCount: 500 }).length > 0, ); - const original = reads(); + let original = reads(); const first = await p.evaluate(() => window.qaUrban.getAnalystRecords(500).map((v) => v.id), ); - for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); - await p.waitForFunction( - (key) => - window.qaUrban - .getDetectableObjects({ maxCount: 500, seed: 0 }) - .some((r) => r.sourceId === key && r.skipLabel), - { timeout: 20000 }, - target.id, + const checkData = async () => { + const stats = await p.evaluate(() => window.qaUrban.getStats()); + if (bus) { + await strict.check( + p, + 'transit', + `${VEHICLES} fresh vehicles (the stale one dropped)`, + stats.count === VEHICLES, + stats.count, + ); + return; + } + const rows = await p.evaluate(() => + window.qaUrban.getAnalystRecords(1000).map((r) => ({ + bikes: r.bikesAvailable, + docks: r.docksAvailable, + renting: r.renting, + })), ); + await strict.check( + p, + 'bikeshare', + `${STATIONS} stations, each with 7 bikes and 13 docks available and renting`, + stats.count === STATIONS && + rows.length === STATIONS && + rows.every((r) => r.bikes === 7 && r.docks === 13 && r.renting), + { count: stats.count, rows: rows.length, sample: rows[0] }, + ); + }; + await checkData(); + for (const kind of ['cesium', 'navara']) { + await strict.switchTo(p, kind); + await selected(target.id); const v = await p.evaluate( (id) => ({ same: @@ -116,12 +254,32 @@ try { assert.ok(v.same); assert.deepEqual(v.ids, first); assert.equal(reads(), original); + await strict.passed( + p, + id, + 'same model, same records, the clicked object still selected, no source read again', + ); + await checkData(); + if (kind === 'cesium') { + // Click-picking on Cesium: clear with a click on empty ground, then + // click the same object again. + await clickEmpty(); + const again = await findTarget(target.id); + await p.mouse.click(again.x, again.y); + await selected(target.id); + await strict.passed( + p, + id, + 'a click on empty ground clears the selection; a click on the object selects it again', + again, + ); + // Selecting a vehicle again reads its trail again. + original = reads(); + } } assert.deepEqual(errors, []); await fx.dispose?.(); console.log( 'PASS viewport activation, real pick and original city data across two switches', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-w2-imagery.mjs b/scripts/qa-w2-imagery.mjs index 50f0ca2c1..3c7015e8e 100644 --- a/scripts/qa-w2-imagery.mjs +++ b/scripts/qa-w2-imagery.mjs @@ -1,9 +1,6 @@ -import {switchGlobe} from './qa-switch-globe.mjs'; import fs from 'node:fs/promises'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; -import { installTileCache } from './qa/tileCache.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { isExpectedPreviewSetupError, redact } from './qa-browserEvidence.mjs'; const args = process.argv.slice(2); @@ -26,8 +23,17 @@ const report = { }; let failLandsat = false, failedTiles = 0; +// The engine the page runs, for the strict assertion manifest. +let engine = null; +const strict = createStrict({ + origin: localPreview(url).origin, + layers: ['recent-imagery'], + // The scenario answers the NASA catalog and imagery tiles itself. + fixtures: ['*://*.earthdata.nasa.gov/*'], +}); const check = (name, value) => { report.checks.push({ name, pass: Boolean(value) }); + if (value && engine) strict.record('recent-imagery', engine, name); console.log(`${value ? 'PASS' : 'FAIL'} ${name}`); }; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -38,7 +44,7 @@ const colors = await Promise.all( .toBuffer(), ), ); -const browser = await puppeteer.launch({ +const browser = await strict.launch({ headless: true, args: [ '--no-sandbox', @@ -51,8 +57,6 @@ const browser = await puppeteer.launch({ try { const page = await browser.newPage(); await page.setViewport({ width: 1280, height: 800, deviceScaleFactor: 1 }); - await blockPaidQaMaps(page); - await installTileCache(page); page.on('pageerror', (e) => report.errors.push(redact(e.message))); page.on('console', (message) => { if (message.type() !== 'error') return; @@ -152,6 +156,7 @@ try { () => window.__gevRecentImagery && window.__godsEyeView, { timeout: 90000 }, ); + engine = await page.evaluate(() => window.__godsEyeView.engine.kind); await page.evaluate(async () => { const app = window.__godsEyeView; await app.styleManager.initialRestorePromise; @@ -231,20 +236,31 @@ try { window.qaImageryOwner = window.__gevRecentImagery.layer; return window.qaImageryOwner.diagnostics().pins; }); - const reads = () => report.requests.filter(r => r.host === 'cmr.earthdata.nasa.gov').length; + const reads = () => + report.requests.filter((r) => r.host === 'cmr.earthdata.nasa.gov').length; const expectedReads = reads(); const initial = await page.evaluate(() => window.__godsEyeView.engine.kind); for (const kind of [initial === 'navara' ? 'cesium' : 'navara', initial]) { - await switchGlobe(page, kind); + await strict.switchTo(page, kind); + engine = kind; const state = await page.evaluate(() => ({ same: window.__gevRecentImagery.layer === window.qaImageryOwner, ...window.qaImageryOwner.diagnostics(), })); - check(kind+' holds both original pins',state.same && state.mode === 'ab' && state.pins.a === first.a && state.pins.b === first.b); + check( + kind + ' holds both original pins', + state.same && + state.mode === 'ab' && + state.pins.a === first.a && + state.pins.b === first.b, + ); await sleep(1800); - const pixels=await capture('switch-'+kind); - check(kind+' renders both pinned dates',red(pixels.left) && green(pixels.right)); - check(kind+' reuses the STAC search', reads() === expectedReads); + const pixels = await capture('switch-' + kind); + check( + kind + ' renders both pinned dates', + red(pixels.left) && green(pixels.right), + ); + check(kind + ' reuses the STAC search', reads() === expectedReads); } } await page.evaluate(() => window.__gevRecentImagery.layer.swapSides()); @@ -452,4 +468,6 @@ try { await browser.close(); await fs.writeFile(output, JSON.stringify(report, null, 2)); } -if (report.checks.some((c) => !c.pass)) process.exitCode = 1; +const failed = report.checks.some((c) => !c.pass); +await strict.finish({ failed: failed ? 'imagery checks failed' : null }); +if (failed) process.exitCode = 1; diff --git a/scripts/qa-weather-engine-switch.mjs b/scripts/qa-weather-engine-switch.mjs index 3514c23a2..a6f9802a4 100644 --- a/scripts/qa-weather-engine-switch.mjs +++ b/scripts/qa-weather-engine-switch.mjs @@ -1,27 +1,23 @@ -import {fromDegrees} from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; +import { fromDegrees } from '../src/geo/cartesian3.js'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: [ + 'weather-radar', + 'weather-satellite', + 'weather-lightning', + 'weather-cyclones', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -32,27 +28,81 @@ try { const ids = ['weather-radar', 'weather-satellite', 'weather-lightning']; const images = new Map(), reads = new Map(); - let bulletin = '10', cycloneReads = 0; + let bulletin = '10', + cycloneReads = 0; function advisory() { const forecast = bulletin === '10'; const storm = { - id:'al012026', name:'QA storm', basin:'AL', classification:'TS', position:{longitude:-79.38, latitude:43.65}, - advisoryNumber:bulletin, issuedAt:times[1], positionAt:times[1], windKt:40, pressureHpa:1000, - movement:{directionDegrees:45,speedKt:10}, advisoryUrl:null,outlookUrl:null, - geometryStatus:forecast?'current':'pending', geometryAdvisoryNumber:forecast?bulletin:null, - track:forecast?{type:'LineString',coordinates:[[-79.38,43.65],[-79.31,43.68]]}:null, - cone:forecast?{type:'Polygon',coordinates:[[[-79.35,43.63],[-79.28,43.63],[-79.28,43.70],[-79.35,43.63]]]}:null, - forecastPoints:forecast?[{tauHours:24,position:{longitude:-79.31,latitude:43.68},windKt:50,gustKt:60}]:[], + id: 'al012026', + name: 'QA storm', + basin: 'AL', + classification: 'TS', + position: { longitude: -79.38, latitude: 43.65 }, + advisoryNumber: bulletin, + issuedAt: times[1], + positionAt: times[1], + windKt: 40, + pressureHpa: 1000, + movement: { directionDegrees: 45, speedKt: 10 }, + advisoryUrl: null, + outlookUrl: null, + geometryStatus: forecast ? 'current' : 'pending', + geometryAdvisoryNumber: forecast ? bulletin : null, + track: forecast + ? { + type: 'LineString', + coordinates: [ + [-79.38, 43.65], + [-79.31, 43.68], + ], + } + : null, + cone: forecast + ? { + type: 'Polygon', + coordinates: [ + [ + [-79.35, 43.63], + [-79.28, 43.63], + [-79.28, 43.7], + [-79.35, 43.63], + ], + ], + } + : null, + forecastPoints: forecast + ? [ + { + tauHours: 24, + position: { longitude: -79.31, latitude: 43.68 }, + windKt: 50, + gustKt: 60, + }, + ] + : [], + }; + return { + schemaVersion: 1, + source: 'QA NOAA', + attribution: 'NOAA fixture', + coverage: 'Atlantic', + fetchedAt: Date.parse(times[1]), + stale: false, + unavailable: false, + reason: null, + storms: [storm], }; - return {schemaVersion:1,source:'QA NOAA',attribution:'NOAA fixture',coverage:'Atlantic',fetchedAt:Date.parse(times[1]), - stale:false,unavailable:false,reason:null,storms:[storm]}; } await p.setRequestInterception(true); p.on('request', async (req) => { const url = new URL(req.url()); if (url.pathname === '/api/cyclones') { cycloneReads++; - return req.respond({status:200,contentType:'application/json',body:JSON.stringify(advisory())}); + return req.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(advisory()), + }); } if (!url.pathname.startsWith('/api/weather/')) return req.continue(); const product = url.searchParams.get('product'); @@ -121,7 +171,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -188,6 +238,7 @@ try { latest[2] > latest[0] + 50, 'lightning must paint cyan over radar', ); + await strict.passed(p, id, 'latest frame paints in stacking order', latest); console.log('latest', id, latest); } await p.evaluate((ids) => { @@ -200,7 +251,7 @@ try { const polls = [...reads.entries()]; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await waitTime(ids, times[0]); @@ -238,6 +289,12 @@ try { polls, 'A globe change must not refetch the observation manifest', ); + await strict.passed( + p, + ids, + 'same entry, shared history clock on the earlier frame, no manifest refetch, lightning above radar', + rgb, + ); console.log( 'PASS', kind, @@ -262,10 +319,11 @@ try { await p.evaluate(() => window.qaRestoreSurface()); await p.evaluate(async (ids) => { const manager = window.__godsEyeView.dataManager; - for (const id of ids) await manager.setEnabled(id, false, {origin:'user'}); - await manager.setEnabled('weather-cyclones', true, {origin:'user'}); + for (const id of ids) + await manager.setEnabled(id, false, { origin: 'user' }); + await manager.setEnabled('weather-cyclones', true, { origin: 'user' }); window.qaStorm = manager.layers.get('weather-cyclones').module; - },ids); + }, ids); const click = async (lon, lat, insideCone) => { const hit = await p.waitForFunction( ({ lon, lat, base, insideCone }) => { @@ -303,30 +361,56 @@ try { await hit.dispose(); }; const before = cycloneReads; - for (const kind of ['navara','cesium','navara']) { - if (await p.evaluate(()=>window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine',kind); + for (const kind of ['navara', 'cesium', 'navara']) { + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await p.select('#globe-engine', kind); await ready(kind); } - await p.waitForFunction(() => window.__godsEyeView.dataManager.layers.get('weather-cyclones').module === window.qaStorm && - window.qaStorm.getDiagnostics().selectedId === 'al012026'); - const d = await p.evaluate(()=>window.qaStorm.getDiagnostics()); - assert.deepEqual([d.storms,d.tracks,d.cones,d.forecastPoints],[1,1,1,1],kind); - await click(-79.43,43.59,false); - await p.waitForFunction(()=>window.qaStorm.getDiagnostics().selectedId === null); - await click(-79.30,43.643,true); - await p.waitForFunction(()=>window.qaStorm.getDiagnostics().selectedId === 'al012026'); - assert.equal(cycloneReads,before); - console.log('PASS',kind,'pick advisory cone and retain selected storm'); + await p.waitForFunction( + () => + window.__godsEyeView.dataManager.layers.get('weather-cyclones') + .module === window.qaStorm && + window.qaStorm.getDiagnostics().selectedId === 'al012026', + ); + const d = await p.evaluate(() => window.qaStorm.getDiagnostics()); + assert.deepEqual( + [d.storms, d.tracks, d.cones, d.forecastPoints], + [1, 1, 1, 1], + kind, + ); + await click(-79.43, 43.59, false); + await p.waitForFunction( + () => window.qaStorm.getDiagnostics().selectedId === null, + ); + await click(-79.3, 43.643, true); + await p.waitForFunction( + () => window.qaStorm.getDiagnostics().selectedId === 'al012026', + ); + assert.equal(cycloneReads, before); + await strict.passed( + p, + 'weather-cyclones', + 'storm, track, cone and forecast point; clicks outside and inside the cone deselect and select; no refetch', + ); + console.log('PASS', kind, 'pick advisory cone and retain selected storm'); } bulletin = '11'; const next = await p.evaluate(async () => { await window.qaStorm.update(); return window.qaStorm.getDiagnostics(); }); - assert.deepEqual([next.storms,next.tracks,next.cones,next.forecastPoints,next.selectedId],[1,0,0,0,'al012026']); - console.log('PASS advisory 11 removes expired geometry while retaining the storm'); + assert.deepEqual( + [ + next.storms, + next.tracks, + next.cones, + next.forecastPoints, + next.selectedId, + ], + [1, 0, 0, 0, 'al012026'], + ); + console.log( + 'PASS advisory 11 removes expired geometry while retaining the storm', + ); assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-wind-engine-switch.mjs b/scripts/qa-wind-engine-switch.mjs index 7a2eb724d..0c31e1bcc 100644 --- a/scripts/qa-wind-engine-switch.mjs +++ b/scripts/qa-wind-engine-switch.mjs @@ -1,26 +1,16 @@ -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=2000000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +// The fixture field is uniform, so the GPU seeds a fixed particle count. +const WIND_PATHS = 7200; +const strict = createStrict({ origin: base.origin, layers: ['wind'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -83,7 +73,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -113,7 +103,7 @@ try { const baseline = reads.length; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await p.waitForFunction( @@ -133,11 +123,27 @@ try { reading: mod.getRowControls().summary.reading, }; }); - assert.equal(check.same, true); - assert.equal(check.params.paused, true); - assert.ok(check.gpu > 1000); - assert.equal(check.reading.scalarValue, '40.0 °C'); - assert.equal(check.reading.from, 'SW'); + await strict.check(p, 'wind', 'same wind model', check.same === true); + await strict.check( + p, + 'wind', + 'paused parameter retained', + check.params.paused === true, + ); + await strict.check( + p, + 'wind', + `${WIND_PATHS} GPU paths`, + check.gpu === WIND_PATHS, + check.gpu, + ); + await strict.check( + p, + 'wind', + 'point reading 40.0 °C from SW', + check.reading.scalarValue === '40.0 °C' && check.reading.from === 'SW', + check.reading, + ); const sample = await sharp(await p.screenshot()) .extract({ left: 530, top: 390, width: 24, height: 24 }) .removeAlpha() @@ -149,22 +155,40 @@ try { return sum / (sample.length / 3); }); console.log(kind, check.gpu, check.reading.wind, [r, g, b]); - assert.ok( + await strict.check( + p, + 'wind', + 'temperature field visible', r > g * 1.35 && r > b * 1.35, - kind + ' temperature field visible', + [r, g, b], + ); + await strict.check( + p, + 'wind', + 'no forecast grid read on the switch', + reads.length === baseline, + reads.length - baseline, ); - assert.equal(reads.length, baseline); } await p.evaluate(() => window.qaWind.setParams({ paused: false })); - await p.waitForFunction(() => window.qaWind.getDiagnostics().gpu.phaseSeconds > 0.2); + await p.waitForFunction( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds > 0.2, + ); await p.evaluate(() => window.qaWind.setParams({ paused: true })); - const frozen = await p.evaluate(() => window.qaWind.getDiagnostics().gpu.phaseSeconds); + const frozen = await p.evaluate( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds, + ); await new Promise((resolve) => setTimeout(resolve, 250)); - assert.equal(await p.evaluate(() => window.qaWind.getDiagnostics().gpu.phaseSeconds), frozen); + await strict.check( + p, + 'wind', + 'pause freezes the particle phase', + (await p.evaluate( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds, + )) === frozen, + ); assert.deepEqual(errors, []); console.log( 'PASS retained forecast grid, pause, temperature field and sampled wind on both engines', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa/tileCache.mjs b/scripts/qa/tileCache.mjs index ec1aba3ff..86070ade0 100644 --- a/scripts/qa/tileCache.mjs +++ b/scripts/qa/tileCache.mjs @@ -27,7 +27,7 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); * The cache entry for a GET of `url` with `accept` (terrain answers by its * Accept header), or null for a host that is not cached. */ -export function tileCacheFile(url, accept, dir) { +export function tileCacheFile(url, accept, dir, hosts = CACHED_TILE_HOSTS) { let host; try { const parsed = new URL(url); @@ -36,7 +36,7 @@ export function tileCacheFile(url, accept, dir) { } catch { return null; } - if (!CACHED_TILE_HOSTS.includes(host)) return null; + if (!hosts.includes(host)) return null; const key = createHash('sha256') .update(`${url}\n${accept ?? ''}`) .digest('hex') @@ -121,12 +121,15 @@ export function createMissLoader({ } /** - * Serves the page's requests to CACHED_TILE_HOSTS from the cache (filling - * it on a miss). Returns the counts. + * Serves the page's requests to CACHED_TILE_HOSTS (or `hosts`) from the + * cache (filling it on a miss). Returns the counts. */ export async function installTileCache( page, - { dir = path.join(os.tmpdir(), 'gev-qa-tiles') } = {}, + { + dir = path.join(os.tmpdir(), 'gev-qa-tiles'), + hosts = CACHED_TILE_HOSTS, + } = {}, ) { fs.mkdirSync(dir, { recursive: true }); const stats = { hits: 0, fetched: 0, cancelled: 0, failed: 0, errors: {} }; @@ -137,7 +140,9 @@ export async function installTileCache( ([name]) => name.toLowerCase() === 'accept', )?.[1]; const file = - request.method === 'GET' ? tileCacheFile(request.url, accept, dir) : null; + request.method === 'GET' + ? tileCacheFile(request.url, accept, dir, hosts) + : null; try { if (!file) return await client.send('Fetch.continueRequest', { requestId }); @@ -187,7 +192,7 @@ export async function installTileCache( } }); await client.send('Fetch.enable', { - patterns: CACHED_TILE_HOSTS.map((host) => ({ + patterns: hosts.map((host) => ({ urlPattern: `https://${host}/*`, })), });