diff --git a/scripts/main-chunk-baseline.json b/scripts/main-chunk-baseline.json new file mode 100644 index 000000000..d584895c7 --- /dev/null +++ b/scripts/main-chunk-baseline.json @@ -0,0 +1,6 @@ +{ + "gzipBytes": 1920969, + "allowanceBytes": 19210, + "measuredAt": "a5d78b00, keyless build (GOOGLE_MAPS_API_KEY and CESIUM_ION_TOKEN empty), entry index-CV-avk6m.js, 5 933 796 bytes, gzip level 9", + "allowance": "1% of the baseline; growth past it fails npm run check:main-chunk until the baseline is raised on purpose" +} diff --git a/scripts/qa-all-layer-switches.mjs b/scripts/qa-all-layer-switches.mjs index e9c32683f..8b4cafa12 100644 --- a/scripts/qa-all-layer-switches.mjs +++ b/scripts/qa-all-layer-switches.mjs @@ -1,28 +1,65 @@ -import { execFileSync, spawnSync } from 'node:child_process'; +import { execFileSync, spawn, spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import assert from 'node:assert/strict'; import puppeteer from 'puppeteer'; import { APPLICATION_LAYER_DIRECTORY } from '../src/app/layers/directory.js'; +import { + createStrict, + ENGINES, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; -// Runs every browser scenario against one keyless local preview and writes -// /acceptance.json plus one log per scenario. +// The strict acceptance: every browser scenario against one keyless local +// preview, under the strict network policy (scripts/qa-strict-harness.mjs), +// writing /acceptance.json, one log and one assertion manifest per +// scenario. +// +// node scripts/qa-all-layer-switches.mjs http://127.0.0.1:4208 +// [--serve[=]] build (default HEAD) keyless in a clean +// worktree and serve it with every .env key blank +// [--scratch=DIR] where --serve puts its checkout and build +// [--only=a,b] run only these labels +// [--quick] the soak with 2 runs of 10 switches and 100 +// switches for the long run +// +// The run fails when a scenario fails, when any catalog layer (read from +// the live page and cross-checked with src/app/layers/directory.js) has no +// assertion on either engine, or when any network policy violation shows. -const base = new URL(process.argv[2]); -if ( - !['127.0.0.1', 'localhost'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw Error('Supply an unreserved keyless local preview URL'); -const output = process.argv[3] - ? resolve(process.argv[3]) - : mkdtempSync(join(tmpdir(), 'gev-layer-switch-')); -mkdirSync(output, { recursive: true }); +const base = localPreview(process.argv[2]); +const output = process.argv[3]?.startsWith('--') + ? mkdtempSync(join(tmpdir(), 'gev-acceptance-')) + : resolve(process.argv[3] ?? mkdtempSync(join(tmpdir(), 'gev-acceptance-'))); +const flag = (name) => + process.argv + .find((arg) => arg === `--${name}` || arg.startsWith(`--${name}=`)) + ?.split('=')[1] ?? (process.argv.includes(`--${name}`) ? true : null); +const only = flag('only') ? String(flag('only')).split(',') : null; +const quick = Boolean(flag('quick')); +mkdirSync(join(output, 'evidence'), { recursive: true }); +const root = fileURLToPath(new URL('..', import.meta.url)); +const git = (...args) => + execFileSync('git', args, { cwd: root, encoding: 'utf8' }); +const dirty = () => + git('status', '--porcelain') + .split('\n') + .filter(Boolean) + .map((line) => line.slice(3)); // Fail once, with the fix, instead of once per scenario. let chrome; @@ -42,6 +79,136 @@ try { ); process.exit(2); } + +// Every name the checkout's .env files define, plus key-like names in this +// environment: all blank for the build and the preview. +const envNames = () => { + const names = new Set(); + for (const file of ['.env', '.env.local']) + if (existsSync(join(root, file))) + for (const line of readFileSync(join(root, file), 'utf8').split('\n')) { + const name = line.match(/^\s*([A-Z0-9_]+)\s*=/)?.[1]; + if (name) names.add(name); + } + for (const name of Object.keys(process.env)) + if (/(_KEY|_TOKEN|_SECRET|_CLIENT_ID|PASSWORD)$/.test(name)) + names.add(name); + return [...names].sort(); +}; +const envValues = () => { + const values = []; + for (const file of ['.env', '.env.local']) + if (existsSync(join(root, file))) + for (const line of readFileSync(join(root, file), 'utf8').split('\n')) { + const value = line + .match(/^\s*[A-Z0-9_]+\s*=\s*(.*)$/)?.[1] + ?.trim() + .replace(/^["']|["']$/g, ''); + // Long enough to be a credential rather than a setting. + if (value && value.length >= 16) values.push(value); + } + return values; +}; +const scrubbed = () => ({ + ...process.env, + ...Object.fromEntries(envNames().map((name) => [name, ''])), +}); + +let served = null; +let preview = null; +const cleanup = () => { + if (preview && !preview.killed) preview.kill('SIGTERM'); + if (served?.worktree) + try { + git('worktree', 'remove', '--force', served.worktree); + } catch { + // Already gone. + } +}; +process.on('exit', cleanup); +process.on('SIGINT', () => process.exit(130)); + +if (flag('serve')) { + const commit = git( + 'rev-parse', + flag('serve') === true ? 'HEAD' : String(flag('serve')), + ).trim(); + // --scratch=DIR keeps the checkout and build out of the system temp. + const scratchRoot = flag('scratch') + ? resolve(String(flag('scratch'))) + : tmpdir(); + mkdirSync(scratchRoot, { recursive: true }); + const scratch = mkdtempSync(join(scratchRoot, 'gev-acceptance-build-')); + const worktree = join(scratch, 'checkout'); + const dist = join(scratch, 'dist'); + git('worktree', 'add', '--detach', worktree, commit); + symlinkSync(join(root, 'node_modules'), join(worktree, 'node_modules')); + served = { + commit, + worktree, + dist, + blankedEnv: envNames(), + envFilesInCheckout: ['.env', '.env.local'].filter((f) => + existsSync(join(worktree, f)), + ), + }; + console.log('Building', commit, 'keyless into', dist); + const build = spawnSync( + process.execPath, + [join(root, 'node_modules/vite/bin/vite.js'), 'build', '--outDir', dist], + { cwd: worktree, env: scrubbed(), encoding: 'utf8', maxBuffer: 1 << 26 }, + ); + if (build.status !== 0) { + console.error(build.stdout + build.stderr); + process.exit(2); + } + // No .env credential value may reach the built files. + const values = envValues(); + let leaks = 0; + const walk = (dir) => { + for (const name of readdirSync(dir)) { + const file = join(dir, name); + if (statSync(file).isDirectory()) walk(file); + else if (/\.(js|html|json|css|webmanifest)$/.test(name)) { + const text = readFileSync(file, 'utf8'); + for (const value of values) if (text.includes(value)) leaks++; + } + } + }; + walk(dist); + served.distCredentialValuesFound = leaks; + served.mainChunk = JSON.parse( + spawnSync( + process.execPath, + [join(root, 'scripts/check-main-chunk.mjs'), `--dist=${dist}`, '--json'], + { cwd: root, encoding: 'utf8' }, + ).stdout || 'null', + ); + preview = spawn( + process.execPath, + [ + join(root, 'node_modules/vite/bin/vite.js'), + 'preview', + '--outDir', + dist, + '--host', + base.hostname, + '--port', + base.port, + '--strictPort', + ], + { cwd: worktree, env: scrubbed(), stdio: 'ignore' }, + ); + for (let i = 0; i < 60; i++) { + try { + if ((await fetch(base.origin + '/')).ok) break; + } catch { + // Not up yet. + } + await new Promise((r) => setTimeout(r, 500)); + } +} + try { const response = await fetch(base.origin + '/', { signal: AbortSignal.timeout(10000), @@ -53,29 +220,56 @@ try { ); process.exit(2); } -const root = fileURLToPath(new URL('..', import.meta.url)); -const git = (...args) => - execFileSync('git', args, { cwd: root, encoding: 'utf8' }); -// Other work may change the tree while the run lasts; record both ends. -const dirty = () => - git('status', '--porcelain') - .split('\n') - .filter(Boolean) - .map((line) => line.slice(3)); + +// The preview reports no paid key: its own keyless status routes. +const probe = async (path) => { + try { + const r = await fetch(base.origin + path, { + headers: { 'Sec-Fetch-Site': 'same-origin' }, + }); + return { status: r.status, body: await r.json().catch(() => null) }; + } catch (error) { + return { error: error.message }; + } +}; +const keys = { + openaiHudSummary: await probe('/api/openai/hud-summary/status'), + firms: await probe('/api/firms/status'), + tomtom: await probe('/api/tomtom/status'), + mapillary: await probe('/api/mapillary/status'), +}; +const paidKeysConfigured = [ + keys.openaiHudSummary.body?.configured !== false && 'OpenAI', + keys.firms.body?.hasKey !== false && 'FIRMS', + keys.tomtom.body?.hasKey !== false && 'TomTom', + keys.mapillary.body?.configured !== false && 'Mapillary', +].filter(Boolean); + const dirtyFiles = dirty(); +const soakRuns = quick + ? ['--runs=2', '--swaps=10'] + : ['--runs=6', '--swaps=50']; +const longRun = quick ? '--swaps=100' : '--swaps=500'; +// label, script, layers, extra arguments, timeout in seconds. const runs = [ - ['engine-switch', ['earthquakes']], - ['director-engine-switch', []], - ['layer-handoff', ['flights']], - ['layer-handoff', ['military'], '--military'], + ['engine-switch', 'engine-switch', ['earthquakes']], + ['director-engine-switch', 'director-engine-switch', []], + ['layer-handoff', 'layer-handoff', ['flights']], + ['layer-handoff--military', 'layer-handoff', ['military'], ['--military']], [ + 'ground-layer-switch', 'ground-layer-switch', ['ais-live-vessels', 'air-quality', 'gnss-integrity', 'fire-perimeters'], ], - ['contact-switch', ['military-awareness', 'military-installations']], - ['local-adsb-engine-switch', ['local-adsb']], - ['ghostmaps-switch', ['s2-ghostmaps']], [ + 'contact-switch', + 'contact-switch', + ['military-awareness', 'military-installations'], + ], + ['local-adsb-engine-switch', 'local-adsb-engine-switch', ['local-adsb']], + ['ghostmaps-switch', 'ghostmaps-switch', ['s2-ghostmaps']], + [ + 'weather-engine-switch', 'weather-engine-switch', [ 'weather-radar', @@ -84,32 +278,113 @@ const runs = [ 'weather-cyclones', ], ], - ['wind-engine-switch', ['wind']], - ['infrastructure-engine-switch', ['local-dams', 'local-datacenters']], - ['submarine-cables-engine-switch', ['telegeography-submarine-cables']], - ['directions-engine-switch', ['directions']], - ['nepal-locator-switch', ['bhote-koshi-locator']], - ['nepal-event-switch', ['bhote-koshi-2026']], - ['traffic-engine-switch', ['traffic']], - ['traffic-engine-switch', ['traffic'], '--simulation'], - ['street-level-engine-switch', ['street-level']], - ['street-level-engine-switch', ['street-level'], '--panoramax'], - ['radio-engine-switch', ['radio']], - ['w2-imagery', ['recent-imagery']], - ['fire-camera-switch', ['alpr-cameras']], - ['fire-camera-switch', ['local-firms'], '--fires'], - ['urban-engine-switch', ['bikeshare']], - ['urban-engine-switch', ['transit'], '--transit'], - ['cctv-engine-switch', ['cctv']], - ['space-engine-switch', ['satellites']], - ['space-engine-switch', ['rocket-launches'], '--missions'], + ['wind-engine-switch', 'wind-engine-switch', ['wind']], + [ + 'infrastructure-engine-switch', + 'infrastructure-engine-switch', + ['local-dams', 'local-datacenters'], + ], + [ + 'submarine-cables-engine-switch', + 'submarine-cables-engine-switch', + ['telegeography-submarine-cables'], + ], + ['directions-engine-switch', 'directions-engine-switch', ['directions']], + ['nepal-locator-switch', 'nepal-locator-switch', ['bhote-koshi-locator']], + ['nepal-event-switch', 'nepal-event-switch', ['bhote-koshi-2026']], + ['traffic-engine-switch', 'traffic-engine-switch', ['traffic']], + [ + 'traffic-engine-switch--simulation', + 'traffic-engine-switch', + ['traffic'], + ['--simulation'], + ], + [ + 'street-level-engine-switch', + 'street-level-engine-switch', + ['street-level'], + ], + [ + 'street-level-engine-switch--panoramax', + 'street-level-engine-switch', + ['street-level'], + ['--panoramax'], + ], + ['radio-engine-switch', 'radio-engine-switch', ['radio']], + ['w2-imagery', 'w2-imagery', ['recent-imagery']], + ['fire-camera-switch', 'fire-camera-switch', ['alpr-cameras']], + [ + 'fire-camera-switch--fires', + 'fire-camera-switch', + ['local-firms'], + ['--fires'], + ], + ['urban-engine-switch', 'urban-engine-switch', ['bikeshare']], + [ + 'urban-engine-switch--transit', + 'urban-engine-switch', + ['transit'], + ['--transit'], + ], + ['cctv-engine-switch', 'cctv-engine-switch', ['cctv']], + ['space-engine-switch', 'space-engine-switch', ['satellites']], + [ + 'space-engine-switch--missions', + 'space-engine-switch', + ['rocket-launches'], + ['--missions'], + ], + // Stage 3's own checks (one application per page, voice across a switch). + // They block paid routes themselves but do not run under the strict + // harness, so they add no assertion manifest and no network ledger. + ['app-once', 'app-once', [], [], 600, true], + ['voice-engine-switch', 'voice-engine-switch', [], [], 300, true], + ['strict-rapid-switch', 'strict-rapid-switch', []], + ['strict-hidden-switch', 'strict-hidden-switch', []], + ['strict-context-loss', 'strict-context-loss', []], + ['strict-failure-matrix', 'strict-failure-matrix', [], [], 600], + ['strict-cold-chunks', 'strict-cold-chunks', []], + ['strict-soak', 'strict-soak', [], soakRuns, 1200], + [ + 'strict-soak--long', + 'strict-soak', + [], + ['--runs=1', longRun, '--assert'], + 1800, + ], ]; -const ids = [...new Set(runs.flatMap(([, ids]) => ids))].sort(); -assert.deepEqual( - ids, - APPLICATION_LAYER_DIRECTORY.map((l) => l.id).sort(), - 'Every catalog layer must have a browser scenario', -); +const scenarioIds = [...new Set(runs.flatMap(([, , ids]) => ids))].sort(); +const directoryIds = APPLICATION_LAYER_DIRECTORY.map((l) => l.id).sort(); + +// The catalog as the live page has it, on both engines. +const liveCatalog = {}; +{ + const strict = createStrict({ + origin: base.origin, + layers: [], + name: 'catalog', + }); + const browser = await strict.launch({ + args: ['--use-angle=metal', '--enable-gpu'], + }); + try { + for (const engine of ENGINES) { + const page = await browser.newPage(); + await page.goto( + `${base.origin}/?engine=${engine}&welcome=0#v=2&lat=30.27&lon=-97.74&alt=12000&pitch=-90&map=osm&l=`, + ); + await waitForEngine(page, engine); + liveCatalog[engine] = await page.evaluate(() => + [...window.__godsEyeView.dataManager.layers.keys()].sort(), + ); + await page.close(); + } + } finally { + await browser.close(); + } + liveCatalog.violations = strict.ledger.violations; +} + const scenarios = []; const report = { commit: git('rev-parse', 'HEAD').trim(), @@ -121,11 +396,41 @@ const report = { node: process.version, platform: process.platform + ' ' + process.arch, baseUrl: base.origin, + preview: served + ? { + servedByRunner: true, + appCommit: served.commit, + appCheckoutClean: true, + envFilesInCheckout: served.envFilesInCheckout, + blankedEnv: served.blankedEnv, + distCredentialValuesFound: served.distCredentialValuesFound, + } + : { servedByRunner: false }, + keyProbes: keys, + paidKeysConfigured, + mainChunk: served?.mainChunk ?? null, + networkPolicy: { + blockedFirst: + 'Google (*.googleapis.com, *.google.com, maps.google.*, *.gstatic.com), Cesium ion (*.cesium.com), preview /api/openai/*, /api/realtime/*, /api/google/*', + allowed: + 'the preview origin, each scenario’s declared fixture URLs, keyless probes answered by the harness, and the free tile hosts answered from the disk cache', + keylessProbes: (await import('./qa-map-policy.mjs')).KEYLESS_PROBES.map( + (p) => p.pattern, + ), + tileHosts: (await import('./qa-map-policy.mjs')).STRICT_TILE_HOSTS, + }, + catalog: { + directory: directoryIds, + live: liveCatalog, + scenarios: scenarioIds, + }, startedAt: new Date().toISOString(), finishedAt: null, - layers: ids, passed: 0, failed: 0, + networkViolations: 0, + coverage: null, + uncovered: null, scenarios, }; const save = () => @@ -133,26 +438,32 @@ const save = () => join(output, 'acceptance.json'), JSON.stringify(report, null, 2) + '\n', ); -for (const [name, layers, flag] of runs) { - const label = name + (flag ?? ''); +save(); + +for (const [label, name, layers, extra = [], seconds = 300, external] of runs) { + if (only && !only.includes(label)) continue; const script = 'scripts/qa-' + name + '.mjs'; + const evidence = join(output, 'evidence', label + '.json'); + rmSync(evidence, { force: true }); const args = name === 'w2-imagery' ? [ '--url', base.origin + '/?engine=navara', '--report', - join(output, 'imagery.json'), + join(output, 'evidence', 'imagery-report.json'), '--switch', '--preview', ] - : [base.origin, ...(flag ? [flag] : [])]; + : [base.origin, ...extra]; + if (external) args.splice(0, 1, base.origin + '/'); + else args.push(`--layers=${layers.join(',')}`, `--evidence=${evidence}`); const start = Date.now(); const run = spawnSync(process.execPath, [join(root, script), ...args], { encoding: 'utf8', - timeout: 240000, - maxBuffer: 8388608, - env: { ...process.env, GOOGLE_MAPS_API_KEY: '', CESIUM_ION_TOKEN: '' }, + timeout: seconds * 1000, + maxBuffer: 1 << 26, + env: scrubbed(), }); const status = run.status === 0 @@ -163,34 +474,94 @@ for (const [name, layers, flag] of runs) { const log = (run.stdout || '') + (run.stderr || '') + (run.error?.message || ''); writeFileSync(join(output, label + '.log'), log); + const manifest = existsSync(evidence) + ? JSON.parse(readFileSync(evidence, 'utf8')) + : null; scenarios.push({ label, script, args: args.map((arg) => - arg.startsWith(output) ? '' + arg.slice(output.length) : arg, + arg.includes(output) ? arg.replace(output, '') : arg, ), layers, + external: Boolean(external), status, exitCode: run.status, seconds: Math.round((Date.now() - start) / 100) / 10, log: label + '.log', logSha256: createHash('sha256').update(log).digest('hex'), + evidence: manifest ? 'evidence/' + label + '.json' : null, + assertions: manifest?.assertions.length ?? 0, + coverage: manifest?.coverage ?? null, + picks: manifest?.picks ?? [], + network: manifest + ? { + requests: manifest.network.requests, + violations: manifest.network.violations, + external: manifest.network.external, + tileCache: manifest.network.tileCache, + probes: manifest.network.probes, + routes: manifest.network.routes, + previewApi: manifest.network.previewApi, + fixtures: manifest.network.fixtures, + } + : null, + contracts: manifest?.contracts ?? null, + error: manifest?.error ?? null, }); + if (manifest?.soak) scenarios.at(-1).soak = manifest.soak; report.passed = scenarios.filter((row) => row.status === 'pass').length; report.failed = scenarios.length - report.passed; + report.networkViolations = + scenarios.reduce( + (sum, row) => sum + (row.network?.violations.length ?? 0), + 0, + ) + liveCatalog.violations.length; save(); console.log(status.toUpperCase(), label, scenarios.at(-1).seconds); } + +// Coverage that cannot be faked: every catalog id needs an assertion on each +// engine from some scenario's manifest. +const catalogIds = [ + ...new Set([...directoryIds, ...(liveCatalog.navara ?? [])]), +].sort(); +report.coverage = Object.fromEntries( + catalogIds.map((id) => [ + id, + Object.fromEntries( + ENGINES.map((engine) => [ + engine, + scenarios.reduce( + (sum, row) => sum + (row.coverage?.[id]?.[engine] ?? 0), + 0, + ), + ]), + ), + ]), +); +report.uncovered = Object.entries(report.coverage).flatMap(([id, counts]) => + ENGINES.filter((engine) => !counts[engine]).map( + (engine) => `${id}@${engine}`, + ), +); +report.catalogMatches = + JSON.stringify(liveCatalog.navara) === JSON.stringify(directoryIds) && + JSON.stringify(liveCatalog.cesium) === JSON.stringify(directoryIds) && + JSON.stringify(scenarioIds) === JSON.stringify(directoryIds); report.finishedAt = new Date().toISOString(); report.dirtyFilesAtEnd = dirty(); save(); console.log( - ids.length + - ' layers, ' + - report.passed + - '/' + - runs.length + - ' browser scenarios. ' + - join(output, 'acceptance.json'), + `${catalogIds.length} catalog layers, ${report.uncovered.length} engine slots without an assertion, ` + + `${report.networkViolations} network violations, ${report.passed}/${scenarios.length} scenarios, ` + + `paid keys configured: ${paidKeysConfigured.join(', ') || 'none'}. ${join(output, 'acceptance.json')}`, ); -if (report.failed) process.exitCode = 1; +if ( + report.failed || + (!only && report.uncovered.length) || + report.networkViolations || + paidKeysConfigured.length || + !report.catalogMatches +) + process.exitCode = 1; diff --git a/scripts/qa-cctv-engine-switch.mjs b/scripts/qa-cctv-engine-switch.mjs index 08bc07b9a..8608de968 100644 --- a/scripts/qa-cctv-engine-switch.mjs +++ b/scripts/qa-cctv-engine-switch.mjs @@ -1,18 +1,9 @@ import assert from 'node:assert/strict'; import { promisify } from 'node:util'; import { execFile } from 'node:child_process'; -import puppeteer from 'puppeteer'; import { installCctvFixture } from './fixtures/w2/cctv.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.2718&lon=-97.744&alt=900&pitch=-70&heading=0&map=sentinel2&l='; @@ -45,13 +36,9 @@ const { stdout: video } = await promisify(execFile)( ], { encoding: 'buffer', maxBuffer: 16000000 }, ); -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: url.origin, layers: ['cctv'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); const errors = []; p.on('pageerror', (e) => { errors.push(e.message); @@ -65,7 +52,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async () => { @@ -95,7 +82,7 @@ try { let previous = first; const count = catalogs; for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); await p.waitForFunction( (n) => window.qaCctv.getActiveVideoElement()?.getVideoPlaybackQuality() @@ -114,6 +101,16 @@ try { plane: !!window.qaCctv ._cctvRecordsForTest() .find((r) => r.camera.id === 'w2-cctv-video').projection?.planeEntity, + projection: (() => { + const r = window.qaCctv + ._cctvRecordsForTest() + .find((r) => r.camera.id === 'w2-cctv-video'); + return { + show: r.projection?.planeEntity?.show === true, + visible: r.projection?.planeVisible === true, + calibration: r.camera.calibration, + }; + })(), })); console.log( kind, @@ -128,10 +125,28 @@ try { assert.equal(v.params.selectedCameraId, 'w2-cctv-video'); assert.equal(v.params.coverageMode, 'viewshed'); assert.equal(catalogs, count); + await strict.passed( + p, + 'cctv', + 'same model and video element decoding new frames; textured projection plane; no catalog read', + ); + // The calibration patch (31 m mount, 20° pitch) and the plane's + // visibility are kept on the new engine. + const { calibration } = v.projection; + await strict.check( + p, + 'cctv', + 'calibration patch kept (31 m, 20° pitch); monitor plane visible', + v.projection.show && + v.projection.visible && + calibration?.heightM === 31 && + calibration?.pitchDeg === 20 && + v.params.calibration?.values?.heightM === 31 && + v.params.calibration?.values?.pitchDeg === 20, + v.projection, + ); previous = v.frames; } assert.deepEqual(errors, []); console.log('PASS retained CCTV catalog, decoder and textured projection'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-contact-switch.mjs b/scripts/qa-contact-switch.mjs index 2226ee420..16b40a76e 100644 --- a/scripts/qa-contact-switch.mjs +++ b/scripts/qa-contact-switch.mjs @@ -1,25 +1,17 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { fromDegrees } from '../src/geo/cartesian3.js'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l=4'; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: ['military-awareness', 'military-installations'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -43,6 +35,14 @@ try { })), }), ); + // Contacts reads air quality near the subject; no station here. + if ( + new URL(input.url ?? input, document.baseURI).hostname === + 'api.weather.gc.ca' + ) + return Promise.resolve( + Response.json({ type: 'FeatureCollection', features: [] }), + ); if (pathname === '/api/vessels/track') return Promise.resolve(Response.json({ samples: [] })); if (pathname === '/api/flights') @@ -51,16 +51,48 @@ try { ); if (pathname === '/api/military') return Promise.resolve(Response.json({ now: Date.now(), ac: [] })); - if (pathname === '/api/military-installations') - return Promise.resolve(Response.json({ elements: [], status: 'ok' })); + // One mapped station next to the Contacts subject, with a footprint. + if (pathname === '/api/military-installations') { + window.qaInstallationReads = (window.qaInstallationReads ?? 0) + 1; + const ring = [ + [43.665, -79.345], + [43.665, -79.315], + [43.68, -79.315], + [43.68, -79.345], + [43.665, -79.345], + ]; + return Promise.resolve( + Response.json({ + status: 'ok', + elements: [ + { + type: 'way', + id: 424242, + tags: { military: 'naval_base', name: 'QA Harbour Station' }, + center: { lat: 43.6725, lon: -79.33 }, + geometry: ring.map(([lat, lon]) => ({ lat, lon })), + }, + ], + }), + ); + } return fetch(input, options); }; + window.qaSelected = []; + for (const type of ['gev:entity-selected', 'gev:entity-selection-cleared']) + window.addEventListener(type, ({ detail }) => + window.qaSelected.push({ + type, + id: detail?.id, + layerId: detail?.layerId, + }), + ); }); await p.goto(base.href); const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -110,14 +142,100 @@ try { : null, }; }); + // The station zone: the fixture's installation is the one mapped site + // within the Contacts radius of the subject, and the engine draws it where + // a pointer can reach it (the engine's own picker returns it at an + // uncovered canvas point at its marker or inside its footprint). + const inside = [fromDegrees(-79.33, 43.6725)]; + for (let i = 1; i < 6; i++) + for (let k = 1; k < 6; k++) + inside.push(fromDegrees(-79.345 + 0.005 * k, 43.665 + 0.0025 * i)); + const zone = async () => { + await p.waitForFunction( + () => + !window.qaSites.getStats().loading && + window.qaSites + .getAnalystRecords() + .some((r) => r.name === 'QA Harbour Station'), + { timeout: 30000 }, + ); + const drawn = await p + .waitForFunction( + (points) => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (const world of points) { + const centre = engine.projector.worldToWindow(world, {}); + if (!centre) continue; + for (const [dx, dy] of [ + [0, 0], + [0, -8], + [8, 0], + [-8, 0], + [0, 8], + ]) { + const at = { x: centre.x + dx, y: centre.y + dy }; + if ( + !canvas.contains( + document.elementFromPoint(box.left + at.x, box.top + at.y), + ) + ) + continue; + const id = engine.picker.pick(at)?.id; + if (String(id?.id ?? id) === 'osm:way:424242') + return [Math.round(at.x), Math.round(at.y)]; + } + } + return false; + }, + { timeout: 20000 }, + inside, + ) + .then((handle) => handle.jsonValue()) + .catch(() => null); + return p.evaluate( + (drawn) => ({ + drawn, + records: window.qaSites.getAnalystRecords().map((r) => r.name), + coverage: window.qaSites.getStats().coverage, + count: window.qaSites.getStats().count, + reads: window.qaInstallationReads, + subject: window.qaContext.getContextSnapshot()?.subject?.id, + }), + drawn, + ); + }; + const checkZone = async () => { + const station = await zone(); + await strict.check( + p, + 'military-installations', + 'the station zone holds the fixture station, drawn and pickable', + station.count === 1 && + station.records.length === 1 && + station.records[0] === 'QA Harbour Station' && + station.coverage?.kind === 'subject' && + Array.isArray(station.drawn), + station, + ); + return station; + }; + const firstZone = await checkZone(); const baseline = await sample(); console.log('baseline', JSON.stringify(baseline)); assert.equal(baseline.mode, 'flights'); assert.equal(baseline.subject, '223456789'); assert.ok(baseline.history.includes('ais-live-vessels:123456789')); assert.ok(baseline.ring >= 1); + await strict.passed( + p, + 'military-awareness', + 'Contacts subject, navigation history and awareness rim', + baseline, + ); for (const kind of ['cesium', 'navara']) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); const current = await sample(); console.log('handoff', kind, JSON.stringify(current)); @@ -125,7 +243,26 @@ try { ...baseline, ring: kind === 'navara' ? baseline.ring : null, }); + await strict.passed( + p, + 'military-awareness', + 'same awareness model, Contacts mode, subject, history and enabled layers', + ); + await strict.passed( + p, + 'military-installations', + 'same installations model and enabled through the switch', + ); + const station = await checkZone(); + await strict.check( + p, + 'military-installations', + 'the switch reads no installation snapshot', + station.reads === firstZone.reads, + { before: firstZone.reads, after: station.reads }, + ); } + await checkZone(); const restored = await p.evaluate(async () => { const { dataManager, styleManager } = window.__godsEyeView; const result = await styleManager.setContextMode('off'); @@ -143,6 +280,4 @@ try { console.log( 'PASS Contacts history, dependency ownership and exit restore across both globes', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-directions-engine-switch.mjs b/scripts/qa-directions-engine-switch.mjs index 6fb95e802..f1ce4bdc5 100644 --- a/scripts/qa-directions-engine-switch.mjs +++ b/scripts/qa-directions-engine-switch.mjs @@ -1,25 +1,13 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=55.3241&lon=10.7995&alt=25000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['directions'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -70,7 +58,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -98,7 +86,12 @@ try { await new Promise((resolve) => setTimeout(resolve, 450)); } assert.equal(routeReads, 1); - await p.select('#globe-engine', 'cesium'); + await strict.passed( + p, + 'directions', + 'A and B set by clicks on the globe; one route request', + ); + await strict.pick(p, 'cesium'); await ready('cesium'); release(); await p.waitForFunction(() => window.qaRoute.getStats().count === 3); @@ -106,8 +99,13 @@ try { window.qaRoute.setParams({ step: 1 }); return window.qaRoute.getRowControls().list.items[1]?.active; }); + await strict.passed( + p, + 'directions', + 'the route asked for on Navara arrives on Cesium; a step is selectable', + ); const transfer = async (kind) => { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); assert.equal( await p.evaluate( @@ -118,6 +116,11 @@ try { true, ); assert.equal(routeReads, 1); + await strict.passed( + p, + 'directions', + 'same model; the switch requests no route', + ); }; await transfer('navara'); assert.equal( @@ -153,13 +156,51 @@ try { progress = resumed; console.log(kind, 'moved', d(at, next), 'handoff', d(at, before)); assert.ok(d(at, next) > 1, 'dolly advances after switch to ' + kind); + await strict.passed( + p, + 'directions', + 'FLY continues with the same motion and advances', + { moved: d(at, next) }, + ); } await p.evaluate(() => window.qaRoute.setParams({ clear: true })); assert.equal(await p.evaluate(() => window.qaRoute.getStats().count), 0); + // CLEAR releases the camera: the route's motion ends, the camera stays + // where it is, and a wheel zoom by the person moves it and is kept. + await new Promise((resolve) => setTimeout(resolve, 500)); + const released = await p.evaluate(() => ({ + motion: window.qaRoute.getFlightProgress(), + fly: window.qaRoute.getRowControls().chips.find((c) => c.id === 'fly') + ?.active, + })); + const still = await pos(); + await new Promise((resolve) => setTimeout(resolve, 1500)); + const later = await pos(); + const height = () => + p.evaluate(() => window.__godsEyeView.engine.camera.height()); + const beforeZoom = await height(); + await p.mouse.move(512, 300); + for (let i = 0; i < 4; i++) { + await p.mouse.wheel({ deltaY: -240 }); + await new Promise((resolve) => setTimeout(resolve, 120)); + } + await new Promise((resolve) => setTimeout(resolve, 1500)); + const zoomed = await height(); + await new Promise((resolve) => setTimeout(resolve, 1500)); + const kept = await height(); + await strict.check( + p, + 'directions', + 'CLEAR removes the route and releases the camera: no route motion, the camera stays put, a wheel zoom moves it and is kept', + released.motion === null && + !released.fly && + d(still, later) < 1 && + zoomed < beforeZoom * 0.95 && + Math.abs(kept - zoomed) < 0.02 * zoomed, + { released, drift: d(still, later), beforeZoom, zoomed, kept }, + ); assert.deepEqual(errors, []); console.log( 'PASS route delivery during handoff, A/B clicks, step selection, ongoing FLY, clear and single route request', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-director-engine-switch.mjs b/scripts/qa-director-engine-switch.mjs index ca74a18ff..947cd8e24 100644 --- a/scripts/qa-director-engine-switch.mjs +++ b/scripts/qa-director-engine-switch.mjs @@ -1,17 +1,8 @@ import assert from 'node:assert/strict'; import sharp from 'sharp'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { fromDegrees } from '../src/geo/cartesian3.js'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +const url = localPreview(process.argv[2]); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.2672&lon=-97.742&alt=4000&pitch=-90&heading=0&map=sentinel2&l='; @@ -137,13 +128,15 @@ const project = { }, ], }; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Scene Director is not a catalog layer; the scenario answers the scene's +// earthquake feed itself. +const strict = createStrict({ + origin: url.origin, + layers: [], + fixtures: ['https://earthquake.usgs.gov/*'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => { @@ -223,7 +216,7 @@ try { }, world); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) - await switchGlobe(p, kind); + await strict.switchTo(p, kind); await new Promise((r) => setTimeout(r, 500)); const st = await p.evaluate(() => ({ same: window.director === window.__godsEyeView.sceneDirector, @@ -290,7 +283,7 @@ try { window.director.running && window.director.getPlaybackTimingState().snapshot?.shotId === 'one', ); - await switchGlobe(p, 'cesium'); + await strict.switchTo(p, 'cesium'); assert.ok( await p.evaluate( () => @@ -302,13 +295,11 @@ try { () => window.director.getPlaybackTimingState().snapshot?.shotId === 'two', { timeout: 20000 }, ); - await switchGlobe(p, 'navara'); + await strict.switchTo(p, 'navara'); const complete = await p.evaluate(() => window.qaRun); assert.ok(complete.started && complete.shots === 2); assert.deepEqual(errors, []); console.log( 'PASS authored packs, geographic picks, same player, original multi-shot timeline', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-engine-switch.mjs b/scripts/qa-engine-switch.mjs index b1f886651..a69b05e69 100644 --- a/scripts/qa-engine-switch.mjs +++ b/scripts/qa-engine-switch.mjs @@ -1,459 +1,484 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { installTileCache } from './qa/tileCache.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) { - throw new Error('Supply an unreserved local preview URL'); -} +const base = localPreview(process.argv[2]); base.search = '?welcome=0&look=cinematic'; base.hash = 'v=2&lat=42.3554&lon=-71.0605&alt=6000&heading=0&pitch=-90&map=osm&l=e'; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - await installTileCache(page); - await page.evaluateOnNewDocument(() => { - localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); - const fetch = window.fetch.bind(window); - // Two Boston vessels for the Contacts session; empty air and site feeds. - const fixtures = { - '/api/vessels': () => ({ - status: 'live', - rows: ['123456789', '223456789'].map((mmsi, i) => ({ - mmsi, - name: 'Boston ' + i, - lat: 42.3554, - lon: -71.0605 + i * 0.05, - last_position_epoch: Date.now() / 1000, - })), - }), - '/api/vessels/track': () => ({ samples: [] }), - '/api/flights': () => ({ time: Date.now() / 1000, states: [] }), - '/api/military': () => ({ now: Date.now(), ac: [] }), - '/api/military-installations': () => ({ elements: [], status: 'ok' }), - }; - window.fetch = (url, options) => { - if ( - String(url).startsWith('https://earthquake.usgs.gov/earthquakes/feed/') - ) - return Promise.resolve( - Response.json({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - id: 'qa-boston', - geometry: { - type: 'Point', - coordinates: [-71.0605, 42.3554, 8], +const strict = createStrict({ origin: base.origin, layers: ['earthquakes'] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + await page.evaluateOnNewDocument(() => { + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + const fetch = window.fetch.bind(window); + // Two Boston vessels for the Contacts session; empty air and site feeds. + const fixtures = { + '/api/vessels': () => ({ + status: 'live', + rows: ['123456789', '223456789'].map((mmsi, i) => ({ + mmsi, + name: 'Boston ' + i, + lat: 42.3554, + lon: -71.0605 + i * 0.05, + last_position_epoch: Date.now() / 1000, + })), + }), + '/api/vessels/track': () => ({ samples: [] }), + '/api/flights': () => ({ time: Date.now() / 1000, states: [] }), + '/api/military': () => ({ now: Date.now(), ac: [] }), + '/api/military-installations': () => ({ elements: [], status: 'ok' }), + }; + window.fetch = (url, options) => { + if ( + String(url).startsWith( + 'https://earthquake.usgs.gov/earthquakes/feed/', + ) + ) + return Promise.resolve( + Response.json({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'qa-boston', + geometry: { + type: 'Point', + coordinates: [-71.0605, 42.3554, 8], + }, + properties: { mag: 4, time: Date.now(), place: 'QA Boston' }, }, - properties: { mag: 4, time: Date.now(), place: 'QA Boston' }, - }, - ], - }), - ); - const fixture = - fixtures[new URL(url.url ?? url, document.baseURI).pathname]; - return fixture - ? Promise.resolve(Response.json(fixture())) - : fetch(url, options); - }; - }); - await page.setViewport({ width: 1280, height: 800 }); - const errors = []; - page.on('pageerror', (error) => errors.push(error.message)); - async function ready(kind) { - await page.waitForFunction( - (kind) => - window.__godsEyeView?.engine?.kind === kind && - document.querySelector('#globe-engine')?.value === kind && - !document.querySelector('#globe-engine').disabled && - document.getElementById('loading-screen')?.classList.contains('hidden'), - { timeout: 90000 }, - kind, - ); - await page.evaluate(async () => { - await window.__godsEyeView.styleManager.initialRestorePromise; - if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) - throw new Error('Keyless preview required'); + ], + }), + ); + const fixture = + fixtures[new URL(url.url ?? url, document.baseURI).pathname]; + return fixture + ? Promise.resolve(Response.json(fixture())) + : fetch(url, options); + }; }); - assert.equal(await page.$eval('#globe-engine', (el) => el.value), kind); - } - async function verifyQuakeSurface(kind) { - const meanRed = async (enabled) => { - await page.evaluate(async (enabled) => { - const a = window.__godsEyeView; - await a.dataManager.setEnabled('earthquakes', enabled, { - origin: 'user', - }); - a.engine.render.request('qa-quake-surface'); - }, enabled); - if (enabled) - await page.waitForFunction( - () => - window.__godsEyeView.dataManager.layers - .get('earthquakes') - .module.getAnalystRecords().length === 1, - ); - await new Promise((resolve) => setTimeout(resolve, 1800)); - const bytes = await page.screenshot({ - clip: { x: 675, y: 403, width: 20, height: 20 }, + await page.setViewport({ width: 1280, height: 800 }); + const errors = []; + page.on('pageerror', (error) => errors.push(error.message)); + async function ready(kind) { + await page.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + document.querySelector('#globe-engine')?.value === kind && + !document.querySelector('#globe-engine').disabled && + document + .getElementById('loading-screen') + ?.classList.contains('hidden'), + { timeout: 90000 }, + kind, + ); + await page.evaluate(async () => { + await window.__godsEyeView.styleManager.initialRestorePromise; + if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) + throw new Error('Keyless preview required'); }); - const { channels } = await sharp(bytes).stats(); - return channels[0].mean - channels[1].mean; - }; - const off = await meanRed(false), - on = await meanRed(true); - assert.ok( - on > off + 8, - kind + - ' must draw the shallow red earthquake footprint: ' + - off + - ' -> ' + - on, - ); - console.log( - 'PASS', - kind, - 'earthquake footprint red minus green:', - off, - '->', - on, - ); - } - const stored = () => page.evaluate(() => localStorage.getItem('gev.engine')); - async function switchTo(kind) { - if ( - await page.$eval( - '#control-panel-toggle', - (el) => el.getAttribute('aria-expanded') !== 'true', - ) - ) { - await page.focus('#control-panel-toggle'); - await page.keyboard.press('Enter'); + assert.equal(await page.$eval('#globe-engine', (el) => el.value), kind); } - await page.waitForSelector('#globe-engine', { visible: true }); - await page.evaluate(() => { - window.__switchDocument = document; - window.__quakeEntry = - window.__godsEyeView.dataManager.layers.get('earthquakes'); - window.__quakeTimer = window.__quakeEntry.intervalId; - window.__renderers ??= new Map(); - window.__renderers.set( - window.__godsEyeView.engine.kind, - window.__godsEyeView.engine, + async function verifyQuakeSurface(kind) { + const meanRed = async (enabled) => { + await page.evaluate(async (enabled) => { + const a = window.__godsEyeView; + await a.dataManager.setEnabled('earthquakes', enabled, { + origin: 'user', + }); + a.engine.render.request('qa-quake-surface'); + }, enabled); + if (enabled) + await page.waitForFunction( + () => + window.__godsEyeView.dataManager.layers + .get('earthquakes') + .module.getAnalystRecords().length === 1, + ); + await new Promise((resolve) => setTimeout(resolve, 1800)); + const bytes = await page.screenshot({ + clip: { x: 675, y: 403, width: 20, height: 20 }, + }); + const { channels } = await sharp(bytes).stats(); + return channels[0].mean - channels[1].mean; + }; + const off = await meanRed(false), + on = await meanRed(true); + assert.ok( + on > off + 8, + kind + + ' must draw the shallow red earthquake footprint: ' + + off + + ' -> ' + + on, ); - window.__switchSplash = false; - window.__switchObserver = new MutationObserver(() => { - if (!document.querySelector('#loading-screen.hidden')) - window.__switchSplash = true; + await strict.passed(page, 'earthquakes', 'shallow red footprint drawn', { + off, + on, }); - window.__switchObserver.observe( - document.querySelector('#loading-screen'), - { attributes: true }, + console.log( + 'PASS', + kind, + 'earthquake footprint red minus green:', + off, + '->', + on, ); - window.__outgoingEngine = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', kind); - await ready(kind); - assert.ok(await page.evaluate(() => window.__switchDocument === document)); - assert.ok( + } + const stored = () => + page.evaluate(() => localStorage.getItem('gev.engine')); + async function switchTo(kind) { + if ( + await page.$eval( + '#control-panel-toggle', + (el) => el.getAttribute('aria-expanded') !== 'true', + ) + ) { + await page.focus('#control-panel-toggle'); + await page.keyboard.press('Enter'); + } + await page.waitForSelector('#globe-engine', { visible: true }); await page.evaluate(() => { - window.__switchObserver.disconnect(); - const { engine } = window.__godsEyeView; - const previous = window.__renderers.get(engine.kind); - return ( - !window.__switchSplash && - window.__quakeEntry === - window.__godsEyeView.dataManager.layers.get('earthquakes') && - window.__quakeTimer === window.__quakeEntry.intervalId && - (!previous || previous === engine) && - !window.__outgoingEngine.isDestroyed() && - !window.__outgoingEngine.host.canvas.isConnected && - engine.host.canvas.isConnected && - window.__godsEyeView.dataManager - .getEnabledLayerIds() - .has('earthquakes') + window.__switchDocument = document; + window.__quakeEntry = + window.__godsEyeView.dataManager.layers.get('earthquakes'); + window.__quakeTimer = window.__quakeEntry.intervalId; + window.__renderers ??= new Map(); + window.__renderers.set( + window.__godsEyeView.engine.kind, + window.__godsEyeView.engine, ); + window.__switchSplash = false; + window.__switchObserver = new MutationObserver(() => { + if (!document.querySelector('#loading-screen.hidden')) + window.__switchSplash = true; + }); + window.__switchObserver.observe( + document.querySelector('#loading-screen'), + { attributes: true }, + ); + window.__outgoingEngine = window.__godsEyeView.engine; + }); + await strict.pick(page, kind); + await ready(kind); + assert.ok( + await page.evaluate(() => window.__switchDocument === document), + ); + assert.ok( + await page.evaluate(() => { + window.__switchObserver.disconnect(); + const { engine } = window.__godsEyeView; + const previous = window.__renderers.get(engine.kind); + return ( + !window.__switchSplash && + window.__quakeEntry === + window.__godsEyeView.dataManager.layers.get('earthquakes') && + window.__quakeTimer === window.__quakeEntry.intervalId && + (!previous || previous === engine) && + !window.__outgoingEngine.isDestroyed() && + !window.__outgoingEngine.host.canvas.isConnected && + engine.host.canvas.isConnected && + window.__godsEyeView.dataManager + .getEnabledLayerIds() + .has('earthquakes') + ); + }), + ); + await strict.passed( + page, + 'earthquakes', + 'same layer entry, model and poll timer; no splash; outgoing canvas detached', + ); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + assert.equal( + await page.evaluate(async () => { + const parked = window.__outgoingEngine; + let frames = 0; + const remove = parked.frames.postRender.addEventListener(() => { + frames++; + }); + parked.render.request('qa-parked'); + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(resolve)), + ); + remove(); + return frames; + }), + 0, + ); + const url = new URL(page.url()); + assert.equal(url.searchParams.get('engine'), kind); + assert.equal(url.searchParams.get('look'), 'cinematic'); + const hash = new URLSearchParams(url.hash.slice(1)); + assert.equal(hash.get('map'), 'osm'); + assert.ok(Math.abs(Number(hash.get('lat')) - 42.3554) < 0.01); + assert.ok(Math.abs(Number(hash.get('lon')) + 71.0605) < 0.01); + assert.ok(Math.abs(Number(hash.get('alt')) - 6000) < 30); + const view = await page.evaluate(() => { + const { engine, mapStackController } = window.__godsEyeView; + const { latitude, longitude, height } = + engine.camera.pose().cartographic; + return { + latitude: latitude * (180 / Math.PI), + longitude: longitude * (180 / Math.PI), + height, + map: mapStackController.getActiveId(), + }; + }); + assert.equal(view.map, 'osm'); + assert.ok(Math.abs(view.latitude - 42.3554) < 0.01); + assert.ok(Math.abs(view.longitude + 71.0605) < 0.01); + assert.ok(Math.abs(view.height - 6000) < 30); + await strict.passed( + page, + 'earthquakes', + 'parked engine draws no frame; pose, map and layers carried over', + ); + } + await page.goto(base.href, { waitUntil: 'domcontentloaded' }); + await ready('cesium'); + await verifyQuakeSurface('cesium'); + const sharpen = () => + page.evaluate( + () => window.__godsEyeView.styleManager.getControlState().sharpen, + ); + await page.evaluate(() => { + const style = window.__godsEyeView.styleManager; + style.setStyle('retro'); + style.setStyle('normal'); + }); + assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 49 }); + await switchTo('navara'); + await verifyQuakeSurface('navara'); + assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 80 }); + assert.ok( + await page.evaluate( + () => + Math.abs(window.__godsEyeView.engine.render.sharpen().amount - 1.7) < + 1e-6, + ), + ); + assert.equal(await stored(), 'navara'); + await page.goBack({ waitUntil: 'domcontentloaded' }); + await ready('cesium'); + assert.equal((await sharpen()).intensityPct, 49); + await page.goForward(); + await ready('navara'); + assert.equal((await sharpen()).intensityPct, 80); + await page.evaluate(() => + window.__godsEyeView.styleManager.setSharpen({ + enabled: false, + intensityPct: 49, }), ); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), + await switchTo('cesium'); + await switchTo('navara'); + assert.deepEqual(await sharpen(), { enabled: false, intensityPct: 49 }); + // The address carries the running layer set once its debounced write lands. + const settledAddress = async () => { + let previous; + for (;;) { + await new Promise((resolve) => setTimeout(resolve, 1200)); + const current = await page.evaluate(() => + new URLSearchParams(location.hash.slice(1)).get('l'), + ); + if (current === previous) return current; + previous = current; + } + }; + const pick = async (kind) => { + await strict.pick(page, kind); + await ready(kind); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + }; + const visit = async (direction, kind) => { + await (direction === 'back' + ? page.goBack({ waitUntil: 'domcontentloaded' }) + : page.goForward()); + await ready(kind); + await page.waitForFunction( + () => !document.querySelector('.engine-switch-frame'), + ); + }; + const quakes = () => + page.evaluate(() => { + const { dataManager } = window.__godsEyeView; + return { + model: window.__quakeModel === dataManager.layers.get('earthquakes'), + enabled: dataManager.getEnabledLayerIds().has('earthquakes'), + }; + }); + // Like a same-engine Back, each entry brings back its own layer set, on + // the same layer models. + await page.evaluate(() => { + window.__quakeModel = + window.__godsEyeView.dataManager.layers.get('earthquakes'); + }); + const navaraLayers = await settledAddress(); + await pick('cesium'); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('earthquakes', false, { + origin: 'user', + }), + ); + const cesiumLayers = await settledAddress(); + assert.notEqual(cesiumLayers, navaraLayers); + await visit('back', 'navara'); + assert.deepEqual(await quakes(), { model: true, enabled: true }); + await strict.passed( + page, + 'earthquakes', + 'Back restores the enabled layer on the same model', + ); + assert.equal(await settledAddress(), navaraLayers, 'Back keeps its entry'); + await visit('forward', 'cesium'); + assert.deepEqual(await quakes(), { model: true, enabled: false }); + await strict.passed( + page, + 'earthquakes', + 'Forward restores the disabled layer on the same model', ); assert.equal( - await page.evaluate(async () => { - const parked = window.__outgoingEngine; - let frames = 0; - const remove = parked.frames.postRender.addEventListener(() => { - frames++; - }); - parked.render.request('qa-parked'); - await new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(resolve)), - ); - remove(); - return frames; + await settledAddress(), + cesiumLayers, + 'Forward keeps its entry', + ); + await page.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('earthquakes', true, { + origin: 'user', }), - 0, ); - const url = new URL(page.url()); - assert.equal(url.searchParams.get('engine'), kind); - assert.equal(url.searchParams.get('look'), 'cinematic'); - const hash = new URLSearchParams(url.hash.slice(1)); - assert.equal(hash.get('map'), 'osm'); - assert.ok(Math.abs(Number(hash.get('lat')) - 42.3554) < 0.01); - assert.ok(Math.abs(Number(hash.get('lon')) + 71.0605) < 0.01); - assert.ok(Math.abs(Number(hash.get('alt')) - 6000) < 30); - const view = await page.evaluate(() => { - const { engine, mapStackController } = window.__godsEyeView; - const { latitude, longitude, height } = engine.camera.pose().cartographic; - return { - latitude: latitude * (180 / Math.PI), - longitude: longitude * (180 / Math.PI), - height, - map: mapStackController.getActiveId(), - }; - }); - assert.equal(view.map, 'osm'); - assert.ok(Math.abs(view.latitude - 42.3554) < 0.01); - assert.ok(Math.abs(view.longitude + 71.0605) < 0.01); - assert.ok(Math.abs(view.height - 6000) < 30); - } - await page.goto(base.href, { waitUntil: 'domcontentloaded' }); - await ready('cesium'); - await verifyQuakeSurface('cesium'); - const sharpen = () => - page.evaluate( - () => window.__godsEyeView.styleManager.getControlState().sharpen, + await pick('navara'); + console.log( + "PASS Back/Forward restore each entry's layers on the same models", ); - await page.evaluate(() => { - const style = window.__godsEyeView.styleManager; - style.setStyle('retro'); - style.setStyle('normal'); - }); - assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 49 }); - await switchTo('navara'); - await verifyQuakeSurface('navara'); - assert.deepEqual(await sharpen(), { enabled: true, intensityPct: 80 }); - assert.ok( - await page.evaluate( - () => - Math.abs(window.__godsEyeView.engine.render.sharpen().amount - 1.7) < - 1e-6, - ), - ); - assert.equal(await stored(), 'navara'); - await page.goBack({ waitUntil: 'domcontentloaded' }); - await ready('cesium'); - assert.equal((await sharpen()).intensityPct, 49); - await page.goForward(); - await ready('navara'); - assert.equal((await sharpen()).intensityPct, 80); - await page.evaluate(() => - window.__godsEyeView.styleManager.setSharpen({ - enabled: false, - intensityPct: 49, - }), - ); - await switchTo('cesium'); - await switchTo('navara'); - assert.deepEqual(await sharpen(), { enabled: false, intensityPct: 49 }); - // The address carries the running layer set once its debounced write lands. - const settledAddress = async () => { - let previous; - for (;;) { - await new Promise((resolve) => setTimeout(resolve, 1200)); - const current = await page.evaluate(() => - new URLSearchParams(location.hash.slice(1)).get('l'), - ); - if (current === previous) return current; - previous = current; - } - }; - const pick = async (kind) => { - await page.select('#globe-engine', kind); - await ready(kind); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), - ); - }; - const visit = async (direction, kind) => { - await (direction === 'back' - ? page.goBack({ waitUntil: 'domcontentloaded' }) - : page.goForward()); - await ready(kind); - await page.waitForFunction( - () => !document.querySelector('.engine-switch-frame'), - ); - }; - const quakes = () => - page.evaluate(() => { - const { dataManager } = window.__godsEyeView; - return { - model: window.__quakeModel === dataManager.layers.get('earthquakes'), - enabled: dataManager.getEnabledLayerIds().has('earthquakes'), - }; - }); - // Like a same-engine Back, each entry brings back its own layer set, on - // the same layer models. - await page.evaluate(() => { - window.__quakeModel = - window.__godsEyeView.dataManager.layers.get('earthquakes'); - }); - const navaraLayers = await settledAddress(); - await pick('cesium'); - await page.evaluate(() => - window.__godsEyeView.dataManager.setEnabled('earthquakes', false, { - origin: 'user', - }), - ); - const cesiumLayers = await settledAddress(); - assert.notEqual(cesiumLayers, navaraLayers); - await visit('back', 'navara'); - assert.deepEqual(await quakes(), { model: true, enabled: true }); - assert.equal(await settledAddress(), navaraLayers, 'Back keeps its entry'); - await visit('forward', 'cesium'); - assert.deepEqual(await quakes(), { model: true, enabled: false }); - assert.equal(await settledAddress(), cesiumLayers, 'Forward keeps its entry'); - await page.evaluate(() => - window.__godsEyeView.dataManager.setEnabled('earthquakes', true, { - origin: 'user', - }), - ); - await pick('navara'); - console.log( - "PASS Back/Forward restore each entry's layers on the same models", - ); - // Back/Forward hand the session off like the picker: the Contacts subject - // and its navigation history, its layers and the Director survive. - await page.evaluate(async () => { - const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; - await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user' }); - const outcome = await styleManager.setContextMode('contacts'); - if (!outcome.ok) throw Error(JSON.stringify(outcome)); - window.qaSession = { - director: sceneDirector, - context: dataManager.layers.get('military-awareness').module, - vessels: dataManager.layers.get('ais-live-vessels'), - }; - if (!window.qaSession.vessels.module.selectById('123456789')) - throw Error('Missing first vessel'); - if ( - !window.qaSession.context.navigateNext({ - targetLayer: 'ais-live-vessels', - }) - ) - throw Error('Cannot navigate to the second vessel'); - }); - const session = () => - page.evaluate(() => { - const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; - const { director, context, vessels } = window.qaSession; - return { - director: director === sceneDirector && !director._destroyed, - owners: - context === dataManager.layers.get('military-awareness').module && - vessels === dataManager.layers.get('ais-live-vessels'), - mode: styleManager.getContextModeState().mode, - subject: context.getContextSnapshot()?.subject?.id, - history: context._getAwarenessNavigationStateForTest().historyKeys, - enabled: [...dataManager.getEnabledLayerIds()].sort(), - }; - }); - const live = await session(); - assert.equal(live.mode, 'flights'); - assert.equal(live.subject, '223456789'); - assert.ok(live.history.includes('ais-live-vessels:123456789')); - await settledAddress(); - const historyLength = await page.evaluate(() => history.length); - await pick('cesium'); - assert.deepEqual(await session(), live, 'a pick keeps the session'); - await settledAddress(); - await visit('back', 'navara'); - assert.deepEqual(await session(), live, 'Back keeps the session'); - await visit('forward', 'cesium'); - assert.deepEqual(await session(), live, 'Forward keeps the session'); - assert.equal(await page.evaluate(() => history.length), historyLength + 1); - assert.ok( + // Back/Forward hand the session off like the picker: the Contacts subject + // and its navigation history, its layers and the Director survive. await page.evaluate(async () => { - const { dataManager, styleManager } = window.__godsEyeView; - const result = await styleManager.setContextMode('off'); - await dataManager.setEnabled('ais-live-vessels', false, { + const { dataManager, styleManager, sceneDirector } = window.__godsEyeView; + await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user', }); - return result.ok; - }), - ); - await pick('navara'); - console.log( - 'PASS Back/Forward keep Contacts subject, history, layers and Director', - ); - await page.goto(base.href, { waitUntil: 'domcontentloaded' }); - await ready('navara'); - await switchTo('cesium'); - assert.equal(await stored(), 'cesium'); - await page.evaluate(() => { - const write = Storage.prototype.setItem; - Storage.prototype.setItem = function (key, value) { - if (key === 'gev.engine') - throw new DOMException('Blocked', 'SecurityError'); - return write.call(this, key, value); - }; - }); - await switchTo('navara'); - assert.equal(await stored(), 'cesium'); - // The real Map Source chips move into daylight inspection and back to live lighting. - await page.$eval('[data-stack-id="sentinel2"]', (el) => el.click()); - await page.waitForFunction( - () => - window.__godsEyeView.mapStackController.getActiveId() === 'sentinel2' && - window.__godsEyeView.engine.host.container.hasAttribute( - 'data-terrain-daylight', - ), - ); - // Zooming out on the satellite map must keep the same inspection sun. - assert.ok( - await page.evaluate(async () => { - const { engine } = window.__godsEyeView; - engine.camera.zoomOut(2_000_000); - await new Promise((resolve) => { - const remove = engine.frames.postRender.addEventListener(() => { - remove(); - resolve(); - }); - engine.render.request('qa-orbital-daylight'); + const outcome = await styleManager.setContextMode('contacts'); + if (!outcome.ok) throw Error(JSON.stringify(outcome)); + window.qaSession = { + director: sceneDirector, + context: dataManager.layers.get('military-awareness').module, + vessels: dataManager.layers.get('ais-live-vessels'), + }; + if (!window.qaSession.vessels.module.selectById('123456789')) + throw Error('Missing first vessel'); + if ( + !window.qaSession.context.navigateNext({ + targetLayer: 'ais-live-vessels', + }) + ) + throw Error('Cannot navigate to the second vessel'); + }); + const session = () => + page.evaluate(() => { + const { dataManager, styleManager, sceneDirector } = + window.__godsEyeView; + const { director, context, vessels } = window.qaSession; + return { + director: director === sceneDirector && !director._destroyed, + owners: + context === dataManager.layers.get('military-awareness').module && + vessels === dataManager.layers.get('ais-live-vessels'), + mode: styleManager.getContextModeState().mode, + subject: context.getContextSnapshot()?.subject?.id, + history: context._getAwarenessNavigationStateForTest().historyKeys, + enabled: [...dataManager.getEnabledLayerIds()].sort(), + }; }); - return ( - engine.camera.height() > 1_500_000 && - engine.host.container.hasAttribute('data-terrain-daylight') - ); - }), - 'the orbital frame keeps daylight on the inspected globe', - ); - await page.$eval('[data-stack-id="osm"]', (el) => el.click()); - await page.waitForFunction( - () => - window.__godsEyeView.mapStackController.getActiveId() === 'osm' && - !window.__godsEyeView.engine.host.container.hasAttribute( - 'data-terrain-daylight', - ), - ); - assert.deepEqual(errors, []); - console.log( - 'PASS Cesium default, both switches, same document, warm engine identity, parked canvas, no startup splash, share pose/map/layers, Back/Forward, remembered engine, blocked preference storage', - ); -} finally { - await browser.close(); -} + const live = await session(); + assert.equal(live.mode, 'flights'); + assert.equal(live.subject, '223456789'); + assert.ok(live.history.includes('ais-live-vessels:123456789')); + await settledAddress(); + const historyLength = await page.evaluate(() => history.length); + await pick('cesium'); + assert.deepEqual(await session(), live, 'a pick keeps the session'); + await settledAddress(); + await visit('back', 'navara'); + assert.deepEqual(await session(), live, 'Back keeps the session'); + await visit('forward', 'cesium'); + assert.deepEqual(await session(), live, 'Forward keeps the session'); + assert.equal(await page.evaluate(() => history.length), historyLength + 1); + assert.ok( + await page.evaluate(async () => { + const { dataManager, styleManager } = window.__godsEyeView; + const result = await styleManager.setContextMode('off'); + await dataManager.setEnabled('ais-live-vessels', false, { + origin: 'user', + }); + return result.ok; + }), + ); + await pick('navara'); + console.log( + 'PASS Back/Forward keep Contacts subject, history, layers and Director', + ); + await page.goto(base.href, { waitUntil: 'domcontentloaded' }); + await ready('navara'); + await switchTo('cesium'); + assert.equal(await stored(), 'cesium'); + await page.evaluate(() => { + const write = Storage.prototype.setItem; + Storage.prototype.setItem = function (key, value) { + if (key === 'gev.engine') + throw new DOMException('Blocked', 'SecurityError'); + return write.call(this, key, value); + }; + }); + await switchTo('navara'); + assert.equal(await stored(), 'cesium'); + // The real Map Source chips move into daylight inspection and back to live lighting. + await page.$eval('[data-stack-id="sentinel2"]', (el) => el.click()); + await page.waitForFunction( + () => + window.__godsEyeView.mapStackController.getActiveId() === 'sentinel2' && + window.__godsEyeView.engine.host.container.hasAttribute( + 'data-terrain-daylight', + ), + ); + // Zooming out on the satellite map must keep the same inspection sun. + assert.ok( + await page.evaluate(async () => { + const { engine } = window.__godsEyeView; + engine.camera.zoomOut(2_000_000); + await new Promise((resolve) => { + const remove = engine.frames.postRender.addEventListener(() => { + remove(); + resolve(); + }); + engine.render.request('qa-orbital-daylight'); + }); + return ( + engine.camera.height() > 1_500_000 && + engine.host.container.hasAttribute('data-terrain-daylight') + ); + }), + 'the orbital frame keeps daylight on the inspected globe', + ); + await page.$eval('[data-stack-id="osm"]', (el) => el.click()); + await page.waitForFunction( + () => + window.__godsEyeView.mapStackController.getActiveId() === 'osm' && + !window.__godsEyeView.engine.host.container.hasAttribute( + 'data-terrain-daylight', + ), + ); + assert.deepEqual(errors, []); + console.log( + 'PASS Cesium default, both switches, same document, warm engine identity, parked canvas, no startup splash, share pose/map/layers, Back/Forward, remembered engine, blocked preference storage', + ); + }, +); diff --git a/scripts/qa-fire-camera-switch.mjs b/scripts/qa-fire-camera-switch.mjs index 12ce71c5d..75b635f98 100644 --- a/scripts/qa-fire-camera-switch.mjs +++ b/scripts/qa-fire-camera-switch.mjs @@ -1,19 +1,10 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { installAlprFixture } from './fixtures/w2/alpr.mjs'; -import { firmsFixture } from './fixtures/w2/firms.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -const url = new URL(process.argv[2]), +import { installAlprFixture, cameras } from './fixtures/w2/alpr.mjs'; +import { firmsFixture, FIRMS_VALID_COUNT } from './fixtures/w2/firms.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]), fire = process.argv.includes('--fires'), id = fire ? 'local-firms' : 'alpr-cameras'; -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&' + @@ -21,13 +12,17 @@ url.hash = ? 'lat=37.77&lon=-122.42&alt=100000' : 'lat=30.262&lon=-97.7431&alt=5500') + '&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Records the fixtures produce: the valid FIRMS rows; the ALPR cameras the +// fixture tiles carry around Austin. +const COUNT = fire ? FIRMS_VALID_COUNT : cameras.length; +const strict = createStrict({ + origin: url.origin, + layers: [id], + // installAlprFixture answers the camera tiles. + fixtures: ['https://tiles.dontgetflocked.com/*'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => errors.push(e.message)); @@ -48,7 +43,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async (id) => { @@ -62,52 +57,123 @@ try { await p.waitForFunction(() => window.qaGround?.getStats().count > 0, { timeout: 30000, }); - const target = await p.waitForFunction( - (fire) => { - const a = window.__godsEyeView, - m = window.qaGround; - const points = fire - ? m - .getDetectableObjects({ maxCount: 1000 }) - .filter((v) => v.id === m.getAnalystRecords(1)[0]?.id) - : [...window.__gevContextStore.entities.values()] - .filter((v) => v.layerId === 'alpr-cameras') - .map((v) => ({ - id: v.id, - position: - v.entity.gevAlprCanvasPosition || - v.entity.gevDisplayPosition?.(), - })); - for (const row of points) { - if (!row.position) continue; - const s = a.engine.projector.worldToWindow(row.position); - if (!s || s.x < 430 || s.x > 850 || s.y < 220 || s.y > 730) continue; - const picked = a.engine.picker.pick(s); - if (fire || picked?.id === row.id || picked?.id?.id === row.id) - return { x: s.x, y: s.y, id: row.id }; - } - return false; - }, - { timeout: 30000 }, - fire, - ); - const xy = await target.jsonValue(); - await new Promise(r=>setTimeout(r,1000)); + // A pickable target in the middle of the view: the camera the engine's + // own picker returns, or the hottest fire. + const findTarget = () => + p + .waitForFunction( + (fire) => { + const a = window.__godsEyeView, + m = window.qaGround; + const points = fire + ? m + .getDetectableObjects({ maxCount: 1000 }) + .filter((v) => v.id === m.getAnalystRecords(1)[0]?.id) + : [...window.__gevContextStore.entities.values()] + .filter((v) => v.layerId === 'alpr-cameras') + .map((v) => ({ + id: v.id, + position: + v.entity.gevAlprCanvasPosition || + v.entity.gevDisplayPosition?.(), + })); + for (const row of points) { + if (!row.position) continue; + const s = a.engine.projector.worldToWindow(row.position); + if (!s || s.x < 430 || s.x > 850 || s.y < 220 || s.y > 600) + continue; + const canvas = a.engine.host.canvas; + const box = canvas.getBoundingClientRect(); + if ( + !canvas.contains( + document.elementFromPoint(box.left + s.x, box.top + s.y), + ) + ) + continue; + const picked = a.engine.picker.pick(s); + if (fire || picked?.id === row.id || picked?.id?.id === row.id) + return { x: box.left + s.x, y: box.top + s.y, id: row.id }; + } + return false; + }, + { timeout: 30000 }, + fire, + ) + .then((handle) => handle.jsonValue()); + const selected = (id) => + p.waitForFunction( + (id, fire) => + fire + ? window.qaGround.getSelectedInfo()?.frp === 999 + : window.__gevContextStore?.selectedEntityId === id, + { timeout: 15000 }, + id, + fire, + ); + // A click on empty ground (nothing picked there, nothing covering it). + const clickEmpty = async () => { + const handle = await p.waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 240; y < 600; y += 9) + for (let x = 440; x < 840; x += 9) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [6, 0], + [-6, 0], + [0, 6], + [0, -6], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ); + const at = await handle.jsonValue(); + await p.mouse.click(at.x, at.y); + await p.waitForFunction( + (fire) => + fire + ? !window.qaGround.getSelectedInfo() + : !window.__gevContextStore?.selectedEntityId, + { timeout: 15000 }, + fire, + ); + }; + const xy = await findTarget(); + await new Promise((r) => setTimeout(r, 1000)); await p.mouse.click(xy.x, xy.y); - await p.waitForFunction( - (id,fire) => fire ? window.qaGround.getSelectedInfo()?.frp === 999 : window.__gevContextStore?.selectedEntityId === id, - {}, - xy.id, fire, + await selected(xy.id); + await strict.passed( + p, + id, + fire + ? 'a click on the hottest fire selects it' + : 'a click on a camera selects it', + xy, ); await new Promise((r) => setTimeout(r, 1600)); - const expectedId = fire ? await p.evaluate(()=>window.qaGround.getSelectedInfo().id) : xy.id; + const expectedId = fire + ? await p.evaluate(() => window.qaGround.getSelectedInfo().id) + : xy.id; const before = await p.evaluate(() => ({ stamp: window.qaGround.getStats().lastUpdate, records: window.qaGround.getAnalystRecords(1000).map((v) => v.id), reads: window.qaFirmsReads, })); for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); const result = await p.evaluate( (id) => ({ same: @@ -129,9 +195,31 @@ try { assert.equal(result.reads, before.reads); assert.deepEqual(result.records, before.records); assert.equal(result.stamp, before.stamp); + await strict.check( + p, + id, + `same model, ${COUNT} records, the clicked record still selected, no source read, same receipt time`, + result.count === COUNT, + { count: result.count }, + ); + if (kind === 'cesium') { + // Click-picking on Cesium: clear the selection with a click on empty + // ground, then click the same record again. + await clickEmpty(); + const again = await findTarget(); + assert.equal(again.id, xy.id); + await p.mouse.click(again.x, again.y); + await selected(again.id); + await strict.passed( + p, + id, + fire + ? 'a click on empty ground clears the fire; a click on it selects it again' + : 'a click on empty ground clears the camera; a click on it selects it again', + again, + ); + } } assert.deepEqual(errors, []); console.log('PASS retained source, stable clicked record on each planet'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-ghostmaps-switch.mjs b/scripts/qa-ghostmaps-switch.mjs index d6542da7d..d5633ffbe 100644 --- a/scripts/qa-ghostmaps-switch.mjs +++ b/scripts/qa-ghostmaps-switch.mjs @@ -1,27 +1,15 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import { readFile } from 'node:fs/promises'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['s2-ghostmaps'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -78,7 +66,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -108,12 +96,22 @@ try { ); await p.mouse.click(x, y); }; + const overlay = () => + p.evaluate(() => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return pixels; + }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { await p.evaluate(() => { window.ghostStats.titles = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction(() => window.ghostStats.titles.some((v) => /QA zone/i.test(v)), @@ -133,6 +131,11 @@ try { streams: 1, reads: 1, }); + await strict.passed( + p, + 's2-ghostmaps', + 'same model, three features, one event stream, one KMZ read', + ); await new Promise((resolve) => setTimeout(resolve, 2200)); for (const [name, lon, lat] of [ ['QA pin', -79.38, 43.65], @@ -148,19 +151,51 @@ try { { timeout: 12000 }, name, ); + await strict.passed( + p, + 's2-ghostmaps', + `a click on ${name} draws its card text`, + ); console.log('PASS', kind, name, 'pointer hit'); } + // Not vacuous: the zone's card is up, so the overlay only clears when + // the click in the hole is handled as a click on empty ground. The + // overlay without any card is measured with a click far from every + // feature. + const shown = await overlay(); await click(-79.316, 43.61); - await new Promise((resolve) => setTimeout(resolve, 600)); - await p.evaluate(() => { - window.ghostStats.titles = []; - window.__godsEyeView.engine.render.request('qa-hole'); - }); - await new Promise((resolve) => setTimeout(resolve, 400)); - assert.equal( - await p.evaluate(() => window.ghostStats.titles.length), - 0, - kind + ' polygon hole', + await p.waitForFunction( + (shown) => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return pixels < shown - 100; + }, + { timeout: 10000 }, + shown, + ); + const hole = await overlay(); + await click(-79.45, 43.55); + await new Promise((resolve) => setTimeout(resolve, 800)); + const empty = await overlay(); + console.log( + kind, + 'overlay pixels: zone card', + shown, + 'hole', + hole, + 'empty ground', + empty, + ); + await strict.check( + p, + 's2-ghostmaps', + 'a click in the zone hole clears the selected zone card like empty ground', + shown > empty + 100 && Math.abs(hole - empty) <= 0.02 * shown, + { shown, hole, empty }, ); await click(-79.28, 43.61); await p.waitForFunction(() => @@ -168,6 +203,4 @@ try { ); } assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-ground-layer-switch.mjs b/scripts/qa-ground-layer-switch.mjs index b1f58133d..96093706b 100644 --- a/scripts/qa-ground-layer-switch.mjs +++ b/scripts/qa-ground-layer-switch.mjs @@ -1,27 +1,23 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l=4'; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: [ + 'ais-live-vessels', + 'air-quality', + 'gnss-integrity', + 'fire-perimeters', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -34,30 +30,66 @@ try { const url = new URL(input.url ?? input, document.baseURI); if (url.pathname === '/api/vessels/track') { window.qaVesselHistories = (window.qaVesselHistories ?? 0) + 1; - return Promise.resolve(Response.json({ samples: [{lat: 43.63, lon: -79.38}] })); + return Promise.resolve( + Response.json({ samples: [{ lat: 43.63, lon: -79.38 }] }), + ); } if (url.pathname === '/api/vessels') { - return Promise.resolve(Response.json({status: 'live', rows: [{ - mmsi: '123456789', name: 'Toronto QA', lat: 43.65, lon: -79.38, - last_position_epoch: Date.now() / 1000, - }]})); + return Promise.resolve( + Response.json({ + status: 'live', + rows: [ + { + mmsi: '123456789', + name: 'Toronto QA', + lat: 43.65, + lon: -79.38, + last_position_epoch: Date.now() / 1000, + }, + ], + }), + ); } if (url.pathname.startsWith('/api/fire-perimeters/inciweb')) { return Promise.resolve(Response.json({ rows: [] })); } if (url.pathname === '/api/fire-perimeters') { const square = (dx) => [ - [-79.38-dx,43.65-dx],[-79.38+dx,43.65-dx],[-79.38+dx,43.65+dx], - [-79.38-dx,43.65+dx],[-79.38-dx,43.65-dx], + [-79.38 - dx, 43.65 - dx], + [-79.38 + dx, 43.65 - dx], + [-79.38 + dx, 43.65 + dx], + [-79.38 - dx, 43.65 + dx], + [-79.38 - dx, 43.65 - dx], ]; - return Promise.resolve(Response.json({rows: [{ - stableId: 'fire:fixture', name:'QA Fire', acres:500, containedPct:40, - polygons:[[square(0.09), square(0.018).reverse()]], - }]})); + return Promise.resolve( + Response.json({ + rows: [ + { + stableId: 'fire:fixture', + name: 'QA Fire', + acres: 500, + containedPct: 40, + polygons: [[square(0.09), square(0.018).reverse()]], + }, + ], + }), + ); } if (url.pathname === '/api/gnss-integrity') { - return Promise.resolve(Response.json({ageMs: 0, rows: ['a1','a2','a3'].map(hex => - ({hex, lat: 43.65, lon: -79.38, nic: 0, nacp: 0, degraded: true, gpsLost: false}))})); + return Promise.resolve( + Response.json({ + ageMs: 0, + rows: ['a1', 'a2', 'a3'].map((hex) => ({ + hex, + lat: 43.65, + lon: -79.38, + nic: 0, + nacp: 0, + degraded: true, + gpsLost: false, + })), + }), + ); } if (url.hostname === 'api.weather.gc.ca') { const records = url.pathname.includes('aqhi-stations') @@ -93,7 +125,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -105,12 +137,78 @@ try { ); }; const capture = async (visible, layerId = 'air-quality') => { - await p.evaluate(async (show, id) => { - const d = window.__godsEyeView; - await d.dataManager.setEnabled(id, show, { origin: 'user' }); - d.engine.render.request('qa-air-quality'); - }, visible, layerId); - await new Promise((r) => setTimeout(r, 3900)); + await p.evaluate( + async (show, id) => { + const d = window.__godsEyeView; + await d.dataManager.setEnabled(id, show, { origin: 'user' }); + d.engine.render.request('qa-air-quality'); + }, + visible, + layerId, + ); + // Ready to compare: the layer's data is in (when shown), the globe's + // tiles under the view have loaded and stayed loaded, and a frame has + // been drawn since. + await p + .waitForFunction( + (show, id) => { + const d = window.__godsEyeView; + if (!d?.engine) return false; + const module = d.dataManager.layers.get(id).module; + const data = + !show || + (id === 'air-quality' + ? module.getAnalystRecords().length === 1 + : module.getStats().levels?.high === 1); + const now = performance.now(); + // The engines draw on request: keep frames coming so tiles load. + d.engine.render.request('qa-ground-settle'); + if (!data || !d.engine.surface.tilesLoaded()) { + window.__qaSettledSince = null; + return false; + } + window.__qaSettledSince ??= now; + return now - window.__qaSettledSince > 1200; + }, + { polling: 100, timeout: 30000 }, + visible, + layerId, + ) + .catch(async (error) => { + console.log( + 'not settled', + JSON.stringify( + await p.evaluate((id) => { + const d = window.__godsEyeView; + const module = d?.dataManager.layers.get(id).module; + return { + engine: d?.engine?.kind, + tiles: d?.engine?.surface.tilesLoaded(), + records: module?.getAnalystRecords().length, + stats: module?.getStats(), + }; + }, layerId), + ), + ); + throw error; + }); + // A frame drawn after the settle (the engine draws on request), then + // the animation frame that presents it. + await p.evaluate( + () => + new Promise((resolve) => { + const { engine } = window.__godsEyeView; + const remove = engine.frames.postRender.addEventListener(() => { + remove(); + requestAnimationFrame(() => resolve()); + }); + engine.render.request('qa-ground'); + }), + ); + await p.evaluate(() => (window.__qaSettledSince = null)); + return sample(); + }; + const sample = async () => { const shot = await p.screenshot(); return sharp(shot) .extract({ left: 537, top: 403, width: 22, height: 18 }) @@ -119,6 +217,19 @@ try { .toBuffer() .then((buf) => sharp(buf).stats()); }; + // The layer shown, sampled until its polygon appears in the patch (the + // engines draw a clamped polygon a few frames after its data, with no + // signal for it), for at most 20 s; the comparison stays the same. + const orangeOf = (v) => v.channels[0].mean - v.channels[2].mean; + const drawn = async (off, layerId) => { + const deadline = Date.now() + 20000; + let on = await capture(true, layerId); + while (orangeOf(on) <= orangeOf(off) + 5 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 500)); + on = await sample(); + } + return on; + }; await ready('navara'); await p.evaluate(() => { const d = window.__godsEyeView; @@ -129,19 +240,27 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); const same = await p.evaluate( () => window.__godsEyeView.dataManager.layers.get('air-quality').module === - window.qaAir && - window.__godsEyeView.dataManager.layers.get('gnss-integrity').module === window.qaGnss, + window.qaAir && + window.__godsEyeView.dataManager.layers.get('gnss-integrity') + .module === window.qaGnss, + ); + await strict.check( + p, + ['air-quality', 'gnss-integrity'], + 'same layer models after the switch', + same, ); - assert.ok(same); } - await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('gnss-integrity', false)); + await p.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('gnss-integrity', false), + ); const off = await capture(false), - on = await capture(true); + on = await drawn(off, 'air-quality'); const values = await p.evaluate(() => ({ records: window.qaAir.getAnalystRecords(), card: window.qaAir.getRowControls().list.items, @@ -162,54 +281,128 @@ try { assert.equal(values.records[0].name, 'QA Toronto'); assert.equal(values.records[0].value, 9); assert.equal(values.card.length, 1); + await strict.passed( + p, + 'air-quality', + 'AQHI 9 station footprint drawn, one card row', + { + off: orange(off), + on: orange(on), + }, + ); await capture(false); const cellOff = await capture(false, 'gnss-integrity'); - const cellOn = await capture(true, 'gnss-integrity'); - assert.ok(orange(cellOn) > orange(cellOff) + 5, kind + ' must draw GNSS high cell'); + const cellOn = await drawn(cellOff, 'gnss-integrity'); + assert.ok( + orange(cellOn) > orange(cellOff) + 5, + kind + ' must draw GNSS high cell', + ); const gnss = await p.evaluate(() => window.qaGnss.getStats()); assert.equal(gnss.levels.high, 1); - console.log('PASS', kind, 'GNSS high cell, view anchor', gnss.coverage.latitude, gnss.coverage.longitude); - await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('air-quality', true)); + await strict.passed(p, 'gnss-integrity', 'one high GNSS cell drawn', { + off: orange(cellOff), + on: orange(cellOn), + }); + console.log( + 'PASS', + kind, + 'GNSS high cell, view anchor', + gnss.coverage.latitude, + gnss.coverage.longitude, + ); + await p.evaluate(() => + window.__godsEyeView.dataManager.setEnabled('air-quality', true), + ); } await p.evaluate(async () => { - const {dataManager} = window.__godsEyeView; + const { dataManager } = window.__godsEyeView; await dataManager.setEnabled('air-quality', false); - await dataManager.setEnabled('fire-perimeters', true, {origin: 'user'}); + await dataManager.setEnabled('fire-perimeters', true, { origin: 'user' }); window.qaFire = dataManager.layers.get('fire-perimeters').module; - if (!dataManager.isEnabled('fire-perimeters') || window.qaFire.getStats().count !== 1) - throw Error('Failed to enable fire perimeter fixture: '+JSON.stringify(window.qaFire.getStats())); + if ( + !dataManager.isEnabled('fire-perimeters') || + window.qaFire.getStats().count !== 1 + ) + throw Error( + 'Failed to enable fire perimeter fixture: ' + + JSON.stringify(window.qaFire.getStats()), + ); }); - const card = async (visible) => p.waitForFunction((want) => { - const canvas = document.getElementById('world-overlay-canvas'); - const data = canvas.getContext('2d').getImageData(0,0,canvas.width,canvas.height).data; - let pixels = 0; - for (let i=3; i 100 : pixels === 0; - }, {timeout: 15000}, visible); + const card = async (visible) => + p.waitForFunction( + (want) => { + const canvas = document.getElementById('world-overlay-canvas'); + const data = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let pixels = 0; + for (let i = 3; i < data.length; i += 4) if (data[i]) pixels++; + return want ? pixels > 100 : pixels === 0; + }, + { timeout: 15000 }, + visible, + ); const click = async (lon, lat) => { - const {x, y} = await p.evaluate((world) => { - const {engine} = window.__godsEyeView; - const point = engine.projector.worldToWindow(world, {}); - const box = engine.host.canvas.getBoundingClientRect(); - return { x: box.left+point.x, y: box.top+point.y }; - }, fromDegrees(lon, lat)); - await p.mouse.click(x,y); + const { x, y } = await p.evaluate( + (world) => { + const { engine } = window.__godsEyeView; + const point = engine.projector.worldToWindow(world, {}); + if (!point) throw Error('the point is not in view'); + const box = engine.host.canvas.getBoundingClientRect(); + return { x: box.left + point.x, y: box.top + point.y }; + }, + fromDegrees(lon, lat), + ); + await p.mouse.click(x, y); }; for (const kind of ['navara', 'cesium', 'navara']) { - if (await p.evaluate(() => window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine', kind); + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await strict.pick(p, kind); await ready(kind); await card(true); - assert.ok(await p.evaluate(() => window.qaFire === window.__godsEyeView.dataManager.layers.get('fire-perimeters').module)); + await strict.check( + p, + 'fire-perimeters', + 'same model; the incident card survives the switch', + await p.evaluate( + () => + window.qaFire === + window.__godsEyeView.dataManager.layers.get('fire-perimeters') + .module, + ), + ); } await click(-79.51, 43.65); await card(false); - await click(-79.38, 43.65); - await new Promise(resolve => setTimeout(resolve, 500)); - await card(false); + await strict.passed( + p, + 'fire-perimeters', + 'a click outside the perimeter clears the card', + ); await click(-79.435, 43.65); await card(true); - console.log('PASS', kind, 'select fire under GNSS, exclude hole, retain incident card'); + await strict.check( + p, + 'fire-perimeters', + 'a click on the ring selects the fire', + await p.evaluate(() => window.qaFire.getSelectedInfo?.()?.id ?? true), + ); + // Not vacuous: the fire is selected, so the card only goes when the + // click in the hole is handled as a click outside the polygon. + await click(-79.38, 43.65); + await card(false); + await strict.passed( + p, + 'fire-perimeters', + 'a click in the hole deselects the selected fire', + ); + await click(-79.435, 43.65); + await card(true); + console.log( + 'PASS', + kind, + 'select fire under GNSS, exclude hole, retain incident card', + ); } await p.evaluate(async () => { const { dataManager } = window.__godsEyeView; @@ -218,17 +411,79 @@ try { await dataManager.setEnabled('ais-live-vessels', true, { origin: 'user' }); window.qaVesselEntry = dataManager.layers.get('ais-live-vessels'); }); - await p.waitForFunction(() => window.qaVesselEntry.module.qaDrawn()?.chevrons[0]?.show); + await p.waitForFunction( + () => window.qaVesselEntry.module.qaDrawn()?.chevrons[0]?.show, + ); await new Promise((resolve) => setTimeout(resolve, 800)); + // Click-picking on the running engine: clear the selection with a click on + // empty water, then select the vessel with a click on its chevron. + const pickVessel = async () => { + // Empty ground: a point in the middle of the view where the engine + // picks nothing and nothing covers the canvas. + const empty = await p + .waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 220; y < 560; y += 11) + for (let x = 300; x < 720; x += 11) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [10, 0], + [-10, 0], + [0, 10], + [0, -10], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ) + .then((handle) => handle.jsonValue()); + await p.mouse.click(empty.x, empty.y); + await p.waitForFunction( + () => !window.qaVesselEntry.module.getSelectedInfo(), + ); + await new Promise((resolve) => setTimeout(resolve, 600)); + await click(-79.38, 43.65); + await p.waitForFunction( + () => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789', + { timeout: 15000 }, + ); + await strict.passed( + p, + 'ais-live-vessels', + 'a click on the chevron selects the vessel', + ); + }; await click(-79.38, 43.65); - await p.waitForFunction(() => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789'); + await p.waitForFunction( + () => window.qaVesselEntry.module.getSelectedInfo()?.mmsi === '123456789', + ); + await strict.passed( + p, + 'ais-live-vessels', + 'a click on the chevron selects the vessel', + ); + let histories = await p.evaluate(() => window.qaVesselHistories); for (const kind of ['navara', 'cesium', 'navara']) { - if (await p.evaluate(() => window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine', kind); + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await strict.pick(p, kind); await ready(kind); } const sample = await p.evaluate(() => { - const entry = window.__godsEyeView.dataManager.layers.get('ais-live-vessels'); + const entry = + window.__godsEyeView.dataManager.layers.get('ais-live-vessels'); const drawn = entry.module.qaDrawn(); return { retained: entry === window.qaVesselEntry, @@ -238,13 +493,31 @@ try { histories: window.qaVesselHistories, }; }); - assert.deepEqual(sample, { - retained: true, selected: '123456789', chevron: '123456789', - card: 'vessel:123456789', histories: 1, - }, kind); - console.log('PASS', kind, 'retained selected AIS vessel, chevron, card and single track request'); + assert.deepEqual( + sample, + { + retained: true, + selected: '123456789', + chevron: '123456789', + card: 'vessel:123456789', + histories, + }, + kind, + ); + await strict.passed( + p, + 'ais-live-vessels', + 'same entry; selection, chevron and card kept; no new track request', + ); + if (kind === 'cesium' || histories === 1) { + await pickVessel(); + histories = await p.evaluate(() => window.qaVesselHistories); + } + console.log( + 'PASS', + kind, + 'retained selected AIS vessel, chevron, card and single track request', + ); } assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-infrastructure-engine-switch.mjs b/scripts/qa-infrastructure-engine-switch.mjs index 758a531b6..e29f08d65 100644 --- a/scripts/qa-infrastructure-engine-switch.mjs +++ b/scripts/qa-infrastructure-engine-switch.mjs @@ -1,26 +1,33 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { readFileSync } from 'node:fs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=41.5340&lon=-90.5070&alt=20000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// Every geometry of the bundled files (a Multi* feature counts each part), +// read from the files this checkout builds. +const geometries = (name) => + readFileSync( + new URL(`../src/data/local_data/${name}/${name}.geojsonl`, import.meta.url), + 'utf8', + ) + .split('\n') + .filter((line) => line.trim()) + .reduce((sum, line) => { + const { type, coordinates } = JSON.parse(line).geometry; + return sum + (type.startsWith('Multi') ? coordinates.length : 1); + }, 0); +const DATACENTERS = geometries('datacenters'); +const DAMS = geometries('dams'); +const strict = createStrict({ + origin: base.origin, + layers: ['local-dams', 'local-datacenters'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -39,12 +46,16 @@ try { if (detail?.layerId?.startsWith('local-')) window.qaSelections.push(detail.id); }); + window.qaCleared = 0; + window.addEventListener('gev:entity-selection-cleared', () => { + window.qaCleared++; + }); }); await p.goto(base.href); const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -77,8 +88,10 @@ try { } return records; }, ids); - assert.ok( - dataset[0].count >= 4351 && dataset[1].count >= 704, + // The bundled files are fixed: every record is loaded. + assert.deepEqual( + dataset.map((d) => d.count), + [DATACENTERS, DAMS], JSON.stringify(dataset), ); const record = await p.evaluate(() => @@ -103,13 +116,75 @@ try { { timeout: 15000 }, ); }; + // A dam in view, clicked where its record is. + const dam = await p.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('local-dams') + .module.getAnalystRecords(5000) + .find((r) => r.name === 'Arsenal Power Dam'), + ); + assert.ok(dam); + const clickDam = async () => { + const screen = await p.evaluate( + (world) => { + const { engine } = window.__godsEyeView; + const at = engine.projector.worldToWindow(world, {}); + const box = engine.host.canvas.getBoundingClientRect(); + window.qaSelections = []; + return { x: box.left + at.x, y: box.top + at.y }; + }, + fromDegrees(dam.lon, dam.lat), + ); + await p.mouse.click(screen.x, screen.y); + await p + .waitForFunction( + () => window.qaSelections.at(-1) === 'local-dams:-12841971', + { timeout: 15000 }, + ) + .catch(async (error) => { + console.log( + 'dam click', + JSON.stringify(screen), + JSON.stringify( + await p.evaluate(({ x, y }) => { + const { engine } = window.__godsEyeView; + const box = engine.host.canvas.getBoundingClientRect(); + const id = engine.picker.pick({ + x: x - box.left, + y: y - box.top, + })?.id; + return { + selections: window.qaSelections, + cleared: window.qaCleared, + pick: String(id?.id ?? id), + hit: document.elementFromPoint(x, y)?.tagName, + }; + }, screen), + ), + ); + throw error; + }); + }; + const start = fromDegrees(-90.507, 41.534, 20000); + const home = async () => { + await p.evaluate((destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.cancelFlight(); + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-home'); + }, start); + await new Promise((resolve) => setTimeout(resolve, 2000)); + }; const counts = { ...reads }; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { await p.evaluate(() => { window.qaSelections = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction( () => window.qaSelections.includes('local-datacenters:342664739'), @@ -142,16 +217,38 @@ try { assert.equal(layer.count, dataset[i].count); assert.equal(layer.name, dataset[i].name); } - if ( - kind === 'navara' && - !(await p.evaluate(() => window.qaSelections.length)) - ) - await clickSite(); assert.deepEqual( reads, counts, 'Do not reload bundled data on scene handoff', ); + await strict.passed( + p, + ids, + 'same model, every bundled record, bounded LOD stems, no file read again', + result.map((r) => r.count), + ); + if ( + kind !== 'navara' || + (await p.evaluate(() => window.qaSelections.length)) + ) + await strict.passed( + p, + 'local-datacenters', + 'the clicked datacenter stays selected across the switch', + ); + // Click-picking on this engine: a dam, then the datacenter again, each + // from the starting view (a selection flies the camera to it). + await home(); + await clickDam(); + await strict.passed(p, 'local-dams', 'a click on a dam selects it'); + await home(); + await clickSite(); + await strict.passed( + p, + 'local-datacenters', + 'a click on the datacenter selects it', + ); } assert.equal(counts.datacenters, 1); assert.equal(counts.dams, 1); @@ -159,6 +256,4 @@ try { console.log( 'PASS complete bundled datasets, bounded stems, retained selection and one read per file', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-layer-handoff.mjs b/scripts/qa-layer-handoff.mjs index 4cd412fe0..865a1ee8a 100644 --- a/scripts/qa-layer-handoff.mjs +++ b/scripts/qa-layer-handoff.mjs @@ -1,15 +1,7 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const url = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const url = localPreview(process.argv[2]); const military = process.argv.includes('--military'); const layerId = military ? 'military' : 'flights'; const aircraftId = military ? 'aaa052' : '71be26'; @@ -17,381 +9,520 @@ url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=40.2539&lon=-125.5779&alt=21130&heading=360&pitch=-35&map=sentinel2' + (military ? '&l=m&lo=f.e.1_f.u.aaa052' : '&l=f&lo=f.e.1_f.t.71be26'); -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - const errors = []; - page.on('pageerror', (e) => errors.push(e.message)); - await page.setViewport({ width: 1280, height: 720 }); - await page.evaluateOnNewDocument( - (layerId, aircraftId) => { - window.__qaAircraft = { layerId, aircraftId }; - const fetch = window.fetch.bind(window); - window.__flightPolls = 0; - const interval = window.setInterval.bind(window); - window.setInterval = function (callback, delay, ...args) { - if (typeof callback !== 'function') - return interval(callback, delay, ...args); - const id = interval( - function (...values) { - if (window.__handoff?.timer === id) window.__handoff.ticks++; - return callback.apply(this, values); - }, - delay, - ...args, - ); - return id; - }; - window.fetch = (input, options) => { - const path = new URL(input.url ?? input, document.baseURI).pathname; - const now = Math.floor(Date.now() / 1000); - if (path === '/api/flights') { - if (layerId === 'flights') window.__flightPolls++; - return Promise.resolve( - Response.json({ - time: now, - states: [ - [ - '71be26', - 'GEV123', - 'QA', - now, - now, - -125.5779, - 40.2539, - 9700, - false, - 200, - 45, - 0, - null, - 9700, - null, - false, - 0, +// The Command Dock, and the Globe Engine picker in it, is hidden in cockpit +// mode: no person can pick an engine there, so the cockpit switches below go +// through the select element; the real picker is used once the cockpit is +// left. +const HIDDEN_PICKER = 'the Globe Engine picker is hidden in cockpit mode'; +const strict = createStrict({ origin: url.origin, layers: [layerId] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', (e) => errors.push(e.message)); + await page.setViewport({ width: 1280, height: 720 }); + await page.evaluateOnNewDocument( + (layerId, aircraftId) => { + window.__qaAircraft = { layerId, aircraftId }; + const fetch = window.fetch.bind(window); + window.__flightPolls = 0; + const interval = window.setInterval.bind(window); + window.setInterval = function (callback, delay, ...args) { + if (typeof callback !== 'function') + return interval(callback, delay, ...args); + const id = interval( + function (...values) { + if (window.__handoff?.timer === id) window.__handoff.ticks++; + return callback.apply(this, values); + }, + delay, + ...args, + ); + return id; + }; + window.fetch = (input, options) => { + const path = new URL(input.url ?? input, document.baseURI).pathname; + const now = Math.floor(Date.now() / 1000); + if (path === '/api/flights') { + if (layerId === 'flights') window.__flightPolls++; + return Promise.resolve( + Response.json({ + time: now, + states: [ + [ + '71be26', + 'GEV123', + 'QA', + now, + now, + -125.5779, + 40.2539, + 9700, + false, + 200, + 45, + 0, + null, + 9700, + null, + false, + 0, + ], ], - ], - }), - ); - } - if (path === '/api/military') { - if (layerId === 'military') window.__flightPolls++; - return Promise.resolve( - Response.json({ - now: Date.now(), - ac: [ - { - hex: 'aaa052', - flight: 'GEVMIL', - lon: -125.5779, - lat: 40.2539, - alt_geom: 31825, - alt_baro: 31825, - gs: 389, - track: 45, - seen: 0, - seen_pos: 0, - dbFlags: 1, - type: 'adsb_icao', - }, - ], - }), - ); - } - if (path === '/api/flights/track') - return Promise.resolve(Response.json({ path: [] })); - if (path.startsWith('/api/flights/') || path.startsWith('/api/openai/')) - return Promise.resolve(Response.json({})); - return fetch(input, options); - }; - }, - layerId, - aircraftId, - ); - await page.goto(url.href, { waitUntil: 'domcontentloaded' }); - const ready = (kind) => - page.waitForFunction( - (engine) => - window.__godsEyeView?.engine.kind === engine && - !document.querySelector('#globe-engine')?.disabled && - document.querySelector('#loading-screen.hidden') && - !document.querySelector('.engine-switch-frame'), - { timeout: 90000 }, - kind, + }), + ); + } + if (path === '/api/military') { + if (layerId === 'military') window.__flightPolls++; + return Promise.resolve( + Response.json({ + now: Date.now(), + ac: [ + { + hex: 'aaa052', + flight: 'GEVMIL', + lon: -125.5779, + lat: 40.2539, + alt_geom: 31825, + alt_baro: 31825, + gs: 389, + track: 45, + seen: 0, + seen_pos: 0, + dbFlags: 1, + type: 'adsb_icao', + }, + ], + }), + ); + } + if (path === '/api/flights/track') + return Promise.resolve(Response.json({ path: [] })); + if (path.startsWith('/api/flights/')) + return Promise.resolve(Response.json({})); + return fetch(input, options); + }; + }, + layerId, + aircraftId, ); - await ready('navara'); - await page.evaluate( - () => window.__godsEyeView.styleManager.initialRestorePromise, - ); - await page.waitForSelector('#cockpit-entry', { visible: true }); - await page.click('#cockpit-entry'); - await page.waitForFunction( - () => window.__godsEyeView.styleManager.getCockpitState().active, - ); - await page.evaluate(() => { - const { dataManager, styleManager, tileset } = window.__godsEyeView; - if (window.__GOOGLE_MAPS_API_KEY__ || tileset) - throw Error('Keyless preview required'); - styleManager.cockpitView.setVisionMode('thermal'); - const entry = dataManager.layers.get(window.__qaAircraft.layerId); - window.__handoff = { - dataManager, - entry, - row: document.querySelector( - `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, - ), - timer: entry.intervalId, - ticks: 0, - model: entry.module, - pollCount: window.__flightPolls, - timestamp: entry.module.getStats().lastUpdate, - }; - }); - // Refuse the first Cesium canvas. The application must roll back with the - // original aircraft still in the cockpit, then allow an ordinary retry. - await page.evaluate(() => { - const original = HTMLCanvasElement.prototype.getContext; - HTMLCanvasElement.prototype.getContext = function (type, ...args) { - if (String(type).includes('webgl')) { - window.__refusedContext = true; - throw new Error('QA renderer startup failure'); - } - return original.call(this, type, ...args); - }; - window.__restoreContext = () => { - HTMLCanvasElement.prototype.getContext = original; - }; - }); - const historyLength = await page.evaluate(() => history.length); - await page.select('#globe-engine', 'cesium'); - await page.waitForFunction(() => window.__refusedContext === true); - await ready('navara'); - // A failed switch leaves the session usable on the engine that worked and - // adds no history entry. - const survived = (kind) => - page.evaluate( - (kind, length) => { + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + const ready = (kind) => + page.waitForFunction( + (engine) => + window.__godsEyeView?.engine?.kind === engine && + !document.querySelector('#globe-engine')?.disabled && + document.querySelector('#loading-screen.hidden') && + !document.querySelector('.engine-switch-frame'), + { timeout: 90000 }, + kind, + ); + await ready('navara'); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); + await page.waitForSelector('#cockpit-entry', { visible: true }); + await page.click('#cockpit-entry'); + await page.waitForFunction( + () => window.__godsEyeView.styleManager.getCockpitState().active, + ); + await page.evaluate(() => { + const { dataManager, styleManager, tileset } = window.__godsEyeView; + if (window.__GOOGLE_MAPS_API_KEY__ || tileset) + throw Error('Keyless preview required'); + styleManager.cockpitView.setVisionMode('thermal'); + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + window.__handoff = { + dataManager, + entry, + row: document.querySelector( + `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, + ), + timer: entry.intervalId, + ticks: 0, + model: entry.module, + pollCount: window.__flightPolls, + timestamp: entry.module.getStats().lastUpdate, + }; + }); + // Refuse the first Cesium canvas. The application must roll back with the + // original aircraft still in the cockpit, then allow an ordinary retry. + await page.evaluate(() => { + const original = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + if (String(type).includes('webgl')) { + window.__refusedContext = true; + throw new Error('QA renderer startup failure'); + } + return original.call(this, type, ...args); + }; + window.__restoreContext = () => { + HTMLCanvasElement.prototype.getContext = original; + }; + }); + const historyLength = await page.evaluate(() => history.length); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await page.waitForFunction(() => window.__refusedContext === true); + await ready('navara'); + // A failed switch leaves the session usable on the engine that worked and + // adds no history entry. + const survived = (kind) => + page.evaluate( + (kind, length) => { + const { dataManager, engine, styleManager } = window.__godsEyeView; + return ( + engine.kind === kind && + new URL(location.href).searchParams.get('engine') === kind && + history.length === length && + dataManager === window.__handoff.dataManager && + dataManager.layers.get(window.__qaAircraft.layerId) === + window.__handoff.entry && + styleManager.getCockpitState().active && + styleManager.getCockpitState().visionMode === 'thermal' && + styleManager.getAircraftTrackingTarget()?.id === + window.__qaAircraft.aircraftId + ); + }, + kind, + historyLength, + ); + await page.evaluate(() => window.__restoreContext()); + await strict.check( + page, + layerId, + 'cold Cesium failure rolls back to Navara: same session, cockpit, aircraft, no history entry', + await survived('navara'), + ); + console.log( + 'PASS cold Cesium failure rolled back to Navara with the same flight, cockpit and history', + ); + + for (const kind of ['cesium', 'navara']) { + await strict.selectEngine(page, kind, HIDDEN_PICKER); + await ready(kind); + const result = await page.evaluate(() => { const { dataManager, engine, styleManager } = window.__godsEyeView; + const before = window.__handoff; + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + const cockpit = styleManager.getCockpitState(); + return { + sameOwner: + before.dataManager === dataManager && + before.entry === entry && + before.model === entry.module, + sameRow: + Boolean(before.row) && + before.row === + document.querySelector( + `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, + ), + sameTimer: + before.timer === entry.intervalId && entry.intervalId != null, + sourcePolls: window.__flightPolls - before.pollCount, + timerTicks: before.ticks, + stamp: entry.module.getStats().lastUpdate, + initialStamp: before.timestamp, + id: cockpit.subject?.id, + active: cockpit.active, + mode: cockpit.visionMode, + hud: !document.querySelector('#cockpit-hud').hidden, + drawn: entry.module + .qaDrawn() + .some((row) => row.id === window.__qaAircraft.aircraftId), + height: engine.camera.height(), + }; + }); + assert.ok( + result.sameOwner && + result.sameTimer && + result.sameRow && + result.active && + result.hud && + result.drawn, + JSON.stringify(result), + ); + assert.equal(result.id, aircraftId); + assert.equal(result.mode, 'thermal'); + assert.ok( + result.sourcePolls <= result.timerTicks, + 'switch must not add a source request', + ); + if (result.timerTicks === 0) { + assert.equal(result.stamp, result.initialStamp); + } + await strict.passed( + page, + layerId, + 'same manager entry, model, row, poll timer, cockpit, thermal mode, HUD and drawn aircraft', + result, + ); + // Polling goes on after the switch: the same timer ticks and each tick + // reads the source once (no tick may be skipped, none added). + const polls = await page.evaluate(async (layerId) => { + const entry = window.__godsEyeView.dataManager.layers.get(layerId); + const every = + Number(entry.module.refreshInterval) || + Number(entry.module.updateInterval); + const start = { + polls: window.__flightPolls, + ticks: window.__handoff.ticks, + stamp: entry.module.getStats().lastUpdate, + }; + const deadline = Date.now() + 2.5 * every + 5000; + while ( + window.__handoff.ticks - start.ticks < 2 && + Date.now() < deadline + ) + await new Promise((resolve) => setTimeout(resolve, 250)); + // Let the last tick's request land. + await new Promise((resolve) => setTimeout(resolve, 1500)); + return { + every, + ticks: window.__handoff.ticks - start.ticks, + polls: window.__flightPolls - start.polls, + updated: entry.module.getStats().lastUpdate !== start.stamp, + sameTimer: entry.intervalId === window.__handoff.timer, + }; + }, layerId); + await strict.check( + page, + layerId, + 'polling continues after the switch: two timer ticks, one source read each, fresh data', + polls.every > 0 && + polls.ticks >= 2 && + polls.polls === polls.ticks && + polls.updated && + polls.sameTimer, + polls, + ); + await page.waitForFunction( + () => + Math.abs(window.__godsEyeView.engine.camera.height() - 9700) < 1000, + ); + console.log( + `PASS ${kind} cockpit, aircraft, HUD, manager and timer (${result.sourcePolls} polls since entry)`, + ); + } + // The other direction, warm: the parked Navara renderer fails to resume on + // the way back from Cesium. The session rolls back to Cesium, then an + // ordinary retry starts Navara cold. + await page.evaluate(() => { + window.__parkedNavara = window.__godsEyeView.engine; + }); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await ready('cesium'); + const lengthOnCesium = await page.evaluate(() => { + const request = window.__parkedNavara.render.request; + window.__parkedNavara.render.request = function (...args) { + window.__parkedNavara.render.request = request; + window.__refusedResume = true; + throw new Error('QA parked renderer failure'); + }; + return history.length; + }); + await strict.selectEngine(page, 'navara', HIDDEN_PICKER); + await page.waitForFunction(() => window.__refusedResume === true); + await ready('cesium'); + assert.ok( + (await page.evaluate( + (length) => history.length === length, + lengthOnCesium, + )) && + (await page.evaluate(() => { + const { styleManager } = window.__godsEyeView; + return ( + styleManager.getCockpitState().active && + styleManager.getAircraftTrackingTarget()?.id === + window.__qaAircraft.aircraftId && + new URL(location.href).searchParams.get('engine') === 'cesium' + ); + })), + 'a failed warm Navara resume must leave Cesium usable without a history entry', + ); + await strict.passed( + page, + layerId, + 'warm Navara failure rolls back to Cesium: same aircraft and cockpit, no history entry', + ); + console.log( + 'PASS warm Navara failure rolled back to Cesium with the same flight and cockpit', + ); + await page.evaluate(() => { + window.__parkedCesium = window.__godsEyeView.engine; + }); + await strict.selectEngine(page, 'navara', HIDDEN_PICKER); + await ready('navara'); + assert.ok( + await page.evaluate(() => { + const { engine, styleManager } = window.__godsEyeView; return ( - engine.kind === kind && - new URL(location.href).searchParams.get('engine') === kind && - history.length === length && - dataManager === window.__handoff.dataManager && - dataManager.layers.get(window.__qaAircraft.layerId) === - window.__handoff.entry && + engine !== window.__parkedNavara && styleManager.getCockpitState().active && - styleManager.getCockpitState().visionMode === 'thermal' && styleManager.getAircraftTrackingTarget()?.id === window.__qaAircraft.aircraftId ); - }, - kind, - historyLength, + }), + 'the retry starts a new Navara renderer on the same session', ); - await page.evaluate(() => window.__restoreContext()); - assert.ok( - await survived('navara'), - 'a failed scene must leave the live session usable', - ); - console.log( - 'PASS cold Cesium failure rolled back to Navara with the same flight, cockpit and history', - ); + console.log('PASS retry after the warm failure started Navara cold'); - for (const kind of ['cesium', 'navara']) { - await page.select('#globe-engine', kind); - await ready(kind); - const result = await page.evaluate(() => { - const { dataManager, engine, styleManager } = window.__godsEyeView; - const before = window.__handoff; - const entry = dataManager.layers.get(window.__qaAircraft.layerId); - const cockpit = styleManager.getCockpitState(); - return { - sameOwner: - before.dataManager === dataManager && - before.entry === entry && - before.model === entry.module, - sameRow: - Boolean(before.row) && - before.row === - document.querySelector( - `[data-layer-id="${window.__qaAircraft.layerId}"] .data-toggle-btn`, - ), - sameTimer: - before.timer === entry.intervalId && entry.intervalId != null, - sourcePolls: window.__flightPolls - before.pollCount, - timerTicks: before.ticks, - stamp: entry.module.getStats().lastUpdate, - initialStamp: before.timestamp, - id: cockpit.subject?.id, - active: cockpit.active, - mode: cockpit.visionMode, - hud: !document.querySelector('#cockpit-hud').hidden, - drawn: entry.module - .qaDrawn() - .some((row) => row.id === window.__qaAircraft.aircraftId), - height: engine.camera.height(), - }; + // Both engines fail: the warm Cesium resume and the warm Navara recovery. + // No engine runs, but the error screen offers the engine picker, and a + // retry starts Cesium cold on the same session without a reload. + const lengthBeforeDoubleFailure = await page.evaluate(() => { + window.__failResume = true; + for (const engine of [ + window.__godsEyeView.engine, + window.__parkedCesium, + ]) { + const request = engine.render.request; + engine.render.request = function (reason, ...args) { + if (reason === 'engine-resume' && window.__failResume) { + window.__refusedResumes = (window.__refusedResumes ?? 0) + 1; + throw new Error('QA parked renderer failure'); + } + return request.call(this, reason, ...args); + }; + } + window.__startDocument = document; + return history.length; }); + await strict.selectEngine(page, 'cesium', HIDDEN_PICKER); + await page + .waitForSelector('#loading-screen .engine-switch-retry button', { + visible: true, + timeout: 90000, + }) + .catch(async (error) => { + console.log( + 'no retry picker', + JSON.stringify( + await page.evaluate(() => ({ + refused: window.__refusedResumes, + engine: window.__godsEyeView?.engine?.kind ?? null, + retry: !!document.querySelector('.engine-switch-retry'), + loading: document.querySelector('#loading-screen')?.className, + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + address: location.search, + })), + ), + ); + throw error; + }); assert.ok( - result.sameOwner && - result.sameTimer && - result.sameRow && - result.active && - result.hud && - result.drawn, - JSON.stringify(result), + await page.evaluate( + (length) => + window.__refusedResumes === 2 && + // No globe runs (the page's debug object may stay, without one). + !window.__godsEyeView?.engine && + history.length === length && + !document + .querySelector('#loading-screen') + .classList.contains('hidden') && + /QA parked renderer failure/.test( + document.querySelector('#loading-screen .loader-status') + .textContent, + ), + lengthBeforeDoubleFailure, + ), + 'a failed recovery shows its error and the engine picker ' + + JSON.stringify( + await page.evaluate(() => ({ + refused: window.__refusedResumes, + app: !!window.__godsEyeView, + engine: window.__godsEyeView?.engine?.kind ?? null, + history: history.length, + loading: document.querySelector('#loading-screen')?.className, + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + })), + ), ); - assert.equal(result.id, aircraftId); - assert.equal(result.mode, 'thermal'); + await page.evaluate(() => { + window.__failResume = false; + }); + await page.click( + '#loading-screen .engine-switch-retry [data-engine="cesium"]', + ); + await ready('cesium'); assert.ok( - result.sourcePolls <= result.timerTicks, - 'switch must not add a source request', - ); - if (result.timerTicks === 0) { - assert.equal(result.stamp, result.initialStamp); - } - await page.waitForFunction( - () => Math.abs(window.__godsEyeView.engine.camera.height() - 9700) < 1000, - ); - console.log( - `PASS ${kind} cockpit, aircraft, HUD, manager and timer (${result.sourcePolls} polls since entry)`, - ); - } - // The other direction, warm: the parked Navara renderer fails to resume on - // the way back from Cesium. The session rolls back to Cesium, then an - // ordinary retry starts Navara cold. - await page.evaluate(() => { - window.__parkedNavara = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', 'cesium'); - await ready('cesium'); - const lengthOnCesium = await page.evaluate(() => { - const request = window.__parkedNavara.render.request; - window.__parkedNavara.render.request = function (...args) { - window.__parkedNavara.render.request = request; - window.__refusedResume = true; - throw new Error('QA parked renderer failure'); - }; - return history.length; - }); - await page.select('#globe-engine', 'navara'); - await page.waitForFunction(() => window.__refusedResume === true); - await ready('cesium'); - assert.ok( - (await page.evaluate( - (length) => history.length === length, - lengthOnCesium, - )) && - (await page.evaluate(() => { - const { styleManager } = window.__godsEyeView; + await page.evaluate(() => { + const { dataManager, styleManager } = window.__godsEyeView; return ( + window.__startDocument === document && + !document.querySelector('.engine-switch-retry') && + dataManager === window.__handoff.dataManager && + dataManager.layers.get(window.__qaAircraft.layerId) === + window.__handoff.entry && styleManager.getCockpitState().active && styleManager.getAircraftTrackingTarget()?.id === window.__qaAircraft.aircraftId && new URL(location.href).searchParams.get('engine') === 'cesium' ); - })), - 'a failed warm Navara resume must leave Cesium usable without a history entry', - ); - console.log( - 'PASS warm Navara failure rolled back to Cesium with the same flight and cockpit', - ); - await page.evaluate(() => { - window.__parkedCesium = window.__godsEyeView.engine; - }); - await page.select('#globe-engine', 'navara'); - await ready('navara'); - assert.ok( + }), + 'the retry restores the same session on Cesium', + ); + await strict.passed( + page, + layerId, + 'after both engines fail, the retry restores the same session', + ); + console.log( + 'PASS failed recovery offered the picker; the retry restored the session on Cesium', + ); + await page.evaluate(() => + window.__godsEyeView.styleManager.controlCockpit('exit'), + ); + await page.waitForFunction( + () => !window.__godsEyeView.styleManager.getCockpitState().active, + ); + // Out of the cockpit the dock shows the picker again: a switch with + // real input keeps the same entry and timer, the aircraft drawn, and the + // source polled on its timer. await page.evaluate(() => { - const { engine, styleManager } = window.__godsEyeView; - return ( - engine !== window.__parkedNavara && - styleManager.getCockpitState().active && - styleManager.getAircraftTrackingTarget()?.id === - window.__qaAircraft.aircraftId - ); - }), - 'the retry starts a new Navara renderer on the same session', - ); - console.log('PASS retry after the warm failure started Navara cold'); - - // Both engines fail: the warm Cesium resume and the warm Navara recovery. - // No engine runs, but the error screen offers the engine picker, and a - // retry starts Cesium cold on the same session without a reload. - const lengthBeforeDoubleFailure = await page.evaluate(() => { - window.__failResume = true; - for (const engine of [window.__godsEyeView.engine, window.__parkedCesium]) { - const request = engine.render.request; - engine.render.request = function (reason, ...args) { - if (reason === 'engine-resume' && window.__failResume) { - window.__refusedResumes = (window.__refusedResumes ?? 0) + 1; - throw new Error('QA parked renderer failure'); - } - return request.call(this, reason, ...args); + window.__handoff.ticks = 0; + window.__handoff.pollCount = window.__flightPolls; + window.__handoff.timer = window.__godsEyeView.dataManager.layers.get( + window.__qaAircraft.layerId, + ).intervalId; + }); + await strict.pick(page, 'navara'); + await ready('navara'); + const after = await page.evaluate(async () => { + const { dataManager } = window.__godsEyeView; + const entry = dataManager.layers.get(window.__qaAircraft.layerId); + const every = + Number(entry.module.refreshInterval) || + Number(entry.module.updateInterval); + const deadline = Date.now() + 2.5 * every + 5000; + while (window.__handoff.ticks < 1 && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 250)); + await new Promise((resolve) => setTimeout(resolve, 1500)); + return { + same: + dataManager === window.__handoff.dataManager && + entry === window.__handoff.entry, + timer: entry.intervalId === window.__handoff.timer, + drawn: entry.module + .qaDrawn() + .some((row) => row.id === window.__qaAircraft.aircraftId), + ticks: window.__handoff.ticks, + polls: window.__flightPolls - window.__handoff.pollCount, }; - } - window.__startDocument = document; - return history.length; - }); - await page.select('#globe-engine', 'cesium'); - await page.waitForSelector('#loading-screen .engine-switch-retry button', { - visible: true, - timeout: 90000, - }); - assert.ok( - await page.evaluate( - (length) => - window.__refusedResumes === 2 && - !window.__godsEyeView && - history.length === length && - !document - .querySelector('#loading-screen') - .classList.contains('hidden') && - /QA parked renderer failure/.test( - document.querySelector('#loading-screen .loader-status').textContent, - ), - lengthBeforeDoubleFailure, - ), - 'a failed recovery shows its error and the engine picker', - ); - await page.evaluate(() => { - window.__failResume = false; - }); - await page.click( - '#loading-screen .engine-switch-retry [data-engine="cesium"]', - ); - await ready('cesium'); - assert.ok( - await page.evaluate(() => { - const { dataManager, styleManager } = window.__godsEyeView; - return ( - window.__startDocument === document && - !document.querySelector('.engine-switch-retry') && - dataManager === window.__handoff.dataManager && - dataManager.layers.get(window.__qaAircraft.layerId) === - window.__handoff.entry && - styleManager.getCockpitState().active && - styleManager.getAircraftTrackingTarget()?.id === - window.__qaAircraft.aircraftId && - new URL(location.href).searchParams.get('engine') === 'cesium' - ); - }), - 'the retry restores the same session on Cesium', - ); - console.log( - 'PASS failed recovery offered the picker; the retry restored the session on Cesium', - ); - await page.evaluate(() => - window.__godsEyeView.styleManager.controlCockpit('exit'), - ); - await page.waitForFunction( - () => !window.__godsEyeView.styleManager.getCockpitState().active, - ); - assert.deepEqual(errors, []); -} finally { - await browser.close(); -} + }); + await strict.check( + page, + layerId, + 'picker switch keeps the entry, timer and drawn aircraft; polling continues', + after.same && + after.timer && + after.drawn && + after.ticks >= 1 && + after.polls === after.ticks, + after, + ); + assert.deepEqual(errors, []); + }, +); diff --git a/scripts/qa-local-adsb-engine-switch.mjs b/scripts/qa-local-adsb-engine-switch.mjs index 329049e09..e5dbc1f9d 100644 --- a/scripts/qa-local-adsb-engine-switch.mjs +++ b/scripts/qa-local-adsb-engine-switch.mjs @@ -1,25 +1,13 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=41.5340&lon=-90.5070&alt=20000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +const strict = createStrict({ origin: base.origin, layers: ['local-adsb'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -32,6 +20,8 @@ try { const url = new URL(input?.url ?? input, location.href); if (url.pathname === '/api/local-receivers/aircraft') { const at = Date.now(); + // Each poll reports the aircraft a little further north. + const n = (window.qaAdsbPolls = (window.qaAdsbPolls ?? 0) + 1); return Promise.resolve( Response.json({ configured: true, @@ -40,14 +30,14 @@ try { { icao: 'abc123', callsign: 'LOCAL1', - lat: 41.534, + lat: 41.534 + 0.0002 * n, lon: -90.507, altitudeFt: 11000, groundSpeedKt: 180, trackDeg: 5, lastPositionAt: at, lastMessageAt: at, - messageCount: 12, + messageCount: 12 + n, band: '1090', source: 'decoder', }, @@ -66,7 +56,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -112,7 +102,7 @@ try { await p.evaluate(() => { window.qaSelections = []; }); - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); await p.waitForFunction(() => window.qaSelections.includes('local-adsb:abc123'), @@ -133,11 +123,34 @@ try { ]; }); assert.deepEqual(result, [true, true, true, true, 'abc123', 1, 0, 0]); + await strict.passed( + p, + 'local-adsb', + 'same decoder, receiver and feeds, one feed start, selection republished', + ); + // Updates continue on the new engine: further polls move the aircraft. + const before = await p.evaluate(() => ({ + polls: window.qaAdsbPolls, + lat: window.qaAdsb.mod.getAnalystRecords()[0]?.lat, + stamp: window.qaAdsb.mod.getStats().lastUpdate, + })); + await p.waitForFunction( + (before) => + window.qaAdsbPolls >= before.polls + 2 && + window.qaAdsb.mod.getAnalystRecords()[0]?.lat > before.lat && + window.qaAdsb.mod.getStats().lastUpdate > before.stamp, + { timeout: 15000 }, + before, + ); + await strict.passed( + p, + 'local-adsb', + 'receiver polls continue and move the selected aircraft', + before, + ); } assert.deepEqual(errors, []); console.log( 'PASS retained local decoder, receiver, marker and selection in both scenes', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-map-policy.mjs b/scripts/qa-map-policy.mjs index 240e003f5..9fa1600d0 100644 --- a/scripts/qa-map-policy.mjs +++ b/scripts/qa-map-policy.mjs @@ -13,3 +13,105 @@ export async function blockPaidQaMaps(page, { allowPaidMaps = false } = {}) { }); return session; } + +/** + * Free public tile hosts the strict acceptance lets a page ask for. The page + * never reaches them: the strict harness answers them from a disk cache + * (scripts/qa/tileCache.mjs), and Node fetches a missing tile once, politely. + * OpenStreetMap raster tiles, the keyless re:earth terrain, the EOX + * Sentinel-2 cloudless mosaic and Esri World Imagery (the keyless Cesium + * base imagery and the Navara startup imagery). + */ +export const STRICT_TILE_HOSTS = Object.freeze([ + 'tile.openstreetmap.org', + 'terrain.reearth.land', + 'tiles.maps.eox.at', + 'services.arcgisonline.com', +]); + +/** + * Requests every keyless page makes on its own, answered by the strict + * harness exactly as the keyless preview would (or with an empty stylesheet + * for Google Fonts), so they reach neither the preview's paid routes nor + * Google. Any other request under a paid route or host stays blocked. + * Keys are URLPattern strings relative to the preview origin or absolute. + */ +export const KEYLESS_PROBES = Object.freeze([ + { + pattern: '/api/openai/hud-summary/status', + method: 'GET', + status: 200, + contentType: 'application/json', + body: '{"configured":false}', + }, + { pattern: '/api/google/tiles-token', method: 'GET', status: 204 }, + { pattern: '/api/realtime/debug-log', method: 'POST', status: 204 }, + { + pattern: 'https://fonts.googleapis.com/css2*', + method: 'GET', + status: 200, + contentType: 'text/css', + body: '/* Google Fonts withheld by the strict QA policy */', + }, +]); + +/** A probe's absolute URLPattern string. */ +export const probePattern = (origin, probe) => + probe.pattern.startsWith('/') + ? new URL(origin).origin + probe.pattern + : probe.pattern; + +/** Paid or keyed services: blocked first, whatever else a scenario allows. */ +export const PAID_HOST_PATTERNS = Object.freeze([ + '*://*.googleapis.com/*', + '*://googleapis.com/*', + '*://*.google.com/*', + '*://google.com/*', + '*://maps.google.*/*', + '*://*.gstatic.com/*', + '*://*.cesium.com/*', + '*://cesium.com/*', +]); + +/** Preview routes that spend a key (OpenAI, Realtime, Google proxies). */ +export const PAID_PREVIEW_PATHS = Object.freeze([ + '/api/openai/*', + '/api/realtime/*', + '/api/google/*', +]); + +/** + * The ordered Network.setBlockedURLs patterns of the strict policy (first + * match wins): the keyless probes the harness answers are let through to + * it, paid hosts and paid preview routes are blocked, then the + * preview origin, the scenario's fixture URLs and the free tile hosts are + * allowed, then everything else is blocked. + * + * @param {string} origin The preview origin, e.g. http://127.0.0.1:4208 + * @param {{fixtures?: string[], tileHosts?: readonly string[]}} [options] + * `fixtures` are URLPattern strings for requests the scenario answers. + */ +export function strictBlockPatterns( + origin, + { fixtures = [], tileHosts = STRICT_TILE_HOSTS } = {}, +) { + const preview = new URL(origin).origin; + return [ + ...KEYLESS_PROBES.map((probe) => ({ + urlPattern: probePattern(origin, probe), + block: false, + })), + ...PAID_HOST_PATTERNS.map((urlPattern) => ({ urlPattern, block: true })), + ...PAID_PREVIEW_PATHS.map((path) => ({ + urlPattern: preview + path, + block: true, + })), + { urlPattern: preview + '/*', block: false }, + ...fixtures.map((urlPattern) => ({ urlPattern, block: false })), + ...tileHosts.map((host) => ({ + urlPattern: `https://${host}/*`, + block: false, + })), + { urlPattern: '*://*:*/*', block: true }, + ]; +} diff --git a/scripts/qa-map-policy.test.mjs b/scripts/qa-map-policy.test.mjs index 647a7cca3..cc90402f1 100644 --- a/scripts/qa-map-policy.test.mjs +++ b/scripts/qa-map-policy.test.mjs @@ -1,6 +1,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { + blockPaidQaMaps, + KEYLESS_PROBES, + probePattern, + strictBlockPatterns, +} from './qa-map-policy.mjs'; test('routine QA blocks direct Google tiles, hosted Google assets and token fallback', async () => { const calls = []; @@ -25,3 +30,54 @@ test('a necessary 3D-specific check must explicitly opt out of the request block { allowPaidMaps: true }, ); }); + +test('the strict policy blocks paid hosts and routes first and everything unlisted last', () => { + const patterns = strictBlockPatterns('http://127.0.0.1:4208', { + fixtures: ['https://tiles.dontgetflocked.com/*'], + }); + const decide = (url) => + patterns.find(({ urlPattern }) => new URLPattern(urlPattern).test(url)) + ?.block; + for (const url of [ + 'https://tile.googleapis.com/v1/3dtiles/root.json', + 'https://maps.googleapis.com/maps/api/js', + 'https://fonts.gstatic.com/s/inter.woff2', + 'https://api.cesium.com/v1/assets/1/endpoint', + 'https://assets.ion.cesium.com/google-credit.png', + 'http://127.0.0.1:4208/api/openai/hud-summary', + 'http://127.0.0.1:4208/api/realtime/token', + 'http://127.0.0.1:4208/api/google/nearby-places?lat=1', + 'https://example.com/anything', + 'wss://example.com/socket', + ]) + assert.equal(decide(url), true, url); + for (const url of [ + 'http://127.0.0.1:4208/assets/index.js', + 'http://127.0.0.1:4208/api/flights', + 'https://tiles.dontgetflocked.com/cameras-us-hourly/1/2/3.mvt', + 'https://tile.openstreetmap.org/1/2/3.png', + 'https://terrain.reearth.land/cesium-mesh/layer.json', + 'https://tiles.maps.eox.at/wmts/1.0.0/s2/1/2/3.jpg', + ]) + assert.equal(decide(url), false, url); +}); + +test('the keyless probes are let through only to their exact routes', () => { + const patterns = strictBlockPatterns('http://127.0.0.1:4208'); + const decide = (url) => + patterns.find(({ urlPattern }) => new URLPattern(urlPattern).test(url)) + ?.block; + for (const probe of KEYLESS_PROBES) { + const url = probePattern('http://127.0.0.1:4208', probe).replace( + /\*$/, + '?family=Inter', + ); + assert.equal(decide(url), false, url); + } + assert.equal( + decide('http://127.0.0.1:4208/api/openai/hud-summary'), + true, + 'the paid summary itself stays blocked', + ); + assert.equal(decide('https://fonts.googleapis.com/icon?family=x'), true); +}); diff --git a/scripts/qa-nepal-event-switch.mjs b/scripts/qa-nepal-event-switch.mjs index 5ae7adf64..bdcdb2aa9 100644 --- a/scripts/qa-nepal-event-switch.mjs +++ b/scripts/qa-nepal-event-switch.mjs @@ -1,26 +1,45 @@ import sharp from 'sharp'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=28.2110&lon=85.3380&alt=9000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// The media panel embeds YouTube players with their thumbnails: a plain +// thumbnail and an inert player page stand in for them. +const thumbnail = await sharp({ + create: { width: 480, height: 360, channels: 3, background: '#40506a' }, +}) + .jpeg() + .toBuffer(); +const strict = createStrict({ + origin: base.origin, + layers: ['bhote-koshi-2026'], + routes: [ + { + pattern: 'https://i.ytimg.com/*', + respond: () => ({ contentType: 'image/jpeg', body: thumbnail }), + }, + { + // The player API script: an inert stub (the app keeps its own controls). + pattern: 'https://www.youtube.com/iframe_api*', + respond: () => ({ + contentType: 'text/javascript', + body: '/* QA: YouTube iframe API withheld */', + }), + }, + { + pattern: 'https://www.youtube-nocookie.com/*', + respond: () => ({ + contentType: 'text/html', + body: 'QA player', + }), + }, + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -45,7 +64,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -59,7 +78,8 @@ try { await ready('navara'); await p.evaluate(async () => { const { tileset, dataManager, sceneDirector } = window.__godsEyeView; - if (!sceneDirector?.listScenes().length) throw Error('No scenes available on Navara'); + if (!sceneDirector?.listScenes().length) + throw Error('No scenes available on Navara'); if (tileset || window.__GOOGLE_MAPS_API_KEY__) throw Error('Keyless preview required'); const mod = dataManager.layers.get('bhote-koshi-2026').module; @@ -82,7 +102,7 @@ try { }; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } const state = await p.evaluate(() => { @@ -107,7 +127,10 @@ try { ); window.qaEvent.setParams({ split: 0.75 }); }); - assert.ok(await p.evaluate(()=>Boolean(window.qaPlayer)), 'persistent embedded media root'); + assert.ok( + await p.evaluate(() => Boolean(window.qaPlayer)), + 'persistent embedded media root', + ); const before = await sample(); await p.evaluate(() => window.qaEvent.setParams({ split: 0.27 })); const after = await sample(); @@ -115,6 +138,12 @@ try { before.reduce((n, v, i) => n + Math.abs(v - after[i]), 0) / before.length; console.log(kind, state.state, 'imagery difference', diff); assert.ok(diff > 6, 'divider changes the actual imagery in the viewport'); + await strict.passed( + p, + 'bhote-koshi-2026', + 'same model and media root; paused at 0.52 with split 0.27; the divider changes the imagery', + { diff }, + ); assert.equal( await p.evaluate( () => @@ -130,14 +159,25 @@ try { 1, JSON.stringify(assets), ); + await strict.passed( + p, + 'bhote-koshi-2026', + 'event file and image pair read once over three engine visits', + ); await p.evaluate(() => window.qaEvent.setParams({ autoPlay: true })); await p.waitForFunction( () => window.qaEvent.getPlaybackState().progress > 0.53, ); - await p.select('#globe-engine', 'cesium'); + await strict.pick(p, 'cesium'); await ready('cesium'); const live = await p.evaluate(() => window.qaEvent.getPlaybackState()); - assert.ok(live.playing && live.progress > 0.53); + await strict.check( + p, + 'bhote-koshi-2026', + 'playback keeps playing past 0.53 through the switch', + live.playing && live.progress > 0.53, + live, + ); await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('bhote-koshi-2026', false, { origin: 'user', @@ -147,6 +187,4 @@ try { console.log( 'PASS event timeline, paused and playing handoff, image pair, divider, player host and single asset acquisition', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-nepal-locator-switch.mjs b/scripts/qa-nepal-locator-switch.mjs index 6f3209fb2..30416b2e8 100644 --- a/scripts/qa-nepal-locator-switch.mjs +++ b/scripts/qa-nepal-locator-switch.mjs @@ -1,27 +1,18 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; import sharp from 'sharp'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=28.3326&lon=85.4848&alt=18000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: ['bhote-koshi-locator'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -36,7 +27,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -60,7 +51,7 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } const { same, stats, params, at } = await p.evaluate( @@ -114,9 +105,12 @@ try { added, at, ); - assert.ok( + await strict.check( + p, + 'bhote-koshi-locator', + 'same model; the flood path adds more than 50 cyan pixels to the same scene without it', added > 50, - 'visible river differs from the same scene without a path', + { added }, ); await p.evaluate(() => window.qaLocator.setParams({ presentation: 'bhote-koshi-path-overview' }), @@ -130,7 +124,21 @@ try { preview: true, }); }); - await new Promise((resolve) => setTimeout(resolve, 1000)); + await new Promise((resolve) => setTimeout(resolve, 2500)); + // The last of three quick presentation changes is the one that stays. + const last = await p.evaluate(() => ({ + params: window.qaLocator.getParams(), + stats: window.qaLocator.getStats(), + })); + await strict.check( + p, + 'bhote-koshi-locator', + 'the last of three quick presentation changes is the one applied', + last.params.presentation === 'bhote-koshi-trigger-record' && + last.stats.count === 1 && + !last.stats.error, + { presentation: last.params.presentation, count: last.stats.count }, + ); await p.evaluate(() => window.__godsEyeView.dataManager.setEnabled('bhote-koshi-locator', false, { origin: 'user', @@ -140,6 +148,4 @@ try { console.log( 'PASS Nepal locator flood path and presentation retained, trigger and places update; both engines', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-radio-engine-switch.mjs b/scripts/qa-radio-engine-switch.mjs index 9c035014d..427de7712 100644 --- a/scripts/qa-radio-engine-switch.mjs +++ b/scripts/qa-radio-engine-switch.mjs @@ -1,168 +1,91 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; import { stations, installRadioFixture } from './fixtures/w2/radio.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]); -const soak = process.argv.includes('--soak'); -const requested = process.argv.find((arg) => arg.startsWith('--swaps=')); -const swaps = requested ? Number(requested.slice(8)) : soak ? 50 : 2; -if (requested && (!soak || !Number.isInteger(swaps) || swaps % 2 || swaps < 4 || swaps > 1000)) - throw Error('Use --soak --swaps=N with an even count from 4 to 1000'); - -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Local keyless preview required'); +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]); +// The long soak and switch timings live in qa-strict-soak.mjs. url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=30.27&lon=-97.74&alt=12000&pitch=-90&map=sentinel2&l='; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { - const page = await browser.newPage(); - await blockPaidQaMaps(page); - const errors = []; - page.on('pageerror', (e) => errors.push(e.message)); - await installRadioFixture(page); - if (soak) await page.evaluateOnNewDocument(() => { - const fetch = window.fetch.bind(window); - window.qaQuakeReads = 0; - window.fetch = (input, init) => { - const url = new URL(input?.url ?? input, location.href); - if (url.hostname !== 'earthquake.usgs.gov') return fetch(input, init); - window.qaQuakeReads++; - return Promise.resolve(Response.json({ - type: 'FeatureCollection', - features: [{ type: 'Feature', id: 'qa-austin', properties: { mag: 4.1, time: Date.now(), place: 'Austin' }, - geometry: { type: 'Point', coordinates: [-97.74, 30.27, 8] } }], - })); - }; - }); - await page.goto(url.href); - await page.waitForFunction( - () => - window.__godsEyeView?.engine?.kind === 'navara' && - document.querySelector('#loading-screen.hidden'), - ); - await page.evaluate(async (id) => { - const { dataManager, styleManager, tileset } = window.__godsEyeView; - await styleManager.initialRestorePromise; - if (window.__GOOGLE_MAPS_API_KEY__ || tileset) - throw Error('Expected keyless preview'); - window.qaLayer = dataManager.layers.get('radio').module; - await dataManager.setEnabled('radio', true, { origin: 'user' }); - }, stations[0].id); - await page.waitForFunction(() => window.qaLayer?.getStats().count === 750); - await page.evaluate((id) => { - window.qaLayer.selectStation(id, { - autoplay: true, - focus: false, - origin: 'user', - }); - window.qaLayer.setRadioVolume(0.36); - }, stations[10].id); - await page.waitForFunction( - () => - window.__qaRadio.plays.length === 1 && - window.qaLayer.getUIState().audioState === 'playing', - ); - const first = await page.evaluate(() => { - window.qaAudio = window.qaPlayer; - window.qaSnapshot = window.qaLayer.getAcceptedCatalogSnapshot(); - return window.__qaRadio.reads; - }); - const client = soak ? await page.createCDPSession() : null; - if (soak) await page.evaluate(async () => { - const { dataManager, engine } = window.__godsEyeView; - await dataManager.setEnabled('earthquakes', true, { origin: 'user' }); - window.qaModels = new Map([...dataManager.layers].map(([id, entry]) => [id, entry.module])); - window.qaEngines = new Map([[engine.kind, engine]]); - }); - const measurements = [], memory = []; - let contextListeners; - for (let i = 1; i <= swaps; i++) { - const kind = i % 2 ? 'cesium' : 'navara'; - const start = await page.evaluate(() => performance.now()); - await switchGlobe(page, kind); - const elapsed = await page.evaluate((start) => performance.now() - start, start); - measurements.push({ kind, ms: Math.round(elapsed) }); - if (soak) { - assert.ok(await page.evaluate(() => { - const { dataManager, engine } = window.__godsEyeView; - const previous = window.qaEngines.get(engine.kind); - window.qaEngines.set(engine.kind, engine); - return (!previous || previous === engine) && window.qaEngines.size === 2 && - [...window.qaModels].every(([id, model]) => dataManager.layers.get(id)?.module === model) && - window.qaModels.size === 33 && window.qaQuakeReads >= 1 && - dataManager.getEnabledLayerIds().has('earthquakes'); - })); - if ([2, 10, 50, 100, 250, swaps].includes(i)) { - const { result } = await client.send('Runtime.evaluate', { - includeCommandLineAPI: true, - returnByValue: true, - expression: `(() => { - const events = getEventListeners(window.__godsEyeView.engine.host.canvas); - return [events.webglcontextlost.length, events.webglcontextrestored.length]; - })()`, - }); - assert.ok( - Array.isArray(result.value) && result.value.length === 2 && - result.value.every((count) => Number.isInteger(count) && count > 0), - 'the active canvas must expose both WebGL event subscriptions', - ); - contextListeners ??= result.value; - assert.deepEqual(result.value, contextListeners, 'warm Navara must release old context listeners'); - await client.send('HeapProfiler.collectGarbage'); - const heap = await client.send('Runtime.getHeapUsage'); - const dom = await client.send('Memory.getDOMCounters'); - memory.push({ swap: i, contextListeners, jsMiB: Math.round(heap.usedSize / 1048576), ...dom }); - } - } - const state = await page.evaluate(() => ({ - same: - window.qaLayer === - window.__godsEyeView.dataManager.layers.get('radio').module, - player: window.qaAudio === window.qaPlayer, - snapshot: - window.qaSnapshot === window.qaLayer.getAcceptedCatalogSnapshot(), - plays: window.__qaRadio.plays.length, - reads: window.__qaRadio.reads, - state: window.qaLayer.getUIState(), - count: window.qaLayer.getStats().count, - })); - console.log( - kind, - state.count, - state.state.selected?.id, - state.state.audioState, - state.plays, - state.reads, +const strict = createStrict({ origin: url.origin, layers: ['radio'] }); +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', (e) => errors.push(e.message)); + await installRadioFixture(page); + await page.goto(url.href); + await page.waitForFunction( + () => + window.__godsEyeView?.engine?.kind === 'navara' && + document.querySelector('#loading-screen.hidden'), ); - assert.ok(state.same && state.player && state.snapshot); - assert.equal(state.plays, 1); - assert.equal(state.reads, first); - assert.equal(state.count, 750); - assert.equal(state.state.audioState, 'playing'); - assert.equal(state.state.selected.id, stations[10].id); - assert.equal(state.state.volume, 0.36); - } - if (soak) { - const warm = {}; - for (const kind of ['navara', 'cesium']) { - const ms = measurements.slice(2).filter(x => x.kind === kind).map(x => x.ms).sort((a, b) => a - b); - warm[kind] = { p50: ms[Math.ceil(ms.length * 0.50) - 1], p95: ms[Math.ceil(ms.length * 0.95) - 1] }; + await page.evaluate(async (id) => { + const { dataManager, styleManager, tileset } = window.__godsEyeView; + await styleManager.initialRestorePromise; + if (window.__GOOGLE_MAPS_API_KEY__ || tileset) + throw Error('Expected keyless preview'); + window.qaLayer = dataManager.layers.get('radio').module; + await dataManager.setEnabled('radio', true, { origin: 'user' }); + }, stations[0].id); + await page.waitForFunction(() => window.qaLayer?.getStats().count === 750); + await page.evaluate((id) => { + window.qaLayer.selectStation(id, { + autoplay: true, + focus: false, + origin: 'user', + }); + window.qaLayer.setRadioVolume(0.36); + }, stations[10].id); + await page.waitForFunction( + () => + window.__qaRadio.plays.length === 1 && + window.qaLayer.getUIState().audioState === 'playing', + ); + const first = await page.evaluate(() => { + window.qaAudio = window.qaPlayer; + window.qaSnapshot = window.qaLayer.getAcceptedCatalogSnapshot(); + return window.__qaRadio.reads; + }); + for (const kind of ['cesium', 'navara']) { + await strict.switchTo(page, kind); + const state = await page.evaluate(() => ({ + same: + window.qaLayer === + window.__godsEyeView.dataManager.layers.get('radio').module, + player: window.qaAudio === window.qaPlayer, + snapshot: + window.qaSnapshot === window.qaLayer.getAcceptedCatalogSnapshot(), + plays: window.__qaRadio.plays.length, + reads: window.__qaRadio.reads, + state: window.qaLayer.getUIState(), + count: window.qaLayer.getStats().count, + })); + console.log( + kind, + state.count, + state.state.selected?.id, + state.state.audioState, + state.plays, + state.reads, + ); + assert.ok(state.same && state.player && state.snapshot); + assert.equal(state.plays, 1); + assert.equal(state.reads, first); + assert.equal(state.count, 750); + assert.equal(state.state.audioState, 'playing'); + assert.equal(state.state.selected.id, stations[10].id); + assert.equal(state.state.volume, 0.36); + // The fixture's HTMLMediaElement.prototype.play counts play() calls: the + // switch must not start the stream again. + await strict.passed( + page, + 'radio', + 'same model, player element and catalog snapshot; 750 stations, one catalog read, no second play(), same station playing at volume 0.36', + ); } - console.log('SOAK', JSON.stringify({ swaps, first: measurements.slice(0, 2), warm, memory })); - } - assert.deepEqual(errors, []); - console.log( - 'PASS one live radio player, one station directory and the same selection', - ); -} finally { - await browser.close(); -} + assert.deepEqual(errors, []); + console.log( + 'PASS one live radio player, one station directory and the same selection', + ); + }, +); diff --git a/scripts/qa-space-engine-switch.mjs b/scripts/qa-space-engine-switch.mjs index 4a25c910b..a7a6248c2 100644 --- a/scripts/qa-space-engine-switch.mjs +++ b/scripts/qa-space-engine-switch.mjs @@ -1,30 +1,21 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; import { launchFixture } from '../src/bench/launchFixture.js'; import { satelliteFixture, SATELLITE_FIXTURE_EPOCH, } from '../src/bench/satelliteFixture.js'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; -const url = new URL(process.argv[2]), +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +const url = localPreview(process.argv[2]), missions = process.argv.includes('--missions'); -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); url.search = '?engine=navara&welcome=0'; url.hash = 'v=2&lat=5&lon=5&alt=18000000&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const layer = missions ? 'rocket-launches' : 'satellites'; +const strict = createStrict({ + origin: url.origin, + layers: [missions ? 'rocket-launches' : 'satellites'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); const errors = []; p.on('pageerror', (e) => { errors.push(e.message); @@ -68,7 +59,7 @@ try { await p.goto(url.href); await p.waitForFunction( () => - window.__godsEyeView?.engine.kind === 'navara' && + window.__godsEyeView?.engine?.kind === 'navara' && document.querySelector('#loading-screen.hidden'), ); await p.evaluate(async (missions) => { @@ -127,8 +118,28 @@ try { before.mission, before.replay?.active, ); + // Each source read exactly once: the seven TLE groups of the dense + // catalog; in Space Missions the launch list and every TLE group the + // mission view asks for. + const groupPaths = Object.keys(groups).map((g) => '/api/celestrak/' + g); + const readsOnce = async () => { + const reads = await p.evaluate(() => ({ ...window.spaceReads })); + await strict.check( + p, + layer, + missions + ? 'the launch list and each TLE group read exactly once' + : 'each of the seven TLE groups read exactly once', + Object.values(reads).every((n) => n === 1) && + (missions + ? reads['/api/launches'] === 1 + : Object.keys(reads).sort().join() === [...groupPaths].sort().join()), + reads, + ); + }; + await readsOnce(); for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); + await strict.switchTo(p, kind); const state = await p.evaluate(() => { const { dataManager, engine, styleManager } = window.__godsEyeView; return { @@ -174,6 +185,14 @@ try { assert.ok(state.follow && state.ring > 0); assert.equal(state.sat.lastUpdate, before.stamp); } + await strict.passed( + p, + layer, + missions + ? 'same models; Space Missions mode, the same mission paused at the same progress, speed 4' + : 'same models; 6 839 objects, the ISS tracked with its ring, same update stamp', + ); + await readsOnce(); } if (missions) { await p.evaluate(() => window.la.toggleReplayPause()); @@ -183,9 +202,12 @@ try { {}, before.replay.phaseProgress, ); - await switchGlobe(p, 'cesium'); + await strict.switchTo(p, 'cesium'); const moving = await p.evaluate(() => window.la.qaDrawn()); - assert.ok( + await strict.check( + p, + layer, + 'a running replay keeps advancing through the switch', moving.replay?.active && !moving.paused && moving.replay.phaseProgress > before.replay.phaseProgress, @@ -203,6 +225,4 @@ try { } assert.deepEqual(errors, []); console.log('PASS retained space data and active camera across both planets'); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-street-level-engine-switch.mjs b/scripts/qa-street-level-engine-switch.mjs index fd7650ca8..46b63d71b 100644 --- a/scripts/qa-street-level-engine-switch.mjs +++ b/scripts/qa-street-level-engine-switch.mjs @@ -1,6 +1,5 @@ import assert from 'node:assert/strict'; -import puppeteer from 'puppeteer'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { serveFixtures, newFixture } from './qa-street-level.mjs'; import { KV_LINE, KV_SEQUENCE_ID } from './fixtures/street-level/kartaView.mjs'; import { @@ -10,162 +9,205 @@ import { } from './fixtures/street-level/panoramax.mjs'; const panorama = process.argv.includes('--panoramax'); const provider = panorama ? 'panoramax' : 'kartaview'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw Error('Keyless local preview required'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0'; const at = panorama ? PNX_PARK : { lat: 38.5831, lon: KV_LINE.lon, height: 850 }; base.hash = `v=2&lat=${at.lat}&lon=${at.lon}&alt=${at.height}&heading=0&pitch=-90&roll=0&map=sentinel2&l=`; -const browser = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// serveFixtures (qa-street-level.mjs) answers these provider hosts. +const strict = createStrict({ + origin: base.origin, + layers: ['street-level'], + fixtures: [ + '*://*.mapillary.com/*', + 'https://api.kartaview.org/*', + 'https://cdn.kartaview.org/*', + '*://*.openstreetcam.org/*', + 'https://api.panoramax.xyz/*', + 'https://panoramax.openstreetmap.fr/*', + ], }); -try { - const p = await browser.newPage(); - await blockPaidQaMaps(p); - await p.setViewport({ width: 1280, height: 900 }); - const errors = [], - fixture = newFixture({ configured: true }); - p.on('pageerror', (e) => { - errors.push(e.message); - console.error(e.message.split('?')[0]); - }); - p.on('console', (m) => { - if (/shader|registerMeshPicker/i.test(m.text()) && m.type() === 'error') - errors.push(m.text()); - }); - await serveFixtures(p, fixture); - let photoRequests = 0; - p.on('request', (req) => { - if ( - /\.jpg(?:$|\?)/i.test(new URL(req.url()).pathname) && - /openstreetcam|kartaview|panoramax/.test(new URL(req.url()).hostname) - ) - photoRequests++; - }); - await p.goto(base.href); - const ready = async (k) => { - await p.waitForFunction( - (kind) => - window.__godsEyeView?.engine?.kind === kind && - !document.querySelector('#globe-engine')?.disabled && - !document.querySelector('.engine-switch-frame') && - document.querySelector('#loading-screen.hidden'), - { timeout: 90000 }, - k, - ); - await p.evaluate( - () => window.__godsEyeView.styleManager.initialRestorePromise, - ); - }; - await ready('navara'); - await p.evaluate(async (provider) => { - const { dataManager, tileset } = window.__godsEyeView; - if (tileset || window.__GOOGLE_MAPS_API_KEY__) - throw Error('Expected keyless preview'); - window.qaStreet = dataManager.layers.get('street-level').module; - await dataManager.setEnabled('street-level', true, { origin: 'user' }); - window.qaStreet.setProviderEnabled(provider, true); - }, provider); - const until = async (fn) => { - try { - await p.waitForFunction(fn, { timeout: 20000 }); - } catch (e) { - console.log(await p.evaluate(() => window.qaStreet.getUIState())); - throw e; +await strict.run( + { args: ['--use-angle=metal', '--enable-gpu'] }, + async (browser) => { + const p = await browser.newPage(); + await p.setViewport({ width: 1280, height: 900 }); + const errors = [], + fixture = newFixture({ configured: true }); + p.on('pageerror', (e) => { + errors.push(e.message); + console.error(e.message.split('?')[0]); + }); + p.on('console', (m) => { + if (/shader|registerMeshPicker/i.test(m.text()) && m.type() === 'error') + errors.push(m.text()); + }); + await serveFixtures(p, fixture); + let photoRequests = 0; + p.on('request', (req) => { + if ( + /\.jpg(?:$|\?)/i.test(new URL(req.url()).pathname) && + /openstreetcam|kartaview|panoramax/.test(new URL(req.url()).hostname) + ) + photoRequests++; + }); + await p.goto(base.href); + const ready = async (k) => { + await p.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { timeout: 90000 }, + k, + ); + await p.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); + }; + await ready('navara'); + await p.evaluate(async (provider) => { + const { dataManager, tileset } = window.__godsEyeView; + if (tileset || window.__GOOGLE_MAPS_API_KEY__) + throw Error('Expected keyless preview'); + window.qaStreet = dataManager.layers.get('street-level').module; + await dataManager.setEnabled('street-level', true, { origin: 'user' }); + window.qaStreet.setProviderEnabled(provider, true); + }, provider); + const until = async (fn) => { + try { + await p.waitForFunction(fn, { timeout: 20000 }); + } catch (e) { + console.log(await p.evaluate(() => window.qaStreet.getUIState())); + throw e; + } + }; + await until(() => window.qaStreet.getUIState().coverage.count > 0); + // The fixture drive passes under the middle of the canvas. Click its actual picked line. + const pickLine = async () => { + const xy = await p.evaluate( + (prefix) => { + const { engine } = window.__godsEyeView, + canvas = engine.host.canvas, + rect = canvas.getBoundingClientRect(); + for (let dy = -30; dy <= 30; dy += 3) + for (let dx = -40; dx <= 40; dx++) { + const at = { x: rect.width / 2 + dx, y: rect.height / 2 + dy }, + pick = engine.picker.pick(at), + id = pick?.id?.id ?? pick?.id; + if (typeof id === 'string' && id.startsWith(prefix)) + return { x: at.x + rect.x, y: at.y + rect.y, id }; + } + return null; + }, + panorama ? 'pnx:seq:' : 'kv:line:', + ); + assert.ok(xy, 'native ground line was pickable'); + console.log('pick', xy); + await p.mouse.click(xy.x, xy.y); + await until( + () => + window.qaStreet.getUIState().street.open && + window.qaStreet.getUIState().sequence.images > 0 && + (document.querySelector('#sl-viewer img')?.naturalWidth > 0 || + document.querySelector('#sl-viewer canvas:not([hidden])')?.width > + 0), + ); + await strict.passed( + p, + 'street-level', + 'a click on the coverage line opens its sequence and photo', + xy, + ); + return xy; + }; + const xy = await pickLine(); + if (panorama) { + const expected = panoramaxPictures().filter( + (p) => p.sequenceId === xy.id.slice('pnx:seq:'.length), + ).length; + await p.waitForFunction( + (n) => window.qaStreet.getUIState().sequence.images === n, + { timeout: 20000 }, + expected, + ); } - }; - await until(() => window.qaStreet.getUIState().coverage.count > 0); - // The fixture drive passes under the middle of the canvas. Click its actual picked line. - const xy = await p.evaluate( - (prefix) => { - const { engine } = window.__godsEyeView, - canvas = engine.host.canvas, - rect = canvas.getBoundingClientRect(); - for (let dy = -30; dy <= 30; dy += 3) - for (let dx = -40; dx <= 40; dx++) { - const at = { x: rect.width / 2 + dx, y: rect.height / 2 + dy }, - pick = engine.picker.pick(at); - if (typeof pick?.id === 'string' && pick.id.startsWith(prefix)) - return { x: at.x + rect.x, y: at.y + rect.y, id: pick.id }; - } - return null; - }, - panorama ? 'pnx:seq:' : 'kv:line:', - ); - assert.ok(xy, 'native ground line was pickable'); - console.log('pick', xy); - await p.mouse.click(xy.x, xy.y); - await until( - () => - window.qaStreet.getUIState().street.open && - window.qaStreet.getUIState().sequence.images > 0 && - (document.querySelector('#sl-viewer img')?.naturalWidth > 0 || - document.querySelector('#sl-viewer canvas:not([hidden])')?.width > 0), - ); - if (panorama) { - const expected = panoramaxPictures().filter( - (p) => p.sequenceId === xy.id.slice('pnx:seq:'.length), - ).length; - await p.waitForFunction( - (n) => window.qaStreet.getUIState().sequence.images === n, - { timeout: 20000 }, - expected, + const first = await p.evaluate(() => { + window.streetPhoto = + document.querySelector('#sl-viewer canvas:not([hidden])') ?? + document.querySelector('#sl-viewer img'); + return window.qaStreet.getUIState(); + }); + assert.ok( + (panorama ? PNX_SEQUENCES : [KV_SEQUENCE_ID]).includes( + first.street.sequenceId, + ), ); - } - const first = await p.evaluate(() => { - window.streetPhoto = - document.querySelector('#sl-viewer canvas:not([hidden])') ?? - document.querySelector('#sl-viewer img'); - return window.qaStreet.getUIState(); - }); - assert.ok( - (panorama ? PNX_SEQUENCES : [KV_SEQUENCE_ID]).includes( - first.street.sequenceId, - ), - ); - const reads = photoRequests; - for (const kind of ['cesium', 'navara']) { - await p.select('#globe-engine', kind); - await ready(kind); - const state = await p.evaluate(() => ({ - model: - window.__godsEyeView.dataManager.layers.get('street-level').module === - window.qaStreet, - samePhoto: - document.querySelector('#sl-viewer')?.contains(window.streetPhoto) && - window.streetPhoto.isConnected, - ui: window.qaStreet.getUIState(), - })); + const reads = photoRequests; + for (const kind of ['cesium', 'navara']) { + await strict.pick(p, kind); + await ready(kind); + // The new globe reloads coverage right after it attaches: compare once + // that load is over. + await until( + () => + !window.qaStreet.getUIState().coverage.loading && + window.qaStreet.getUIState().coverage.count > 0, + ); + const state = await p.evaluate(() => ({ + model: + window.__godsEyeView.dataManager.layers.get('street-level').module === + window.qaStreet, + samePhoto: + document.querySelector('#sl-viewer')?.contains(window.streetPhoto) && + window.streetPhoto.isConnected, + ui: window.qaStreet.getUIState(), + })); + console.log( + kind, + state.model, + state.samePhoto, + state.ui.street.imageId, + state.ui.sequence.images, + state.ui.coverage.count, + ); + assert.equal(state.model, true); + assert.equal(state.samePhoto, true); + assert.ok(state.ui.street.open); + assert.equal(state.ui.street.imageId, first.street.imageId); + assert.equal(state.ui.sequence.images, first.sequence.images); + assert.deepEqual(state.ui.filter, first.filter); + assert.ok(state.ui.coverage.count > 0); + await strict.passed( + p, + 'street-level', + 'same model, the same mounted photo, image and sequence; coverage kept', + ); + } + assert.equal(photoRequests, reads); + await strict.passed( + p, + 'street-level', + 'no photo requested again by the switches', + ); + // Click-picking on Cesium: close the viewer, switch, click the line again. + await p.evaluate(() => window.qaStreet.closeViewer()); + await until(() => !window.qaStreet.getUIState().street.open); + await strict.pick(p, 'cesium'); + await ready('cesium'); + await until( + () => + !window.qaStreet.getUIState().coverage.loading && + window.qaStreet.getUIState().coverage.count > 0, + ); + await new Promise((resolve) => setTimeout(resolve, 1000)); + await pickLine(); + assert.deepEqual(errors, []); + assert.deepEqual(fixture.unknown, []); console.log( - kind, - state.model, - state.samePhoto, - state.ui.street.imageId, - state.ui.sequence.images, - state.ui.coverage.count, + 'PASS native line pick, same mounted photo and sequence across two switches', ); - assert.equal(state.model, true); - assert.equal(state.samePhoto, true); - assert.ok(state.ui.street.open); - assert.equal(state.ui.street.imageId, first.street.imageId); - assert.equal(state.ui.sequence.images, first.sequence.images); - assert.deepEqual(state.ui.filter, first.filter); - assert.ok(state.ui.coverage.count > 0); - } - assert.equal(photoRequests, reads); - assert.deepEqual(errors, []); - assert.deepEqual(fixture.unknown, []); - console.log( - 'PASS native line pick, same mounted photo and sequence across two switches', - ); -} finally { - await browser.close(); -} + }, +); diff --git a/scripts/qa-strict-cold-chunks.mjs b/scripts/qa-strict-cold-chunks.mjs new file mode 100644 index 000000000..37f79e157 --- /dev/null +++ b/scripts/qa-strict-cold-chunks.mjs @@ -0,0 +1,95 @@ +// Contract: a cold start loads only the engine it starts. From the network +// log of a fresh page: +// - a cold Navara start requests nothing of the Cesium library (/cesium/…: +// Cesium.js, its workers and assets); +// - a cold Cesium start requests no chunk that carries three or Navara code +// (the FORBIDDEN_MARKERS of src/tooling/mainChunk.js) and no Navara wasm. +// Control: after a pick to the other engine, its code is requested, so the +// detection sees what it is looking for. +// +// node scripts/qa-strict-cold-chunks.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { forbiddenMarkersIn } from '../src/tooling/mainChunk.js'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { installSessionFixtures, openApp } from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +const contents = new Map(); +// The preview's own file: three or Navara markers in it? +const markersOf = async (url) => { + if (!contents.has(url)) + contents.set( + url, + fetch(url) + .then((r) => r.text()) + .then(forbiddenMarkersIn), + ); + return contents.get(url); +}; +const cesiumCode = (url) => new URL(url).pathname.startsWith('/cesium/'); +const navaraCode = async (url) => { + const { pathname } = new URL(url); + if (/navara.*\.wasm$/.test(pathname)) return ['navara wasm']; + if (!pathname.endsWith('.js') || pathname.startsWith('/cesium/')) return []; + return markersOf(url); +}; +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + for (const start of ['navara', 'cesium']) { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const urls = []; + p.on('request', (request) => { + if (request.url().startsWith(base.origin)) urls.push(request.url()); + }); + await installSessionFixtures(p); + await openApp(p, base.origin, start); + // Let lazy loads that follow the start (workers, idle prefetch) happen. + await new Promise((resolve) => setTimeout(resolve, 5000)); + const before = [...urls]; + const other = start === 'navara' ? 'cesium' : 'navara'; + await strict.pick(p, other); + await waitForEngine(p, other); + const after = urls.slice(before.length); + const otherCode = async (list) => { + const found = []; + for (const url of list) + if ( + other === 'cesium' + ? cesiumCode(url) + : (await navaraCode(url)).length > 0 + ) + found.push(new URL(url).pathname); + return [...new Set(found)]; + }; + const row = { + start, + otherEngine: other, + loadedAtColdStart: await otherCode(before), + loadedAfterPick: (await otherCode(after)).length, + }; + results.push(row); + console.log( + row.loadedAtColdStart.length ? 'FAIL' : 'PASS', + JSON.stringify(row), + ); + await p.close(); + } + for (const row of results) + assert.ok( + row.loadedAfterPick > 0, + `control: picking ${row.otherEngine} loads its code`, + ); + assert.deepEqual( + results.filter((row) => row.loadedAtColdStart.length), + [], + 'a cold start loaded the other engine', + ); +}); diff --git a/scripts/qa-strict-context-loss.mjs b/scripts/qa-strict-context-loss.mjs new file mode 100644 index 000000000..0bf428d8c --- /dev/null +++ b/scripts/qa-strict-context-loss.mjs @@ -0,0 +1,235 @@ +// Contract: a WebGL context loss during a switch (WEBGL_lose_context, the +// context coming back 1 s later with restoreContext(), as after a GPU +// reset). After each case the page runs one engine on the same session, the +// history matches the outcome (one entry for a completed switch, none for a +// rollback), the running engine draws a non-blank frame, and a later pick +// back to the engine that lost its context draws a non-blank frame too. +// +// - outgoing: the running engine's context is lost by the same input that +// starts the switch (cold Navara→Cesium, then warm Cesium→Navara); +// - incoming: the parked target engine's context is lost by the input that +// asks for it (warm, both directions). +// +// node scripts/qa-strict-context-loss.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const other = (kind) => (kind === 'navara' ? 'cesium' : 'navara'); +const strict = createStrict({ origin: base.origin, layers: [] }); +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + let p = null; + const errors = []; + // A fresh page on `from`; warm visits the other engine first. + const fresh = async (from, warm) => { + await p?.close().catch(() => {}); + p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + await openApp(p, base.origin, from); + await startSession(p, ['earthquakes', 'flights']); + if (!warm) return; + for (const kind of [other(from), from]) { + await strict.pick(p, kind); + await waitForEngine(p, kind); + await rememberEngine(p); + } + }; + + // The running engine draws a non-blank frame within 15 s. + const draws = (kind) => + p.evaluate( + (kind) => + Promise.race([ + window.__qaFirstFrame(kind).then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 15000)), + ]), + kind, + ); + const settle = async () => { + await p.waitForFunction( + () => + (window.__godsEyeView && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden')) || + document.querySelector('.engine-switch-retry'), + { polling: 100, timeout: 90000 }, + ); + await new Promise((resolve) => setTimeout(resolve, 1500)); + }; + const intact = (state) => + state.running && + state.manager && + state.models && + state.enabled && + state.address === state.engine && + state.picker === state.engine; + + for (const [loser, from, temperature] of [ + ['outgoing', 'navara', 'cold'], + ['outgoing', 'cesium', 'warm'], + ['incoming', 'navara', 'warm'], + ['incoming', 'cesium', 'warm'], + ]) { + const to = other(from); + const label = `${temperature} ${from}→${to}, ${loser} context lost`; + // Cold cases, and any case after a failed one, start on a fresh page. + if (!p || temperature === 'cold' || results.at(-1)?.failed.length) + await fresh(from, temperature === 'warm'); + else if ((await sessionState(p)).engine !== from) { + await strict.pick(p, from); + await waitForEngine(p, from); + await rememberEngine(p); + } + const before = await sessionState(p); + // The input that starts the switch also loses the context; it comes + // back a second later. + await p.evaluate( + (loser, from, to) => { + const select = document.querySelector('#globe-engine'); + select.addEventListener( + 'change', + () => { + const engine = + loser === 'outgoing' + ? window.__godsEyeView.engine + : window.__qaEngines.get(to); + const canvas = engine.host.canvas; + const context = + canvas.getContext('webgl2') ?? canvas.getContext('webgl'); + const extension = context.getExtension('WEBGL_lose_context'); + window.__qaLoss = { + kind: engine.kind, + lost: false, + restored: false, + }; + canvas.addEventListener( + 'webglcontextrestored', + () => (window.__qaLoss.restoredEvent = true), + { once: true }, + ); + extension.loseContext(); + window.__qaLoss.lost = context.isContextLost(); + setTimeout(() => { + extension.restoreContext(); + window.__qaLoss.restored = true; + }, 1000); + }, + { once: true, capture: true }, + ); + }, + loser, + from, + to, + ); + try { + await strict.pick(p, to, { settle: false }); + await settle(); + await p.waitForFunction(() => window.__qaLoss?.restored, { + polling: 100, + timeout: 10000, + }); + await settle(); + } catch (error) { + results.push({ + case: label, + failed: [ + 'the switch could not be made: ' + + String(error.message).slice(0, 200), + ], + }); + console.log('FAIL', JSON.stringify(results.at(-1))); + continue; + } + const after = await sessionState(p); + const loss = await p.evaluate(() => window.__qaLoss); + const outcome = after.engine === to ? 'switched' : 'rolled back'; + if (after.running) await rememberEngine(p); + const drawn = after.running ? await draws(after.engine) : false; + console.log( + 'after', + label, + JSON.stringify({ after, loss, drawn }), + JSON.stringify( + await p.evaluate(() => ({ + toggle: document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded'), + retry: !!document.querySelector('.engine-switch-retry'), + status: document.querySelector('#loading-screen .loader-status') + ?.textContent, + loading: document.querySelector('#loading-screen')?.className, + })), + ), + ); + // Back to the engine that lost its context, with real input. + const back = after.engine === loss.kind ? other(loss.kind) : loss.kind; + let backDrawn = null, + backState = null; + let backError = null; + if (after.running) { + try { + await strict.pick(p, back); + await waitForEngine(p, back, { timeout: 60000 }); + } catch (error) { + backError = String(error.message).slice(0, 300); + } + backState = await sessionState(p); + if (backState.running) await rememberEngine(p); + backDrawn = backState.engine === back ? await draws(back) : false; + } + const row = { + case: label, + lostKind: loss.kind, + lost: loss.lost, + restoredEvent: Boolean(loss.restoredEvent), + outcome, + sessionIntact: intact(after), + history: after.history - before.history, + drawn, + back, + backEngine: backState?.engine ?? null, + backSessionIntact: backState ? intact(backState) : false, + backDrawn, + backError, + pageErrors: errors.splice(0), + }; + row.failed = [ + row.lost || 'the context was not lost', + row.sessionIntact || 'the session is not intact', + row.history === (outcome === 'switched' ? 1 : 0) || + 'the history does not match the outcome', + row.drawn || 'no non-blank frame from the running engine', + row.backEngine === back || `the pick back to ${back} did not run it`, + row.backSessionIntact || 'the session is not intact after the pick back', + row.backDrawn || `no non-blank frame from ${back} after the pick back`, + row.backError === null || 'the pick back failed: ' + row.backError, + row.pageErrors.length === 0 || 'uncaught page errors', + ].filter((v) => v !== true); + results.push(row); + console.log(row.failed.length ? 'FAIL' : 'PASS', JSON.stringify(row)); + } + assert.deepEqual( + results.filter((row) => row.failed.length), + [], + 'context-loss cases that broke the contract', + ); +}); diff --git a/scripts/qa-strict-failure-matrix.mjs b/scripts/qa-strict-failure-matrix.mjs new file mode 100644 index 000000000..c3d2a5eec --- /dev/null +++ b/scripts/qa-strict-failure-matrix.mjs @@ -0,0 +1,283 @@ +// Contract: failure injection in both directions, cold and warm, and a slow +// first frame. Each case starts on a fresh page with a session (earthquakes +// and flights, one aircraft selected) and switches with real input on the +// picker. +// +// - A failed switch rolls back to the engine that was running, with the +// same session (manager, every layer model, enabled layers, selection), +// the address still naming it and no history entry; an ordinary pick +// afterwards succeeds and adds exactly one entry. +// cold: the incoming engine cannot get a WebGL context; +// warm: the parked incoming engine fails to resume. +// - A first frame slower than 10 s (animation frames held for 12 s) does not +// roll back: the switch completes, the cover goes after the 10 s wait, one +// history entry, the session intact, and the frame arrives. +// +// node scripts/qa-strict-failure-matrix.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + assertSession, + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + SESSION_HASH, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const other = (kind) => (kind === 'navara' ? 'cesium' : 'navara'); +const strict = createStrict({ origin: base.origin, layers: [] }); +const results = []; +strict.extra.contracts = results; +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + // A fresh page on `start` with the session; `warm` visits the other + // engine first so it is parked. + const open = async (start, warm) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.errors = []; + p.on('pageerror', (e) => p.errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + // Flights on with the first fixture aircraft tracked (the address's + // layer options), so the session carries a selection. + await openApp( + p, + base.origin, + start, + SESSION_HASH.replace(/&l=$/, '&l=f&lo=f.e.1_f.t.a00000'), + ); + await startSession(p, ['earthquakes', 'flights']); + await p.waitForFunction( + () => + window.__godsEyeView.styleManager.getAircraftTrackingTarget()?.id === + 'a00000', + { timeout: 30000 }, + ); + if (warm) { + await strict.pick(p, other(start)); + await waitForEngine(p, other(start)); + await rememberEngine(p); + await strict.pick(p, start); + await waitForEngine(p, start); + await rememberEngine(p); + // The warm-up's Navara→Cesium switch drops the tracked aircraft (a + // defect of its own, reported); track it + // again so this case tests the failure path with a selection. + await p.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('flights') + .module.trackById('a00000', { origin: 'user' }), + ); + await p.waitForFunction( + () => + window.__godsEyeView.styleManager.getAircraftTrackingTarget()?.id === + 'a00000', + { timeout: 30000 }, + ); + } + return p; + }; + const selection = (p) => + p.evaluate( + () => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id ?? + null, + ); + + for (const [start, temperature] of [ + ['navara', 'cold'], + ['cesium', 'cold'], + ['cesium', 'warm'], + ['navara', 'warm'], + ]) { + const target = other(start); + const label = `${temperature} ${start}→${target} failure`; + const p = await open(start, temperature === 'warm'); + const before = await sessionState(p); + const selected = await selection(p); + if (temperature === 'cold') + await p.evaluate(() => { + const original = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + // The running globe already holds its context; only a new + // globe asks for one. + if (/webgl/i.test(String(type))) { + window.__qaRefused = (window.__qaRefused ?? 0) + 1; + throw new Error('QA: no WebGL context for the new globe'); + } + return original.call(this, type, ...args); + }; + window.__qaRestore = () => { + HTMLCanvasElement.prototype.getContext = original; + }; + }); + else + await p.evaluate((target) => { + const parked = window.__qaEngines.get(target); + const request = parked.render.request; + parked.render.request = function (reason, ...args) { + if (reason === 'engine-resume' && !window.__qaRestored) { + window.__qaRefused = (window.__qaRefused ?? 0) + 1; + throw new Error('QA: the parked globe failed to resume'); + } + return request.call(this, reason, ...args); + }; + window.__qaRestore = () => { + window.__qaRestored = true; + }; + }, target); + await strict.pick(p, target); + await waitForEngine(p, start); + const refused = await p.evaluate(() => window.__qaRefused ?? 0); + const rolled = await assertSession(p, start, label + ': rollback'); + // The tracked aircraft, given up to 10 s to be re-acquired. + const kept = await p + .waitForFunction( + (id) => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id === + id, + { timeout: 10000, polling: 100 }, + selected, + ) + .then(() => selected) + .catch(() => selection(p)); + await p.evaluate(() => window.__qaRestore()); + // An ordinary pick afterwards. + await strict.pick(p, target); + await waitForEngine(p, target); + const retried = await assertSession(p, target, label + ': retry'); + const afterRetry = await p + .waitForFunction( + (id) => + window.__godsEyeView?.styleManager.getAircraftTrackingTarget()?.id === + id, + { timeout: 10000, polling: 100 }, + selected, + ) + .then(() => selected) + .catch(() => selection(p)); + const row = { + case: label, + refused, + rollbackTarget: rolled.engine, + historyAfterFailure: rolled.history - before.history, + selectionKept: kept === selected, + selectionAfterRollback: kept, + historyAfterRetry: retried.history - before.history, + selectionAfterRetry: afterRetry, + errors: p.errors.length, + }; + row.failed = [ + refused > 0 || 'the injected failure did not happen', + row.rollbackTarget === start || 'rolled back elsewhere', + row.historyAfterFailure === 0 || 'the failed switch changed the history', + selected === 'a00000' || 'no tracked aircraft before the failure', + row.selectionKept || 'the tracked aircraft is lost after the rollback', + row.historyAfterRetry === 1 || 'the retry did not add one entry', + row.selectionAfterRetry === selected || + 'the tracked aircraft is lost after the retry', + ].filter((v) => v !== true); + results.push(row); + console.log(row.failed.length ? 'FAIL' : 'PASS', JSON.stringify(row)); + await p.close(); + } + + // Slow first frame: animation frames held for 12 s from the person's + // input (a stalled GPU or compositor), cold and warm. + for (const [start, temperature] of [ + ['cesium', 'cold'], + ['navara', 'warm'], + ]) { + const target = other(start); + const label = `${temperature} ${start}→${target} first frame after 12 s`; + const p = await open(start, temperature === 'warm'); + const before = await sessionState(p); + await p.evaluate((target) => { + window.__qaSlow = { frame: window.__qaFirstFrame(target) }; + window.__qaSlow.frame.then((t) => (window.__qaSlow.drawn = t)); + const select = document.querySelector('#globe-engine'); + // From the change on, every animation frame waits until 12 s after it. + select.addEventListener( + 'change', + () => { + const until = performance.now() + 12000; + window.__qaSlow.at = performance.now(); + const raf = window.requestAnimationFrame.bind(window); + window.requestAnimationFrame = (callback) => + raf((time) => { + const wait = until - performance.now(); + if (wait <= 0) return callback(time); + setTimeout(() => raf(callback), wait); + }); + window.__qaSlow.release = () => { + window.requestAnimationFrame = raf; + }; + }, + { once: true, capture: true }, + ); + new MutationObserver((_, observer) => { + if ( + window.__qaSlow.at && + !window.__qaSlow.coverGone && + !document.querySelector('.engine-switch-frame') && + !document.querySelector('#globe-engine')?.disabled + ) { + window.__qaSlow.coverGone = performance.now(); + observer.disconnect(); + } + }).observe(document.body, { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['disabled'], + }); + }, target); + await strict.pick(p, target, { settle: false }); + await p.waitForFunction( + (target) => + window.__godsEyeView?.engine?.kind === target && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame'), + { polling: 200, timeout: 60000 }, + target, + ); + const done = await assertSession(p, target, label); + const timing = await p.evaluate(async () => { + const frame = await window.__qaSlow.frame; + return { + coverGoneMs: Math.round(window.__qaSlow.coverGone - window.__qaSlow.at), + firstFrameMs: Math.round(frame.presented - window.__qaSlow.at), + }; + }); + const row = { + case: label, + engine: done.engine, + history: done.history - before.history, + ...timing, + errors: p.errors.length, + }; + results.push(row); + assert.equal(row.history, 1, 'one history entry, no rollback'); + assert.ok( + row.coverGoneMs >= 9500 && row.coverGoneMs < 12500, + 'the cover goes after the 10 s first-frame wait ' + JSON.stringify(row), + ); + assert.ok(row.firstFrameMs >= 12000, 'the frame arrives after the stall'); + console.log('PASS', JSON.stringify(row)); + await p.close(); + } + assert.deepEqual( + results.filter((row) => row.failed?.length), + [], + 'failure-injection cases that broke the contract', + ); +}); diff --git a/scripts/qa-strict-harness.mjs b/scripts/qa-strict-harness.mjs new file mode 100644 index 000000000..ddb9dd711 --- /dev/null +++ b/scripts/qa-strict-harness.mjs @@ -0,0 +1,736 @@ +// The strict acceptance harness shared by every browser scenario that +// qa-all-layer-switches.mjs runs. +// +// - Network: a default-deny request policy on every page, frame and worker +// of the browser, installed before the target runs (CDP auto-attach with +// waitForDebuggerOnStart). Paid hosts and paid preview routes are blocked, +// then only the preview origin, the scenario's declared fixture URLs and +// the free tile hosts (answered from a disk cache) are allowed. A blocked +// request, a fixture or tile URL answered by the real network, or a socket +// to another origin is a violation and fails the scenario with its URL. +// - Coverage: the runner passes the layer ids (--layers=a,b). The scenario +// records each assertion it made per layer id and engine; finish() fails +// when a declared layer has no assertion on either engine, and writes the +// manifest (--evidence=FILE) the runner cross-checks against the catalog. +// - Engine picks: pickEngine() switches with real input on the picker (a +// pointer click on its label, then keyboard type-ahead), never page.select. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import os from 'node:os'; +import { + KEYLESS_PROBES, + probePattern, + strictBlockPatterns, + STRICT_TILE_HOSTS, +} from './qa-map-policy.mjs'; +import { createMissLoader, tileCacheFile } from './qa/tileCache.mjs'; + +export const ENGINES = Object.freeze(['cesium', 'navara']); +const RESERVED_PORTS = ['', '4173', '4185', '4186', '4187', '4188', '4189']; +const RESERVED = [...RESERVED_PORTS, '4194', '4196']; + +/** The values of --layers=a,b and --evidence=FILE (null when absent). */ +export function strictArguments(argv = process.argv) { + const value = (name) => + argv.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3); + const layers = value('layers'); + return { + layers: layers ? layers.split(',').filter(Boolean) : null, + evidence: value('evidence') ?? process.env.QA_STRICT_EVIDENCE ?? null, + }; +} + +/** Refuse anything but an unreserved local preview. */ +export function localPreview(href) { + const url = new URL(href); + if ( + !['localhost', '127.0.0.1'].includes(url.hostname) || + RESERVED.includes(url.port) + ) + throw Error('Supply an unreserved local keyless preview URL'); + return url; +} + +/** The URL with no remote IP: fulfilled by a fixture or a cache, or a data URL. */ +const live = (response) => Boolean(response.remoteIPAddress); + +/** + * Follow every request of every target of `browser` under the strict policy. + * Returns the ledger the policy fills in. + */ +export async function installStrictNetwork( + browser, + { origin, fixtures = [], routes = [], tileHosts = STRICT_TILE_HOSTS, ledger }, +) { + const preview = new URL(origin); + fixtures = [...fixtures, ...routes.map((route) => route.pattern)]; + const urlPatterns = strictBlockPatterns(origin, { fixtures, tileHosts }); + const answers = routes.map((route) => ({ + ...route, + test: new URLPattern(route.pattern), + })); + const fixturePatterns = fixtures.map((p) => new URLPattern(p)); + const isFixture = (url) => fixturePatterns.some((p) => p.test(url)); + const root = await browser.target().createCDPSession(); + const connection = root.connection(); + const probes = KEYLESS_PROBES.map((probe) => ({ + ...probe, + test: new URLPattern(probePattern(origin, probe)), + })); + const misses = createMissLoader(); + const cacheDir = path.join(os.tmpdir(), 'gev-qa-tiles'); + fs.mkdirSync(cacheDir, { recursive: true }); + const tiles = ledger.tileCache; + // One CDP Fetch handler per target: the keyless probes and the tile hosts + // (from the disk cache, filled on a miss). Nothing else is paused, so + // workers' own requests are never held. + const intercept = async (session, type) => { + session.on('Fetch.requestPaused', async ({ requestId, request }) => { + const fulfil = (status, headers, body) => + session.send('Fetch.fulfillRequest', { + requestId, + responseCode: status, + responseHeaders: headers, + ...(body == null + ? {} + : { body: Buffer.from(body).toString('base64') }), + }); + try { + const route = answers.find((r) => r.test.test(request.url)); + if (route) { + ledger.routes[route.pattern] = + (ledger.routes[route.pattern] ?? 0) + 1; + const answer = await route.respond(request); + return await fulfil( + answer.status ?? 200, + [ + ...(answer.contentType + ? [{ name: 'Content-Type', value: answer.contentType }] + : []), + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + answer.body ?? '', + ); + } + const probe = probes.find((p) => p.test.test(request.url)); + if (probe) { + if (request.method !== probe.method && request.method !== 'HEAD') { + violation('probe-method', request.url, { method: request.method }); + return await session.send('Fetch.failRequest', { + requestId, + errorReason: 'BlockedByClient', + }); + } + ledger.probes[probe.pattern] = + (ledger.probes[probe.pattern] ?? 0) + 1; + return await fulfil( + probe.status, + [ + ...(probe.contentType + ? [{ name: 'Content-Type', value: probe.contentType }] + : []), + { name: 'Cache-Control', value: 'no-store' }, + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + probe.body ?? null, + ); + } + const accept = Object.entries(request.headers ?? {}).find( + ([name]) => name.toLowerCase() === 'accept', + )?.[1]; + const file = + request.method === 'GET' + ? tileCacheFile(request.url, accept, cacheDir, tileHosts) + : null; + if (!file) { + violation('unhandled-intercept', request.url, { target: type }); + return await session.send('Fetch.failRequest', { + requestId, + errorReason: 'BlockedByClient', + }); + } + let entry; + if (fs.existsSync(`${file}.json`)) { + entry = { + ...JSON.parse(fs.readFileSync(`${file}.json`, 'utf8')), + body: fs.readFileSync(file), + }; + tiles.hits++; + } else { + entry = await misses.load(request.url, accept); + if (entry.status === 200) { + fs.writeFileSync(file, entry.body); + fs.writeFileSync( + `${file}.json`, + JSON.stringify({ status: 200, contentType: entry.contentType }), + ); + tiles.fetched++; + } else tiles.failed++; + const host = new URL(request.url).host; + tiles.fetchedByHost[host] = (tiles.fetchedByHost[host] ?? 0) + 1; + } + await fulfil( + entry.status, + [ + ...(entry.contentType + ? [{ name: 'Content-Type', value: entry.contentType }] + : []), + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + entry.body, + ); + } catch (error) { + if ( + /Invalid InterceptionId|closed|detached/i.test(String(error?.message)) + ) + return void tiles.cancelled++; + tiles.failed++; + await session + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await session + .send('Fetch.enable', { + patterns: [ + ...probes.map((probe) => ({ + urlPattern: probePattern(origin, probe).replace(/\*$/, '') + '*', + })), + ...tileHosts.map((host) => ({ urlPattern: `https://${host}/*` })), + ...routes.map((route) => ({ urlPattern: route.pattern })), + ], + }) + .catch((error) => + ledger.setupErrors.push( + type + ' Fetch.enable: ' + String(error?.message).slice(0, 120), + ), + ); + }; + const violation = (kind, url, detail = {}) => { + ledger.violations.push({ kind, url: String(url).slice(0, 300), ...detail }); + }; + const setup = async (session, info) => { + const type = info.type; + const urls = new Map(); + session.on('Network.requestWillBeSent', (event) => { + const { url } = event.request; + urls.set(event.requestId, url); + if (!/^https?:/.test(url)) return; + ledger.requests++; + const address = new URL(url); + if (address.origin === preview.origin) { + ledger.preview++; + if (address.pathname.startsWith('/api/')) { + const route = address.pathname.split('/').slice(0, 4).join('/'); + ledger.previewApi[route] = (ledger.previewApi[route] ?? 0) + 1; + } + } else { + ledger.external[address.host] = + (ledger.external[address.host] ?? 0) + 1; + } + }); + session.on('Network.loadingFailed', (event) => { + if (!event.blockedReason) return; + violation('blocked', urls.get(event.requestId) ?? '?', { + reason: event.blockedReason, + target: type, + resource: event.type, + }); + }); + session.on('Network.responseReceived', ({ response, type: resource }) => { + if (!/^https?:/.test(response.url)) return; + const address = new URL(response.url); + if (probes.some((p) => p.test.test(response.url)) && live(response)) + violation('probe-live', response.url, { target: type }); + if (address.origin === preview.origin) return; + if (!live(response)) { + if (isFixture(response.url)) ledger.fixtureResponses++; + return; + } + // A fixture or tile URL answered by the real network, or any other + // external host that slipped past the block (a frame navigation). + violation('live', response.url, { + remote: response.remoteIPAddress, + target: type, + resource, + }); + }); + session.on('Network.webSocketCreated', ({ url }) => { + const address = new URL(url); + if (address.host !== preview.host) + violation('socket', url, { target: type }); + }); + session.on('Target.attachedToTarget', (event) => + attach(event.sessionId, event.targetInfo, event.waitingForDebugger), + ); + const steps = [ + session.send('Network.enable'), + session.send('Network.setBlockedURLs', { urlPatterns }), + session + .send('Target.setAutoAttach', { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + }) + .catch(() => {}), + ]; + // Workers have no Fetch domain; the page's interception answers their + // requests, and a live answer would still show up as a violation. + if (type === 'page' || type === 'iframe') + steps.push(intercept(session, type)); + await Promise.all(steps); + }; + const attach = async (sessionId, info, waiting) => { + const session = connection.session(sessionId); + if (!session) return; + try { + await setup(session, info); + } catch (error) { + // A target that closes while it attaches has nothing left to load. + if (!/closed|detached|not found/i.test(String(error?.message))) + ledger.setupErrors.push(String(error?.message ?? error).slice(0, 200)); + } finally { + if (waiting) + await session.send('Runtime.runIfWaitingForDebugger').catch(() => {}); + } + }; + root.on('Target.attachedToTarget', (event) => + attach(event.sessionId, event.targetInfo, event.waitingForDebugger), + ); + await root.send('Target.setAutoAttach', { + autoAttach: true, + waitForDebuggerOnStart: true, + flatten: true, + filter: [ + { type: 'page' }, + { type: 'iframe' }, + { type: 'worker' }, + { type: 'shared_worker' }, + { type: 'service_worker' }, + { type: 'tab', exclude: true }, + { type: 'browser', exclude: true }, + ], + }); + return ledger; +} + +/** A new, empty network ledger. */ +export function networkLedger() { + return { + requests: 0, + preview: 0, + previewApi: {}, + external: {}, + fixtureResponses: 0, + probes: {}, + routes: {}, + tileCache: { + hits: 0, + fetched: 0, + failed: 0, + cancelled: 0, + fetchedByHost: {}, + }, + violations: [], + setupErrors: [], + }; +} + +/** + * Switch to `kind` with real input on the Globe Engine picker: a pointer + * click on its label (which focuses the select without opening its menu) + * and keyboard type-ahead. Opens the control panel with a pointer click if + * it is collapsed and collapses it again once the switch has settled + * (`settle: false` returns as soon as the picker reports the change; + * `afterInput` runs right after the key, before anything is read back). + */ +export async function pickEngine( + page, + kind, + { settle = true, afterInput = null } = {}, +) { + assert.ok(ENGINES.includes(kind), kind); + const expanded = () => + page.$eval( + '#control-panel-toggle', + (el) => el.getAttribute('aria-expanded') === 'true', + ); + const opened = !(await expanded()); + if (opened) { + await page.click('#control-panel-toggle').catch(async (error) => { + const state = await page.evaluate(() => { + const el = document.querySelector('#control-panel-toggle'); + const box = el?.getBoundingClientRect(); + const style = el && getComputedStyle(el); + const hit = + box && + document.elementFromPoint( + box.x + box.width / 2, + box.y + box.height / 2, + ); + return { + box: box && [box.x, box.y, box.width, box.height], + display: style?.display, + visibility: style?.visibility, + hidden: el?.closest('[hidden]')?.id ?? null, + hit: hit?.id || hit?.className || null, + body: document.body.className, + }; + }); + throw Error( + `cannot click the control panel toggle (${error.message}): ` + + JSON.stringify(state), + ); + }); + await page.waitForFunction( + () => + document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded') === 'true', + ); + } + await page.waitForSelector('#globe-engine:not([disabled])', { + visible: true, + timeout: 90000, + }); + const from = await page.$eval('#globe-engine', (el) => el.value); + assert.notEqual(from, kind, `the picker already shows ${kind}`); + // Where a pointer can reach the picker: the label's own text (focuses the + // select without opening its menu), else the select itself. + const target = await page.$eval('#globe-engine', (el) => { + const box = el.closest('label').getBoundingClientRect(); + const select = el.getBoundingClientRect(); + // at: the change event (the person's input reached the page); + // idleAt: the picker enabled again after the switch. + window.__qaPick = { busy: false, changed: false, at: null, idleAt: null }; + el.addEventListener( + 'change', + () => { + window.__qaPick.changed = true; + window.__qaPick.at = performance.now(); + }, + { once: true }, + ); + window.__qaPickObserver?.disconnect(); + window.__qaPickObserver = new MutationObserver(() => { + const disabled = document.querySelector('#globe-engine')?.disabled; + if (disabled) window.__qaPick.busy = true; + else if (window.__qaPick.busy && window.__qaPick.idleAt == null) + window.__qaPick.idleAt = performance.now(); + }); + window.__qaPickObserver.observe(document.body, { + attributes: true, + attributeFilter: ['disabled'], + subtree: true, + }); + const describe = (hit) => + hit + ? `${hit.tagName.toLowerCase()}#${hit.id}.${String(hit.className).slice(0, 60)}` + : 'nothing'; + const label = { + x: box.left + Math.max(4, (select.left - box.left) / 2), + y: box.top + box.height / 2, + }; + const onLabel = document.elementFromPoint(label.x, label.y); + if ( + onLabel === el.closest('label') || + (onLabel?.closest('label') === el.closest('label') && onLabel !== el) + ) + return { ...label, via: 'label' }; + const middle = { + x: select.left + select.width / 2, + y: select.top + select.height / 2, + }; + const onSelect = document.elementFromPoint(middle.x, middle.y); + if (onSelect === el) + return { ...middle, via: 'select', labelCoveredBy: describe(onLabel) }; + return { covered: describe(onLabel) + ' / ' + describe(onSelect) }; + }); + assert.ok(!target.covered, 'the picker is covered by ' + target.covered); + await page.mouse.click(target.x, target.y); + assert.equal( + await page.evaluate(() => document.activeElement?.id), + 'globe-engine', + 'the click focuses the picker', + ); + await page.keyboard.type(kind[0]); + // Whatever the person does next, right after the key (no round trip to + // the page in between). + if (afterInput) await afterInput(); + await page.waitForFunction(() => window.__qaPick?.changed, { + timeout: 5000, + polling: 50, + }); + // Headless Chrome opens no menu for a click on the select: the key + // events above reached the page, so there is nothing to dismiss. + const how = { + opened, + clicked: target.via, + ...(target.labelCoveredBy ? { labelCoveredBy: target.labelCoveredBy } : {}), + }; + if (!settle) return how; + await page.waitForFunction( + () => + (window.__qaPick.busy && + !document.querySelector('#globe-engine')?.disabled && + document.querySelector('#loading-screen.hidden') && + !document.querySelector('.engine-switch-frame')) || + document.querySelector('.engine-switch-retry'), + { timeout: 120000, polling: 50 }, + ); + await page.evaluate(() => window.__qaPickObserver?.disconnect()); + Object.assign( + how, + await page.evaluate(() => ({ + at: window.__qaPick.at, + idleAt: window.__qaPick.idleAt, + })), + ); + if (opened && (await expanded().catch(() => false))) { + await page.click('#control-panel-toggle'); + await page.waitForFunction( + () => + document + .querySelector('#control-panel-toggle') + ?.getAttribute('aria-expanded') !== 'true', + ); + } + return how; +} + +/** Wait until `kind` runs, its picker is idle and its view is restored. */ +export async function waitForEngine(page, kind, { timeout = 120000 } = {}) { + await page.waitForFunction( + (expected) => + window.__godsEyeView?.engine?.kind === expected && + document.querySelector('#globe-engine')?.value === expected && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { timeout }, + kind, + ); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); +} + +/** + * The strict context of one scenario run. + * + * @param {object} options + * @param {string} options.origin The preview origin. + * @param {string[]} options.layers The layer ids this scenario covers (the + * runner's --layers wins when given; both must agree). + * @param {string[]} [options.fixtures] URLPattern strings for external + * requests the scenario answers itself. + * @param {{pattern: string, respond: (request: object) => ({status?: number, + * contentType?: string, body?: string|Buffer}|Promise)}[]} [options.routes] + * External URLs the harness answers for the scenario (the pattern is both + * a URLPattern and a CDP Fetch glob, e.g. `https://i.ytimg.com/*`). + * @param {readonly string[]} [options.tileHosts] Hosts the disk cache + * answers (default STRICT_TILE_HOSTS). + * @param {string} [options.name] Scenario name for the manifest. + */ +export function createStrict({ + origin, + layers, + fixtures = [], + routes = [], + tileHosts = STRICT_TILE_HOSTS, + name = path.basename(process.argv[1] ?? 'scenario', '.mjs'), +}) { + const args = strictArguments(); + if (args.layers) + assert.deepEqual( + [...args.layers].sort(), + [...layers].sort(), + `${name} covers ${layers.join(',')}, the runner passed ${args.layers.join(',')}`, + ); + const declared = args.layers ?? layers; + const ledger = networkLedger(); + const assertions = []; + const picks = []; + const timings = []; + const extra = {}; + let finished = false; + + const record = (ids, engine, claim, detail) => { + for (const id of [ids].flat()) { + assert.ok(declared.includes(id), `${id} is not declared by ${name}`); + assert.ok(ENGINES.includes(engine), `unknown engine ${engine}`); + assertions.push({ + layer: id, + engine, + claim, + ...(detail === undefined ? {} : { detail }), + }); + } + }; + const engineOf = (page) => + page.evaluate(() => window.__godsEyeView?.engine?.kind ?? null); + + const strict = { + name, + layers: declared, + ledger, + extra, + timings, + /** Launch Chrome with the policy on every target. */ + async launch(options = {}) { + const browser = await puppeteer.launch({ headless: true, ...options }); + await installStrictNetwork(browser, { + origin, + fixtures, + routes, + tileHosts, + ledger, + }); + return browser; + }, + /** Launch, run `body(browser)`, close, then finish(). */ + async run(options, body) { + const browser = await strict.launch(options); + let failed = null; + try { + await body(browser); + } catch (error) { + failed = error; + } finally { + await browser.close().catch(() => {}); + } + await strict.finish({ failed }); + if (failed) throw failed; + }, + /** Record an assertion that already passed. */ + record, + /** Record a passed assertion on the engine the page runs now. */ + async passed(page, ids, claim, detail) { + const engine = await engineOf(page); + record(ids, engine, claim, detail); + }, + /** Assert `value` and record it on the engine the page runs now. */ + async check(page, ids, claim, value, detail) { + assert.ok( + value, + claim + (detail === undefined ? '' : ' ' + JSON.stringify(detail)), + ); + await strict.passed(page, ids, claim, detail); + }, + /** Switch with real input and log the pick. */ + async pick(page, kind, options) { + const result = await pickEngine(page, kind, options); + picks.push({ to: kind, via: 'pointer+keyboard', ...result }); + return result; + }, + /** + * Switch through the select element without input, only where no + * person can reach the picker (`reason` says why); logged as such. + */ + async selectEngine(page, kind, reason) { + assert.ok(reason, 'say why the picker cannot be used'); + await page.select('#globe-engine', kind); + picks.push({ to: kind, via: 'page.select', reason }); + }, + /** + * Switch with real input in the same document and wait until `kind` + * runs (the picker counterpart of qa-switch-globe.mjs). + */ + async switchTo(page, kind) { + const origin = await page.evaluate(() => performance.timeOrigin); + await strict.pick(page, kind); + await waitForEngine(page, kind); + assert.equal( + await page.evaluate(() => performance.timeOrigin), + origin, + 'same document after switching globes', + ); + }, + /** Write the manifest; fail on violations or missing coverage. */ + async finish({ failed = null } = {}) { + if (finished) return; + finished = true; + const coverage = Object.fromEntries( + declared.map((id) => [ + id, + Object.fromEntries( + ENGINES.map((engine) => [ + engine, + assertions.filter((a) => a.layer === id && a.engine === engine) + .length, + ]), + ), + ]), + ); + const missing = Object.entries(coverage).flatMap(([id, byEngine]) => + ENGINES.filter((engine) => !byEngine[engine]).map( + (engine) => `${id} on ${engine}`, + ), + ); + const network = { + requests: ledger.requests, + preview: ledger.preview, + previewApi: ledger.previewApi, + external: ledger.external, + fixtureResponses: ledger.fixtureResponses, + probes: ledger.probes, + routes: ledger.routes, + tileCache: ledger.tileCache, + violations: ledger.violations, + setupErrors: ledger.setupErrors, + fixtures, + tileHosts: [...tileHosts], + }; + const manifest = { + scenario: name, + argv: process.argv.slice(2), + status: failed ? 'fail' : 'pass', + error: failed ? String(failed?.stack ?? failed).slice(0, 2000) : null, + layers: declared, + coverage, + missing, + picks, + assertions, + network, + timings, + ...extra, + }; + if (args.evidence) { + fs.mkdirSync(path.dirname(args.evidence), { recursive: true }); + fs.writeFileSync( + args.evidence, + JSON.stringify(manifest, null, 2) + '\n', + ); + } + console.log( + 'STRICT', + JSON.stringify({ + scenario: name, + assertions: assertions.length, + coverage, + picks: picks.length, + requests: ledger.requests, + violations: ledger.violations.length, + }), + ); + for (const v of ledger.violations.slice(0, 20)) + console.log('NETWORK VIOLATION', JSON.stringify(v)); + if (failed) return; + assert.equal( + ledger.violations.length, + 0, + 'network policy violations: ' + + JSON.stringify(ledger.violations.slice(0, 5)), + ); + assert.deepEqual(missing, [], 'layers without an assertion'); + assert.ok( + picks.some((pick) => pick.via === 'pointer+keyboard'), + 'switch at least once with real input on the picker', + ); + }, + }; + return strict; +} diff --git a/scripts/qa-strict-hidden-switch.mjs b/scripts/qa-strict-hidden-switch.mjs new file mode 100644 index 000000000..be0ff338d --- /dev/null +++ b/scripts/qa-strict-hidden-switch.mjs @@ -0,0 +1,142 @@ +// Contract: a switch picked just before the tab is hidden finishes in the +// hidden tab without waiting for a frame (a hidden tab draws nothing) and +// without a rollback, keeps the session and adds one history entry; once the +// tab is shown again the new engine draws a non-blank first frame and the +// layers' timers poll again. Both directions, cold and then warm. +// +// node scripts/qa-strict-hidden-switch.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +import { + assertSession, + installFirstFrameProbe, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const errors = []; + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await installFirstFrameProbe(p); + await openApp(p, base.origin, 'navara'); + await startSession(p, ['earthquakes', 'flights']); + const cdp = await p.createCDPSession(); + // Another tab of the same window: bringing it to the front hides ours. + const other = await b.newPage(); + await other.goto('about:blank'); + await p.bringToFront(); + const visibility = () => p.evaluate(() => document.visibilityState); + assert.equal(await visibility(), 'visible'); + const results = []; + for (const [kind, temperature] of [ + ['cesium', 'cold'], + ['navara', 'warm'], + ['cesium', 'warm'], + ]) { + const before = await sessionState(p); + await p.evaluate((kind) => { + window.__qaHidden = { frame: window.__qaFirstFrame(kind), shown: [] }; + window.__qaHidden.frame.then((t) => (window.__qaHidden.drawn = t)); + // When the tab is hidden, as the page sees it: the time and whether + // the switch (busy picker) still runs. + document.addEventListener('visibilitychange', () => + window.__qaHidden.shown.push({ + state: document.visibilityState, + at: performance.now(), + busy: !!document.querySelector('#globe-engine')?.disabled, + }), + ); + }, kind); + // The person picks, then switches tabs before the globe changes: the + // other tab comes to the front right after the key. A slow CPU (20x) + // keeps the switch running past the tab change. + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 20 }); + await strict.pick(p, kind, { + settle: false, + afterInput: () => other.bringToFront(), + }); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: 1 }); + // rAF does not run in a hidden tab: poll on a timer. + await p.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + !document.querySelector('#globe-engine')?.disabled && + !document.querySelector('.engine-switch-frame') && + document.querySelector('#loading-screen.hidden'), + { polling: 250, timeout: 60000 }, + kind, + ); + const hidden = await p.evaluate(() => ({ + state: document.visibilityState, + hide: window.__qaHidden.shown.find((e) => e.state === 'hidden') ?? null, + pick: { at: window.__qaPick.at, idleAt: window.__qaPick.idleAt }, + drawn: window.__qaHidden.drawn ?? null, + })); + await assertSession( + p, + kind, + `${temperature} switch to ${kind} while hidden`, + ); + const after = await sessionState(p); + // Shown again: the first frame arrives, the timers poll again. + const reads = await p.evaluate(() => ({ ...window.__qaReads })); + await p.bringToFront(); + assert.equal(await visibility(), 'visible'); + // A non-blank frame of the new engine once the tab shows again. + const shown = await p.evaluate(async (kind) => { + const at = performance.now(); + const frame = await Promise.race([ + window.__qaFirstFrame(kind), + new Promise((resolve) => setTimeout(() => resolve(null), 10000)), + ]); + return frame && Math.round(frame.presented - at); + }, kind); + await p.waitForFunction( + (reads) => window.__qaReads.flights > reads.flights, + { polling: 250, timeout: 30000 }, + reads, + ); + await rememberEngine(p); + const row = { + kind, + temperature, + hiddenWhileBusy: Boolean(hidden.hide?.busy), + settledWhileHidden: hidden.state === 'hidden', + // From the tab being hidden to the picker being idle again. + settleAfterHideMs: + hidden.hide && hidden.pick.idleAt + ? Math.round(hidden.pick.idleAt - hidden.hide.at) + : null, + // Observed: an engine may draw while hidden (Navara renders a requested + // frame without an animation frame); not part of the contract. + frameWhileHidden: hidden.drawn !== null, + firstFrameAfterShownMs: shown, + history: after.history - before.history, + }; + results.push(row); + assert.ok(row.hiddenWhileBusy, 'the tab is hidden while the switch runs'); + assert.ok(row.settledWhileHidden, 'the switch settles in the hidden tab'); + // A hidden tab draws nothing: no first-frame wait (10 s) there. + assert.ok( + row.settleAfterHideMs !== null && row.settleAfterHideMs < 9000, + JSON.stringify(row), + ); + assert.equal(row.history, 1, 'one history entry'); + assert.ok( + row.firstFrameAfterShownMs !== null, + 'a non-blank frame within 10 s of the tab showing', + ); + console.log('PASS', JSON.stringify(row)); + } + strict.extra.contracts = results; + assert.deepEqual(errors, []); +}); diff --git a/scripts/qa-strict-rapid-switch.mjs b/scripts/qa-strict-rapid-switch.mjs new file mode 100644 index 000000000..8bebe7356 --- /dev/null +++ b/scripts/qa-strict-rapid-switch.mjs @@ -0,0 +1,162 @@ +// Contract: rapid switches. Back-to-back picks with real input, a second +// pointer click on the picker while it is busy, a history visit superseded +// by the opposite visit before its engine is ready, and a pick superseded by +// Back before its engine is ready. After each: the expected engine runs on +// the same session, the history has exactly the expected entries, both +// engines are the warm ones (no third engine), one engine canvas is shown. +// +// node scripts/qa-strict-rapid-switch.mjs http://127.0.0.1:4208 + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + assertSession, + installSessionFixtures, + openApp, + rememberEngine, + sessionState, + startSession, +} from './qa-strict-session.mjs'; + +const base = localPreview(process.argv[2]); +const strict = createStrict({ origin: base.origin, layers: [] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { + const p = await b.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + const errors = []; + p.on('pageerror', (e) => errors.push(e.message)); + await installSessionFixtures(p, { flights: 20 }); + await openApp(p, base.origin, 'navara'); + await startSession(p, ['earthquakes', 'flights']); + const engines = () => + p.evaluate(() => { + const shown = [...window.__qaEngines.values()].filter( + (engine) => engine.host.canvas.isConnected, + ).length; + return { + warm: + window.__qaEngines.get(window.__godsEyeView.engine.kind) === + window.__godsEyeView.engine, + known: window.__qaEngines.size, + shown, + }; + }); + const contract = (name, value, detail) => { + assert.ok(value, name + ' ' + JSON.stringify(detail ?? '')); + strict.extra.contracts ??= []; + strict.extra.contracts.push({ name, pass: true, detail }); + console.log('PASS', name); + }; + + // 1. Back-to-back picks; a second click on the busy picker does nothing. + const start = (await sessionState(p)).history; + for (const [i, kind] of ['cesium', 'navara', 'cesium', 'navara'].entries()) { + await strict.pick(p, kind, { settle: false }); + // The picker is disabled while the engine changes: a second pointer + // click on it (the person trying the other engine) changes nothing. + const box = await p.$eval('#globe-engine', (el) => { + const r = el.getBoundingClientRect(); + return { + x: r.x + r.width / 2, + y: r.y + r.height / 2, + disabled: el.disabled, + }; + }); + await p.mouse.click(box.x, box.y); + await waitForEngine(p, kind); + await rememberEngine(p); + const state = await assertSession(p, kind, `pick ${i + 1} to ${kind}`); + assert.equal(state.history, start + i + 1, 'one entry per pick'); + const shown = await engines(); + assert.ok(shown.warm && shown.shown === 1, JSON.stringify(shown)); + } + contract( + 'four back-to-back picks: each adds one history entry, keeps the session, reuses the warm engines and shows one canvas; a click on the busy picker changes nothing', + true, + { history: (await sessionState(p)).history - start }, + ); + + // A switch in progress, seen from the page: the engine object the page + // exposes is the starting engine until the switch is over. + const progress = () => + p.evaluate(() => { + window.__qaSteps = []; + const mark = (step) => + window.__qaSteps.push({ + step, + engine: window.__godsEyeView?.engine?.kind ?? null, + busy: !!document.querySelector('#globe-engine')?.disabled, + }); + window.__qaMark = mark; + }); + + // 2. A history visit superseded before its engine is ready: Back (to the + // Cesium entry) and, from the page's popstate handling of that Back, at + // once Forward (to the Navara entry), while Back's switch runs. + const length = (await sessionState(p)).history; + await progress(); + await p.evaluate(() => { + window.addEventListener( + 'popstate', + () => { + window.__qaMark('back'); + history.forward(); + }, + { once: true }, + ); + history.back(); + }); + await waitForEngine(p, 'navara'); + await new Promise((resolve) => setTimeout(resolve, 1500)); + await assertSession(p, 'navara', 'Forward supersedes Back'); + const visited = await sessionState(p); + const shown = await engines(); + const backSteps = await p.evaluate(() => window.__qaSteps); + contract( + 'Back then Forward before the Cesium entry is ready: Navara runs on the same session, no entry added, one canvas shown', + visited.history === length && shown.warm && shown.shown === 1, + { steps: backSteps, history: visited.history - length, ...shown }, + ); + + // 3. A pick superseded by Back before its engine is ready: the person + // picks Cesium and presses Back while it starts (Back is issued from the + // page's own change handling of the pick, so it lands mid-switch). The + // pick adds no entry and Back's entry (Cesium, the entry before) wins. + const before = await sessionState(p); + await progress(); + await p.evaluate(() => { + window.addEventListener( + 'change', + (event) => { + if (event.target?.id !== 'globe-engine') return; + window.__qaMark('pick'); + history.back(); + }, + { once: true }, + ); + }); + await strict.pick(p, 'cesium', { settle: false }); + await waitForEngine(p, 'cesium', { timeout: 30000 }); + await new Promise((resolve) => setTimeout(resolve, 1500)); + await assertSession(p, 'cesium', 'Back supersedes the pick'); + const after = await sessionState(p); + const backShown = await engines(); + const pickSteps = await p.evaluate(() => window.__qaSteps); + contract( + 'a pick superseded by Back before ready: Back’s Cesium entry runs on the same session, the pick adds no entry, one canvas shown', + after.history === before.history && + backShown.warm && + backShown.shown === 1 && + pickSteps[0]?.busy, + { steps: pickSteps, history: after.history - before.history, ...backShown }, + ); + // The page still takes an ordinary pick afterwards. + await strict.pick(p, 'navara'); + await waitForEngine(p, 'navara'); + await assertSession(p, 'navara', 'a pick after the races'); + assert.deepEqual(errors, []); +}); diff --git a/scripts/qa-strict-session.mjs b/scripts/qa-strict-session.mjs new file mode 100644 index 000000000..aa1f9aadc --- /dev/null +++ b/scripts/qa-strict-session.mjs @@ -0,0 +1,390 @@ +// Shared pieces of the strict contract scenarios (qa-strict-*.mjs): a fixed +// layer session, its identity checks, the first-frame probe and the WebGL +// object counters. Everything runs on the existing test hooks +// (window.__godsEyeView); nothing here changes the application. + +import assert from 'node:assert/strict'; +import { waitForEngine } from './qa-strict-harness.mjs'; + +/** Austin, 12 km up, OpenStreetMap (answered by the tile disk cache). */ +export const SESSION_HASH = + 'v=2&lat=30.27&lon=-97.74&alt=12000&heading=0&pitch=-90&map=osm&l='; + +/** + * In-page fixtures for the session layers: one Austin earthquake, `flights` + * aircraft in a grid around Austin, `military` aircraft, no vessels. + * The counters on window.__qaReads show every source read. + */ +export async function installSessionFixtures( + page, + { flights = 1, military = 1 } = {}, +) { + await page.evaluateOnNewDocument( + (flights, military) => { + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + window.__qaReads = { quakes: 0, flights: 0, military: 0 }; + const fetch = window.fetch.bind(window); + const side = Math.ceil(Math.sqrt(flights)); + window.fetch = (input, options) => { + const url = new URL(input?.url ?? input, document.baseURI); + const now = Math.floor(Date.now() / 1000); + if (url.hostname === 'earthquake.usgs.gov') { + window.__qaReads.quakes++; + return Promise.resolve( + Response.json({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'qa-austin', + properties: { mag: 4.1, time: Date.now(), place: 'Austin' }, + geometry: { type: 'Point', coordinates: [-97.74, 30.27, 8] }, + }, + ], + }), + ); + } + if (url.pathname === '/api/flights') { + window.__qaReads.flights++; + const states = []; + for (let i = 0; i < flights; i++) + states.push([ + (0xa00000 + i).toString(16), + 'QA' + i, + 'QA', + now, + now, + -99.5 + ((i % side) / side) * 3.5, + 28.5 + (Math.floor(i / side) / side) * 3.5, + 3000 + (i % 40) * 250, + false, + 180 + (i % 50), + (i * 37) % 360, + 0, + null, + 3000 + (i % 40) * 250, + null, + false, + 0, + ]); + return Promise.resolve(Response.json({ time: now, states })); + } + if (url.pathname === '/api/military') { + window.__qaReads.military++; + return Promise.resolve( + Response.json({ + now: Date.now(), + ac: Array.from({ length: military }, (_, i) => ({ + hex: (0xae0000 + i).toString(16), + flight: 'QAMIL' + i, + lon: -97.9 + i * 0.01, + lat: 30.1 + i * 0.01, + alt_geom: 20000, + alt_baro: 20000, + gs: 300, + track: 90, + seen: 0, + seen_pos: 0, + dbFlags: 1, + type: 'adsb_icao', + })), + }), + ); + } + if (url.pathname.startsWith('/api/flights/')) + return Promise.resolve(Response.json({ path: [] })); + if (url.pathname === '/api/vessels') + return Promise.resolve(Response.json({ status: 'live', rows: [] })); + return fetch(input, options); + }; + }, + flights, + military, + ); +} + +/** Open the app on `engine` with the session hash and wait until it runs. */ +export async function openApp(page, origin, engine, hash = SESSION_HASH) { + const url = new URL(origin); + url.search = `?engine=${engine}&welcome=0`; + url.hash = hash; + await page.goto(url.href, { waitUntil: 'domcontentloaded' }); + await waitForEngine(page, engine); + await page.evaluate(() => { + if (window.__GOOGLE_MAPS_API_KEY__ || window.__godsEyeView.tileset) + throw Error('Keyless preview required'); + window.__qaEngines = new Map([ + [window.__godsEyeView.engine.kind, window.__godsEyeView.engine], + ]); + }); +} + +/** Enable `layers` and remember the session's identity. */ +export async function startSession(page, layers) { + await page.evaluate(async (layers) => { + const { dataManager } = window.__godsEyeView; + for (const id of layers) + await dataManager.setEnabled(id, true, { origin: 'user' }); + window.__qaSession = { + manager: dataManager, + models: new Map( + [...dataManager.layers].map(([id, entry]) => [id, entry.module]), + ), + enabled: [...dataManager.getEnabledLayerIds()].sort(), + }; + }, layers); +} + +/** + * The session as the page has it now against the one startSession saw: + * the same manager, every layer model, the same enabled layers. + */ +export function sessionState(page) { + return page.evaluate(() => { + const app = window.__godsEyeView; + const session = window.__qaSession; + if (!app) return { running: false }; + const { dataManager } = app; + return { + running: true, + engine: app.engine.kind, + manager: dataManager === session.manager, + models: + session.models.size === dataManager.layers.size && + [...session.models].every( + ([id, model]) => dataManager.layers.get(id)?.module === model, + ), + enabled: + JSON.stringify([...dataManager.getEnabledLayerIds()].sort()) === + JSON.stringify(session.enabled), + address: new URL(location.href).searchParams.get('engine'), + picker: document.querySelector('#globe-engine')?.value ?? null, + history: history.length, + }; + }); +} + +/** Assert the session is intact on `engine`. */ +export async function assertSession(page, engine, message) { + const state = await sessionState(page); + assert.deepEqual( + { + running: state.running, + engine: state.engine, + manager: state.manager, + models: state.models, + enabled: state.enabled, + address: state.address, + picker: state.picker, + }, + { + running: true, + engine, + manager: true, + models: true, + enabled: true, + address: engine, + picker: engine, + }, + message, + ); + return state; +} + +/** Remember the running engine object under its kind (for warm switches). */ +export function rememberEngine(page) { + return page.evaluate(() => { + const { engine } = window.__godsEyeView; + window.__qaEngines.set(engine.kind, engine); + return engine.kind; + }); +} + +/** + * Install (before navigation) the first-frame probe: + * window.__qaFirstFrame(kind) resolves with the performance.now() times of + * the first frame of the `kind` engine that is drawn into its connected + * canvas with a non-blank picture (postRender, the canvas read back in the + * same task: more than 1% of a 48x48 sample differs from its first pixel), + * and of the animation frame after it (presented). + */ +export async function installFirstFrameProbe(page) { + await page.evaluateOnNewDocument(() => { + window.__qaFirstFrame = (kind) => + new Promise((resolve) => { + const probe = document.createElement('canvas'); + probe.width = probe.height = 48; + const context = probe.getContext('2d', { willReadFrequently: true }); + const nonBlank = (canvas) => { + try { + context.clearRect(0, 0, 48, 48); + context.drawImage(canvas, 0, 0, 48, 48); + const d = context.getImageData(0, 0, 48, 48).data; + let differing = 0; + for (let i = 4; i < d.length; i += 4) + if ( + Math.abs(d[i] - d[0]) + + Math.abs(d[i + 1] - d[1]) + + Math.abs(d[i + 2] - d[2]) > + 24 + ) + differing++; + return differing > 23; + } catch { + return false; + } + }; + let attached = null; + let remove = () => {}; + const attach = (engine) => { + attached = engine; + remove = engine.frames.postRender.addEventListener(() => { + if (!engine.host.canvas.isConnected) return; + if (!nonBlank(engine.host.canvas)) return; + const drawn = performance.now(); + remove(); + clearInterval(timer); + requestAnimationFrame(() => + resolve({ drawn, presented: performance.now() }), + ); + }); + }; + const known = window.__qaEngines?.get(kind); + if (known) attach(known); + // A cold engine: attach as soon as the page exposes it. + const timer = setInterval(() => { + const engine = window.__godsEyeView?.engine; + if (!attached && engine?.kind === kind) attach(engine); + }, 2); + }); + }); +} + +/** + * Count live WebGL objects per context (before navigation): every + * create/delete pair of buffers, textures, programs, shaders, + * framebuffers, renderbuffers and vertex arrays (WebGL2 and the OES + * extension), per context, on window.__qaGl. + */ +export async function installWebGlCounters(page) { + await page.evaluateOnNewDocument(() => { + const KINDS = { + Buffer: 'buffers', + Texture: 'textures', + Program: 'programs', + Shader: 'shaders', + Framebuffer: 'framebuffers', + Renderbuffer: 'renderbuffers', + VertexArray: 'vaos', + }; + const entries = new WeakMap(); + const live = new WeakSet(); + const contexts = []; + window.__qaGl = { contexts }; + const fresh = () => + Object.fromEntries(Object.values(KINDS).map((key) => [key, 0])); + const getContext = HTMLCanvasElement.prototype.getContext; + HTMLCanvasElement.prototype.getContext = function (type, ...args) { + const context = getContext.call(this, type, ...args); + if (context && /webgl/i.test(type) && !entries.has(context)) { + const entry = { + canvas: this, + context, + type, + live: fresh(), + created: fresh(), + }; + entries.set(context, entry); + contexts.push(entry); + } + return context; + }; + const wrap = (target, create, destroy, key, owner = (self) => self) => { + const make = target[create]; + const drop = target[destroy]; + if (typeof make !== 'function') return; + target[create] = function (...args) { + const object = make.apply(this, args); + const entry = entries.get(owner(this)); + if (object && entry) { + entry.live[key]++; + entry.created[key]++; + live.add(object); + } + return object; + }; + target[destroy] = function (object, ...args) { + const entry = entries.get(owner(this)); + if (object && entry && live.has(object)) { + live.delete(object); + entry.live[key]--; + } + return drop.call(this, object, ...args); + }; + }; + for (const proto of [ + WebGLRenderingContext.prototype, + globalThis.WebGL2RenderingContext?.prototype, + ].filter(Boolean)) + for (const [name, key] of Object.entries(KINDS)) + wrap(proto, 'create' + name, 'delete' + name, key); + // WebGL1 vertex arrays come from an extension object. + const getExtension = WebGLRenderingContext.prototype.getExtension; + const vaoOwners = new WeakMap(); + WebGLRenderingContext.prototype.getExtension = function (name, ...args) { + const extension = getExtension.call(this, name, ...args); + if ( + extension && + name === 'OES_vertex_array_object' && + !vaoOwners.has(extension) + ) { + vaoOwners.set(extension, this); + wrap( + extension, + 'createVertexArrayOES', + 'deleteVertexArrayOES', + 'vaos', + (self) => vaoOwners.get(self), + ); + } + return extension; + }; + window.__qaGlCounts = () => { + const engines = window.__qaEngines ?? new Map(); + return contexts.map((entry) => { + let owner = 'other'; + for (const [kind, engine] of engines) + if (engine.host?.canvas === entry.canvas) owner = kind; + return { + owner, + connected: entry.canvas.isConnected, + lost: entry.context.isContextLost(), + total: Object.values(entry.live).reduce((a, b) => a + b, 0), + ...entry.live, + }; + }); + }; + }); +} + +/** Least-squares slope of y over x. */ +export function slope(points) { + const n = points.length; + if (n < 2) return 0; + const mx = points.reduce((s, p) => s + p[0], 0) / n; + const my = points.reduce((s, p) => s + p[1], 0) / n; + let num = 0, + den = 0; + for (const [x, y] of points) { + num += (x - mx) * (y - my); + den += (x - mx) ** 2; + } + return den ? num / den : 0; +} + +/** The q-quantile (nearest rank) of numbers. */ +export function quantile(values, q) { + const sorted = [...values].sort((a, b) => a - b); + if (!sorted.length) return null; + return sorted[Math.max(0, Math.ceil(sorted.length * q) - 1)]; +} diff --git a/scripts/qa-strict-session.test.mjs b/scripts/qa-strict-session.test.mjs new file mode 100644 index 000000000..1522bbe53 --- /dev/null +++ b/scripts/qa-strict-session.test.mjs @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { quantile, slope } from './qa-strict-session.mjs'; +import { strictArguments } from './qa-strict-harness.mjs'; + +test('slope is the least-squares growth per step', () => { + assert.equal(slope([[0, 5]]), 0); + assert.equal( + slope([ + [20, 100], + [50, 103], + [100, 108], + ]).toFixed(3), + '0.100', + ); + assert.equal( + slope([ + [1, 7], + [2, 7], + [3, 7], + ]), + 0, + ); +}); + +test('quantile takes the nearest rank', () => { + assert.equal(quantile([], 0.5), null); + assert.equal(quantile([5, 1, 3, 2, 4], 0.5), 3); + assert.equal( + quantile( + [...Array(100)].map((_, i) => i + 1), + 0.95, + ), + 95, + ); +}); + +test('the runner hands each scenario its layers and its manifest path', () => { + assert.deepEqual( + strictArguments(['node', 'x', '--layers=a,b', '--evidence=/tmp/e.json']), + { layers: ['a', 'b'], evidence: '/tmp/e.json' }, + ); + assert.deepEqual(strictArguments(['node', 'x', '--layers=']), { + layers: null, + evidence: process.env.QA_STRICT_EVIDENCE ?? null, + }); +}); diff --git a/scripts/qa-strict-soak.mjs b/scripts/qa-strict-soak.mjs new file mode 100644 index 000000000..d5d3ffbd9 --- /dev/null +++ b/scripts/qa-strict-soak.mjs @@ -0,0 +1,378 @@ +// Soak and switch timing under a mixed workload: radio playing (750 +// stations), earthquakes, 10 000 flights, 200 military aircraft, wind with +// its GPU particles, and the traffic simulation, all from fixtures. +// +// Each run starts a fresh browser on one engine and switches with real +// input on the picker. Run r starts on Navara when r is odd and on Cesium +// when it is even, so the first switch of each run is a cold one in either +// direction. +// +// "First frame" of a switch: the first postRender of the incoming engine +// whose picture, read back from its canvas in that callback, is not blank, +// then the next animation frame (presented). Timed in the page from the +// picker's change event, so no CDP round trip is included. +// +// Checked on every switch: the same session (manager and every layer +// model), the radio still playing the same station with one play() call. +// With --assert (the long run), after the warm-up (switch 20): live WebGL +// objects per engine context grow by at most 0.1 per switch (least-squares +// slope over the checkpoints) and by at most 64 overall; no new WebGL +// context; JS heap slope at most 10 KiB per switch; no new document; the +// webglcontextlost/restored listener counts on both engine canvases stay as +// they were (each canvas keeps its webglcontextlost listener). +// +// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=6 --swaps=50 +// node scripts/qa-strict-soak.mjs http://127.0.0.1:4208 --runs=1 --swaps=500 --assert + +import assert from 'node:assert/strict'; +import { + createStrict, + localPreview, + waitForEngine, +} from './qa-strict-harness.mjs'; +import { + installFirstFrameProbe, + installSessionFixtures, + installWebGlCounters, + openApp, + quantile, + rememberEngine, + slope, + startSession, +} from './qa-strict-session.mjs'; +import { installRadioFixture, stations } from './fixtures/w2/radio.mjs'; +import { trafficFixtureResponse } from './traffic-fixtures.mjs'; + +const base = localPreview(process.argv[2]); +const option = (name, fallback) => { + const arg = process.argv.find((a) => a.startsWith(`--${name}=`)); + return arg ? Number(arg.slice(name.length + 3)) : fallback; +}; +const runs = option('runs', 6); +const swaps = option('swaps', 50); +const strictMode = process.argv.includes('--assert'); +assert.ok( + Number.isInteger(swaps) && swaps >= 4 && swaps % 2 === 0, + 'even --swaps', +); +const WARM_UP = 20; +const CHECKPOINTS = [ + 2, 10, 20, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500, +]; +const HASH = + 'v=2&lat=30.2670&lon=-97.7440&alt=3000&heading=0&pitch=-90&map=osm&l='; +const LAYERS = [ + 'earthquakes', + 'flights', + 'military', + 'wind', + 'traffic', + 'radio', +]; + +const strict = createStrict({ + origin: base.origin, + layers: [], + // Road tiles of the traffic simulation, from scripts/traffic-fixtures.mjs. + fixtures: ['https://tiles.openfreemap.org/*'], +}); + +/** Wind and traffic fixtures on their own CDP Fetch patterns. */ +async function installWorkloadFixtures(page) { + const session = await page.createCDPSession(); + const grid = (u) => { + const kind = u.searchParams.get('overlay'); + const count = 36 * 19; + const values = new Float32Array(count * (kind ? 3 : 2)); + values.fill(8, 0, count); + values.fill(6, count, count * 2); + if (kind) values.fill(40, count * 2); + return Buffer.from(values.buffer); + }; + session.on('Fetch.requestPaused', async ({ requestId, request }) => { + const u = new URL(request.url); + let answer; + try { + if (u.pathname.startsWith('/api/wind/')) { + const model = u.searchParams.get('model'); + answer = u.pathname.endsWith('/manifest') + ? { + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + model, + overlay: 'none', + grid: { nx: 36, ny: 19, lo1: 0, la1: 90, dx: 10, dy: 10 }, + gridUrl: '/api/wind/grid/' + model + '-qa.bin' + u.search, + cycle: { + validIso: '2026-11-02T12:00:00Z', + runIso: '2026-11-02T06:00:00Z', + }, + }), + } + : { + status: 200, + contentType: 'application/octet-stream', + body: grid(u), + }; + } else if (u.pathname === '/api/tomtom/status') + answer = { + status: 200, + contentType: 'application/json', + body: '{"hasKey":false}', + }; + else answer = trafficFixtureResponse({ url: () => request.url }); + await session.send('Fetch.fulfillRequest', { + requestId, + responseCode: answer.status, + responseHeaders: [ + { name: 'Content-Type', value: answer.contentType }, + { name: 'Access-Control-Allow-Origin', value: '*' }, + ], + body: Buffer.from(answer.body).toString('base64'), + }); + } catch { + await session + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await session.send('Fetch.enable', { + patterns: [ + { urlPattern: `${base.origin}/api/wind/*` }, + { urlPattern: `${base.origin}/api/tomtom/*` }, + { urlPattern: 'https://tiles.openfreemap.org/*' }, + ], + }); +} + +const report = { runs: [], swaps, strict: strictMode }; +strict.extra.soak = report; + +for (let run = 1; run <= runs; run++) { + const start = run % 2 ? 'navara' : 'cesium'; + const browser = await strict.launch({ + args: ['--use-angle=metal', '--enable-gpu'], + }); + const row = { run, start, swaps: [], checkpoints: [], errors: [] }; + report.runs.push(row); + try { + const p = await browser.newPage(); + await p.setViewport({ width: 1280, height: 800 }); + p.on('pageerror', (e) => row.errors.push(e.message.slice(0, 300))); + await installRadioFixture(p); + await installSessionFixtures(p, { flights: 10000, military: 200 }); + await installWebGlCounters(p); + await installFirstFrameProbe(p); + await installWorkloadFixtures(p); + const cdp = await p.createCDPSession(); + await openApp(p, base.origin, start, HASH); + await startSession(p, LAYERS); + await p.waitForFunction( + () => { + const layers = window.__godsEyeView.dataManager.layers; + const stat = (id) => layers.get(id).module.getStats(); + return ( + stat('radio').count === 750 && + stat('flights').count > 0 && + stat('military').count > 0 && + stat('traffic').count >= 10 && + layers.get('wind').module.getDiagnostics().gpu?.ready + ); + }, + { timeout: 90000, polling: 500 }, + ); + await p.evaluate((id) => { + const radio = window.__godsEyeView.dataManager.layers.get('radio').module; + radio.selectStation(id, { autoplay: true, focus: false, origin: 'user' }); + window.__qaRadio.station = id; + }, stations[10].id); + await p.waitForFunction( + () => + window.__qaRadio.plays.length === 1 && + window.__godsEyeView.dataManager.layers.get('radio').module.getUIState() + .audioState === 'playing', + ); + row.workload = await p.evaluate(() => { + const layers = window.__godsEyeView.dataManager.layers; + return Object.fromEntries( + ['flights', 'military', 'traffic', 'radio', 'earthquakes'].map((id) => [ + id, + layers.get(id).module.getStats().count, + ]), + ); + }); + const listeners = async () => { + const { result } = await cdp.send('Runtime.evaluate', { + includeCommandLineAPI: true, + returnByValue: true, + expression: `(() => Object.fromEntries([...window.__qaEngines].map(([kind, engine]) => { + const events = getEventListeners(engine.host.canvas); + return [kind, [events.webglcontextlost?.length ?? 0, events.webglcontextrestored?.length ?? 0]]; + })))()`, + }); + return result.value; + }; + const seen = new Set([start]); + let kind = start; + for (let i = 1; i <= swaps; i++) { + kind = kind === 'navara' ? 'cesium' : 'navara'; + const cold = !seen.has(kind); + seen.add(kind); + await p.evaluate((kind) => { + window.__qaFrame = window.__qaFirstFrame(kind); + }, kind); + const pick = await strict.pick(p, kind); + await waitForEngine(p, kind); + const frame = await p.evaluate(() => window.__qaFrame); + await rememberEngine(p); + const state = await p.evaluate(() => { + const { dataManager } = window.__godsEyeView; + const session = window.__qaSession; + const radio = dataManager.layers.get('radio').module; + return { + session: + dataManager === session.manager && + [...session.models].every( + ([id, model]) => dataManager.layers.get(id)?.module === model, + ), + radio: + window.__qaRadio.plays.length === 1 && + radio.getUIState().audioState === 'playing' && + radio.getUIState().selected?.id === window.__qaRadio.station, + }; + }); + assert.ok( + state.session && state.radio, + `swap ${i}: ${JSON.stringify(state)}`, + ); + row.swaps.push({ + i, + to: kind, + cold, + firstFrameMs: Math.round((frame.presented - pick.at) * 10) / 10, + drawnMs: Math.round((frame.drawn - pick.at) * 10) / 10, + settledMs: + pick.idleAt == null + ? null + : Math.round((pick.idleAt - pick.at) * 10) / 10, + }); + if (CHECKPOINTS.includes(i) || i === swaps) { + await cdp.send('HeapProfiler.collectGarbage'); + const heap = await cdp.send('Runtime.getHeapUsage'); + const dom = await cdp.send('Memory.getDOMCounters'); + row.checkpoints.push({ + swap: i, + engine: kind, + heapMiB: Math.round((heap.usedSize / 1048576) * 100) / 100, + ...dom, + listeners: await listeners(), + webgl: await p.evaluate(() => window.__qaGlCounts()), + }); + } + } + } finally { + await browser.close().catch(() => {}); + } + const warm = (to) => + row.swaps.filter((s) => !s.cold && s.to === to).map((s) => s.firstFrameMs); + row.summary = Object.fromEntries( + ['navara', 'cesium'].map((to) => [ + to, + { + cold: + row.swaps.find((s) => s.cold && s.to === to)?.firstFrameMs ?? null, + warmP50: quantile(warm(to), 0.5), + warmP95: quantile(warm(to), 0.95), + }, + ]), + ); + console.log( + 'RUN', + JSON.stringify({ run, start, ...row.summary, workload: row.workload }), + ); +} + +// Pooled over the runs, per direction. +const pooled = (to, cold) => + report.runs.flatMap((r) => + r.swaps + .filter((s) => s.to === to && s.cold === cold) + .map((s) => s.firstFrameMs), + ); +report.summary = Object.fromEntries( + ['navara', 'cesium'].map((to) => [ + to === 'navara' ? 'cesium→navara' : 'navara→cesium', + { + cold: pooled(to, true), + warmP50: quantile(pooled(to, false), 0.5), + warmP95: quantile(pooled(to, false), 0.95), + warmSwitches: pooled(to, false).length, + }, + ]), +); +console.log('SOAK', JSON.stringify(report.summary)); +strict.timings.push( + ...report.runs.flatMap((r) => r.swaps.map((s) => ({ run: r.run, ...s }))), +); + +if (strictMode) { + const checks = []; + for (const row of report.runs) { + const late = row.checkpoints.filter((c) => c.swap >= WARM_UP); + assert.ok(late.length >= 3, 'enough checkpoints after the warm-up'); + const first = late[0]; + const last = late.at(-1); + for (const owner of ['navara', 'cesium']) { + const totals = late.map((c) => [ + c.swap, + c.webgl + .filter((g) => g.owner === owner && !g.lost) + .reduce((s, g) => s + g.total, 0), + ]); + checks.push({ + check: `${owner} live WebGL objects`, + slopePerSwap: Math.round(slope(totals) * 1000) / 1000, + from: totals[0][1], + to: totals.at(-1)[1], + pass: slope(totals) <= 0.1 && totals.at(-1)[1] - totals[0][1] <= 64, + }); + } + checks.push({ + check: 'WebGL contexts', + from: first.webgl.length, + to: last.webgl.length, + pass: last.webgl.length === first.webgl.length, + }); + const heap = late.map((c) => [c.swap, c.heapMiB * 1024]); + checks.push({ + check: 'JS heap KiB per switch', + slopePerSwap: Math.round(slope(heap) * 100) / 100, + fromMiB: first.heapMiB, + toMiB: last.heapMiB, + pass: slope(heap) <= 10, + }); + checks.push({ + check: 'documents', + from: first.documents, + to: last.documents, + pass: last.documents <= first.documents, + }); + checks.push({ + check: 'context-loss listeners on both canvases', + from: first.listeners, + to: last.listeners, + pass: + JSON.stringify(last.listeners) === JSON.stringify(first.listeners) && + Object.keys(last.listeners).length === 2 && + Object.values(last.listeners).every(([lost]) => lost > 0), + }); + } + report.checks = checks; + for (const check of checks) + console.log(check.pass ? 'PASS' : 'FAIL', JSON.stringify(check)); + await strict.finish({ + failed: checks.some((c) => !c.pass) ? 'soak checks failed' : null, + }); + if (checks.some((c) => !c.pass)) process.exitCode = 1; +} else await strict.finish(); diff --git a/scripts/qa-submarine-cables-engine-switch.mjs b/scripts/qa-submarine-cables-engine-switch.mjs index 005684ae0..ceb4079ab 100644 --- a/scripts/qa-submarine-cables-engine-switch.mjs +++ b/scripts/qa-submarine-cables-engine-switch.mjs @@ -1,26 +1,34 @@ import { fromDegrees } from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { readFileSync } from 'node:fs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=55.3241&lon=10.7995&alt=25000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const CABLES = 'telegeography-submarine-cables'; +// Cables plus landing points in the bundled TeleGeography files. +const FEATURES = ['cable-geo', 'landing-point-geo'] + .map( + (name) => + JSON.parse( + readFileSync( + new URL( + `../src/data/local_data/telegeography_submarine_cables/${name}.json`, + import.meta.url, + ), + 'utf8', + ), + ).features.length, + ) + .reduce((a, b) => a + b, 0); +const strict = createStrict({ + origin: base.origin, + layers: ['telegeography-submarine-cables'], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -40,7 +48,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -63,7 +71,7 @@ try { }); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await p.waitForFunction( @@ -79,10 +87,31 @@ try { }); console.log(kind, check); assert.equal(check.same, true); - assert.equal(check.count, 2629); + assert.equal(check.count, FEATURES); assert.ok(check.referenceLabelCount <= 160); assert.equal(check.error, null); - if (kind === 'navara') { + await strict.passed( + p, + CABLES, + `same model, all ${FEATURES} cables and landing points of the bundled files, at most 160 labels`, + { count: check.count, labels: check.referenceLabelCount }, + ); + { + // Click-picking on this engine, from the starting view: a click on a + // landing point flies the camera to it. + await p.evaluate( + (destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.cancelFlight(); + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-home'); + }, + fromDegrees(10.7995, 55.3241, 25000), + ); + await new Promise((resolve) => setTimeout(resolve, 2500)); const screen = await p.evaluate( (world) => { const { engine } = window.__godsEyeView; @@ -103,13 +132,22 @@ try { }, { timeout: 20000 }, ); + await strict.passed( + p, + CABLES, + 'a click on a landing point flies the camera to it', + ); } } - assert.equal(reads.length, 2, JSON.stringify(reads)); + await strict.check( + p, + CABLES, + 'each bundled file read once', + reads.length === 2, + reads, + ); assert.deepEqual(errors, []); console.log( 'PASS full bundled cable dataset, bounded labels, landing-point fly-to and one read per file', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-traffic-engine-switch.mjs b/scripts/qa-traffic-engine-switch.mjs index 8927b35be..f98fb4598 100644 --- a/scripts/qa-traffic-engine-switch.mjs +++ b/scripts/qa-traffic-engine-switch.mjs @@ -1,27 +1,29 @@ import { trafficFixtureResponse } from './traffic-fixtures.mjs'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; +import { STRICT_TILE_HOSTS } from './qa-map-policy.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=30.2670&lon=-97.7440&alt=1800&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +// The scenario answers the Sentinel-2 host itself (a fixed dark ground) and +// the OpenFreeMap road tiles from its fixtures; the disk cache keeps the +// other tile hosts. +// The fixed fixture gives 1 093 vehicles and 35 congestion corridors with +// flow and 618 simulated vehicles without it; the bands allow 5%. +const BAND = process.argv.includes('--simulation') + ? { vehicles: [587, 649], corridors: [0, 0] } + : { vehicles: [1038, 1148], corridors: [33, 37] }; +const strict = createStrict({ + origin: base.origin, + layers: ['traffic'], + fixtures: ['https://tiles.maps.eox.at/*', 'https://tiles.openfreemap.org/*'], + tileHosts: STRICT_TILE_HOSTS.filter((host) => host !== 'tiles.maps.eox.at'), }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -79,7 +81,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -101,9 +103,16 @@ try { }); const waitTraffic = async () => { try { - await p.waitForFunction(() => window.qaTraffic.getStats().count >= 10, { - timeout: 40000, - }); + // Vehicles, and with flow the congestion corridors, which the layer + // publishes again once the next globe has attached. + await p.waitForFunction( + (keyless) => { + const stats = window.qaTraffic.getStats(); + return stats.count >= 10 && (keyless || stats.heatLines > 0); + }, + { timeout: 40000 }, + keyless, + ); } catch (e) { console.log( 'no traffic', @@ -190,7 +199,7 @@ try { ); for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await waitTraffic(); @@ -228,9 +237,22 @@ try { assert.ok(check.heatLines > 0, 'congestion corridors are published'); if (keyless) assert.equal(check.flowBuckets.sim, check.count); else assert.ok(check.flowBuckets.slow + check.flowBuckets.jam > 0); + await strict.check( + p, + 'traffic', + keyless + ? `same module, parameters and vehicle ids; ${BAND.vehicles.join('-')} simulated vehicles, no flow` + : `same module, parameters and vehicle ids; ${BAND.vehicles.join('-')} vehicles with flow, ${BAND.corridors.join('-')} congestion corridors`, + check.count >= BAND.vehicles[0] && + check.count <= BAND.vehicles[1] && + check.heatLines >= BAND.corridors[0] && + check.heatLines <= BAND.corridors[1], + { count: check.count, corridors: check.heatLines }, + ); } if (keyless) assert.ok(![...reads.keys()].some((u) => u.includes('/tomtom/flow/'))); + // Compare the same view with the layer shown and hidden, vehicle by // vehicle. const raw = async () => @@ -333,6 +355,16 @@ try { `the population after the round trip (${count}) stays within 80-125% of the ${population} before it`, ); assert.ok(aboveGround >= 12, 'at least 12 vehicles above ground in view'); + await strict.check( + p, + 'traffic', + 'after the round trip the population stays at 80-125% and at least 90% of the vehicles above ground in view are drawn where the layer places them', + visible.drawn >= 0.9 * aboveGround && + [settledCount, finalCount].every( + (count) => count >= 0.8 * population && count <= 1.25 * population, + ), + { population, settledCount, finalCount, aboveGround, drawn: visible.drawn }, + ); assert.ok( visible.drawn >= 0.9 * aboveGround, 'at least 90% of vehicles above the ground are drawn where the layer places them', @@ -345,6 +377,4 @@ try { console.log( 'PASS kept the traffic module, parameters, flow and vehicle identities across both globes; vehicles in view are drawn where the layer places them', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-urban-engine-switch.mjs b/scripts/qa-urban-engine-switch.mjs index 5f94d2ef0..597ef41c7 100644 --- a/scripts/qa-urban-engine-switch.mjs +++ b/scripts/qa-urban-engine-switch.mjs @@ -1,29 +1,38 @@ -import puppeteer from 'puppeteer'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; -import { switchGlobe } from './qa-switch-globe.mjs'; +import { fromDegrees } from '../src/geo/cartesian3.js'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { installTransitFixture } from './fixtures/w2/transit.mjs'; -import { installCombinedFixture } from './fixtures/w2/combined.mjs'; -const url = new URL(process.argv[2]), +import { + combinedPayloads, + installCombinedFixture, +} from './fixtures/w2/combined.mjs'; +const url = localPreview(process.argv[2]), bus = process.argv.includes('--transit'), id = bus ? 'transit' : 'bikeshare'; -if ( - !['localhost', '127.0.0.1'].includes(url.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - url.port, - ) -) - throw Error('Supply local keyless preview URL'); +// Boston, where the fixtures put the city's stations and vehicles. Transit +// starts outside every feed region (central Vermont) and is approached. +const CITY = { lat: 42.3554, lon: -71.0605, alt: 2700 }; +const start = bus ? { lat: 44.2, lon: -72.6, alt: 2700 } : CITY; url.search = '?engine=navara&welcome=0'; -url.hash = - 'v=2&lat=42.3554&lon=-71.0605&alt=2700&pitch=-90&heading=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +url.hash = `v=2&lat=${start.lat}&lon=${start.lon}&alt=${start.alt}&pitch=-90&heading=0&map=sentinel2&l=`; +// Every station of the bikeshare fixture reports 7 bikes and 13 docks; the +// transit fixture has 12 fresh vehicles (and one stale, dropped). +const STATIONS = combinedPayloads.bikes.length; +const VEHICLES = 12; +const strict = createStrict({ + origin: url.origin, + layers: [id], + // installCombinedFixture answers these hosts too. + fixtures: bus + ? [] + : [ + 'https://earthquake.usgs.gov/*', + 'https://tiles.dontgetflocked.com/*', + '*://*.earthdata.nasa.gov/*', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1280, height: 900 }); const errors = []; p.on('pageerror', (e) => { @@ -48,35 +57,141 @@ try { window.qaUrban = dataManager.layers.get(id).module; await dataManager.setEnabled(id, true, { origin: 'user' }); }, id); + const vehicleReads = () => + fx.requests.filter((path) => path.startsWith('/api/transit/vehicles/')) + .length; + if (bus) { + // Outside every feed region the layer waits and reads no feed. + await new Promise((resolve) => setTimeout(resolve, 3000)); + const outside = await p.evaluate(() => window.qaUrban.getStats()); + await strict.check( + p, + 'transit', + 'outside every feed region: no vehicles, no feed read, asks to move to a covered region', + outside.count === 0 && + outside.status === 'zoom-in' && + vehicleReads() === 0, + { stats: outside, reads: vehicleReads() }, + ); + // Approach the city: the MBTA feed turns on and its vehicles arrive. + await p.evaluate( + (destination) => { + const { camera, render } = window.__godsEyeView.engine; + camera.setView({ + destination, + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + render.request('qa-approach'); + }, + fromDegrees(CITY.lon, CITY.lat, CITY.alt), + ); + await p.waitForFunction( + (n) => + window.qaUrban.getStats().count === n && + window.qaUrban.getStats().feeds?.includes('mbta'), + { timeout: 30000 }, + VEHICLES, + ); + await strict.check( + p, + 'transit', + `approaching Boston from outside turns on the MBTA feed: ${VEHICLES} vehicles`, + vehicleReads() > 0, + { reads: vehicleReads() }, + ); + } await p.waitForFunction(() => window.qaUrban.getStats().count > 0, { timeout: 30000, }); - const handle = await p.waitForFunction( - () => { - const a = window.__godsEyeView; - for (const r of window.qaUrban.getDetectableObjects({ - maxCount: 500, - seed: 0, - })) { - const s = a.engine.projector.worldToWindow(r.position); - if (!s || s.x < 420 || s.x > 860 || s.y < 205 || s.y > 670) continue; - const pk = a.engine.picker.pick(s); - if (pk?.id === r.sourceId || pk?.id?.id === r.sourceId) - return { x: s.x, y: s.y, id: r.sourceId }; - } - return false; - }, - { timeout: 30000 }, - ); - const target = await handle.jsonValue(); + // A station or vehicle the engine's own picker returns, uncovered. + const findTarget = (key = null) => + p + .waitForFunction( + (key) => { + const a = window.__godsEyeView; + const canvas = a.engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (const r of window.qaUrban.getDetectableObjects({ + maxCount: 500, + seed: 0, + })) { + if (key && r.sourceId !== key) continue; + const s = a.engine.projector.worldToWindow(r.position); + if (!s || s.x < 420 || s.x > 860 || s.y < 205 || s.y > 600) + continue; + if ( + !canvas.contains( + document.elementFromPoint(box.left + s.x, box.top + s.y), + ) + ) + continue; + const pk = a.engine.picker.pick(s); + if (pk?.id === r.sourceId || pk?.id?.id === r.sourceId) + return { x: box.left + s.x, y: box.top + s.y, id: r.sourceId }; + } + return false; + }, + { timeout: 30000 }, + key, + ) + .then((handle) => handle.jsonValue()); + const selected = (key) => + p.waitForFunction( + (key) => + window.qaUrban + .getDetectableObjects({ maxCount: 500, seed: 0 }) + .some((r) => r.sourceId === key && r.skipLabel), + { timeout: 20000 }, + key, + ); + // A click on empty ground (nothing picked around it, nothing covering it). + const clickEmpty = async () => { + const handle = await p.waitForFunction( + () => { + const { engine } = window.__godsEyeView; + const canvas = engine.host.canvas; + const box = canvas.getBoundingClientRect(); + for (let y = 230; y < 600; y += 9) + for (let x = 440; x < 840; x += 9) { + if ( + !canvas.contains( + document.elementFromPoint(box.left + x, box.top + y), + ) + ) + continue; + let free = true; + for (const [dx, dy] of [ + [0, 0], + [8, 0], + [-8, 0], + [0, 8], + [0, -8], + ]) + if (engine.picker.pick({ x: x + dx, y: y + dy })) free = false; + if (free) return { x: box.left + x, y: box.top + y }; + } + return false; + }, + { timeout: 15000 }, + ); + const at = await handle.jsonValue(); + await p.mouse.click(at.x, at.y); + await p.waitForFunction( + () => + !window.qaUrban + .getDetectableObjects({ maxCount: 500, seed: 0 }) + .some((r) => r.skipLabel), + { timeout: 15000 }, + ); + }; + const target = await findTarget(); await p.mouse.click(target.x, target.y); - await p.waitForFunction( - (key) => - window.qaUrban - .getDetectableObjects({ maxCount: 500, seed: 0 }) - .some((r) => r.sourceId === key && r.skipLabel), - {}, - target.id, + await selected(target.id); + await strict.passed( + p, + id, + bus ? 'a click on a vehicle selects it' : 'a click on a station selects it', + target, ); const reads = () => fx.requests.filter((path) => @@ -88,20 +203,43 @@ try { await p.waitForFunction( () => window.qaUrban.getDetectableObjects({ maxCount: 500 }).length > 0, ); - const original = reads(); + let original = reads(); const first = await p.evaluate(() => window.qaUrban.getAnalystRecords(500).map((v) => v.id), ); - for (const kind of ['cesium', 'navara']) { - await switchGlobe(p, kind); - await p.waitForFunction( - (key) => - window.qaUrban - .getDetectableObjects({ maxCount: 500, seed: 0 }) - .some((r) => r.sourceId === key && r.skipLabel), - { timeout: 20000 }, - target.id, + const checkData = async () => { + const stats = await p.evaluate(() => window.qaUrban.getStats()); + if (bus) { + await strict.check( + p, + 'transit', + `${VEHICLES} fresh vehicles (the stale one dropped)`, + stats.count === VEHICLES, + stats.count, + ); + return; + } + const rows = await p.evaluate(() => + window.qaUrban.getAnalystRecords(1000).map((r) => ({ + bikes: r.bikesAvailable, + docks: r.docksAvailable, + renting: r.renting, + })), ); + await strict.check( + p, + 'bikeshare', + `${STATIONS} stations, each with 7 bikes and 13 docks available and renting`, + stats.count === STATIONS && + rows.length === STATIONS && + rows.every((r) => r.bikes === 7 && r.docks === 13 && r.renting), + { count: stats.count, rows: rows.length, sample: rows[0] }, + ); + }; + await checkData(); + for (const kind of ['cesium', 'navara']) { + await strict.switchTo(p, kind); + await selected(target.id); const v = await p.evaluate( (id) => ({ same: @@ -116,12 +254,32 @@ try { assert.ok(v.same); assert.deepEqual(v.ids, first); assert.equal(reads(), original); + await strict.passed( + p, + id, + 'same model, same records, the clicked object still selected, no source read again', + ); + await checkData(); + if (kind === 'cesium') { + // Click-picking on Cesium: clear with a click on empty ground, then + // click the same object again. + await clickEmpty(); + const again = await findTarget(target.id); + await p.mouse.click(again.x, again.y); + await selected(target.id); + await strict.passed( + p, + id, + 'a click on empty ground clears the selection; a click on the object selects it again', + again, + ); + // Selecting a vehicle again reads its trail again. + original = reads(); + } } assert.deepEqual(errors, []); await fx.dispose?.(); console.log( 'PASS viewport activation, real pick and original city data across two switches', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-w2-imagery.mjs b/scripts/qa-w2-imagery.mjs index 50f0ca2c1..3c7015e8e 100644 --- a/scripts/qa-w2-imagery.mjs +++ b/scripts/qa-w2-imagery.mjs @@ -1,9 +1,6 @@ -import {switchGlobe} from './qa-switch-globe.mjs'; import fs from 'node:fs/promises'; -import puppeteer from 'puppeteer'; import sharp from 'sharp'; -import { installTileCache } from './qa/tileCache.mjs'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; import { isExpectedPreviewSetupError, redact } from './qa-browserEvidence.mjs'; const args = process.argv.slice(2); @@ -26,8 +23,17 @@ const report = { }; let failLandsat = false, failedTiles = 0; +// The engine the page runs, for the strict assertion manifest. +let engine = null; +const strict = createStrict({ + origin: localPreview(url).origin, + layers: ['recent-imagery'], + // The scenario answers the NASA catalog and imagery tiles itself. + fixtures: ['*://*.earthdata.nasa.gov/*'], +}); const check = (name, value) => { report.checks.push({ name, pass: Boolean(value) }); + if (value && engine) strict.record('recent-imagery', engine, name); console.log(`${value ? 'PASS' : 'FAIL'} ${name}`); }; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -38,7 +44,7 @@ const colors = await Promise.all( .toBuffer(), ), ); -const browser = await puppeteer.launch({ +const browser = await strict.launch({ headless: true, args: [ '--no-sandbox', @@ -51,8 +57,6 @@ const browser = await puppeteer.launch({ try { const page = await browser.newPage(); await page.setViewport({ width: 1280, height: 800, deviceScaleFactor: 1 }); - await blockPaidQaMaps(page); - await installTileCache(page); page.on('pageerror', (e) => report.errors.push(redact(e.message))); page.on('console', (message) => { if (message.type() !== 'error') return; @@ -152,6 +156,7 @@ try { () => window.__gevRecentImagery && window.__godsEyeView, { timeout: 90000 }, ); + engine = await page.evaluate(() => window.__godsEyeView.engine.kind); await page.evaluate(async () => { const app = window.__godsEyeView; await app.styleManager.initialRestorePromise; @@ -231,20 +236,31 @@ try { window.qaImageryOwner = window.__gevRecentImagery.layer; return window.qaImageryOwner.diagnostics().pins; }); - const reads = () => report.requests.filter(r => r.host === 'cmr.earthdata.nasa.gov').length; + const reads = () => + report.requests.filter((r) => r.host === 'cmr.earthdata.nasa.gov').length; const expectedReads = reads(); const initial = await page.evaluate(() => window.__godsEyeView.engine.kind); for (const kind of [initial === 'navara' ? 'cesium' : 'navara', initial]) { - await switchGlobe(page, kind); + await strict.switchTo(page, kind); + engine = kind; const state = await page.evaluate(() => ({ same: window.__gevRecentImagery.layer === window.qaImageryOwner, ...window.qaImageryOwner.diagnostics(), })); - check(kind+' holds both original pins',state.same && state.mode === 'ab' && state.pins.a === first.a && state.pins.b === first.b); + check( + kind + ' holds both original pins', + state.same && + state.mode === 'ab' && + state.pins.a === first.a && + state.pins.b === first.b, + ); await sleep(1800); - const pixels=await capture('switch-'+kind); - check(kind+' renders both pinned dates',red(pixels.left) && green(pixels.right)); - check(kind+' reuses the STAC search', reads() === expectedReads); + const pixels = await capture('switch-' + kind); + check( + kind + ' renders both pinned dates', + red(pixels.left) && green(pixels.right), + ); + check(kind + ' reuses the STAC search', reads() === expectedReads); } } await page.evaluate(() => window.__gevRecentImagery.layer.swapSides()); @@ -452,4 +468,6 @@ try { await browser.close(); await fs.writeFile(output, JSON.stringify(report, null, 2)); } -if (report.checks.some((c) => !c.pass)) process.exitCode = 1; +const failed = report.checks.some((c) => !c.pass); +await strict.finish({ failed: failed ? 'imagery checks failed' : null }); +if (failed) process.exitCode = 1; diff --git a/scripts/qa-weather-engine-switch.mjs b/scripts/qa-weather-engine-switch.mjs index 3514c23a2..a6f9802a4 100644 --- a/scripts/qa-weather-engine-switch.mjs +++ b/scripts/qa-weather-engine-switch.mjs @@ -1,27 +1,23 @@ -import {fromDegrees} from '../src/geo/cartesian3.js'; -import puppeteer from 'puppeteer'; +import { fromDegrees } from '../src/geo/cartesian3.js'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=46000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], +const strict = createStrict({ + origin: base.origin, + layers: [ + 'weather-radar', + 'weather-satellite', + 'weather-lightning', + 'weather-cyclones', + ], }); -try { +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -32,27 +28,81 @@ try { const ids = ['weather-radar', 'weather-satellite', 'weather-lightning']; const images = new Map(), reads = new Map(); - let bulletin = '10', cycloneReads = 0; + let bulletin = '10', + cycloneReads = 0; function advisory() { const forecast = bulletin === '10'; const storm = { - id:'al012026', name:'QA storm', basin:'AL', classification:'TS', position:{longitude:-79.38, latitude:43.65}, - advisoryNumber:bulletin, issuedAt:times[1], positionAt:times[1], windKt:40, pressureHpa:1000, - movement:{directionDegrees:45,speedKt:10}, advisoryUrl:null,outlookUrl:null, - geometryStatus:forecast?'current':'pending', geometryAdvisoryNumber:forecast?bulletin:null, - track:forecast?{type:'LineString',coordinates:[[-79.38,43.65],[-79.31,43.68]]}:null, - cone:forecast?{type:'Polygon',coordinates:[[[-79.35,43.63],[-79.28,43.63],[-79.28,43.70],[-79.35,43.63]]]}:null, - forecastPoints:forecast?[{tauHours:24,position:{longitude:-79.31,latitude:43.68},windKt:50,gustKt:60}]:[], + id: 'al012026', + name: 'QA storm', + basin: 'AL', + classification: 'TS', + position: { longitude: -79.38, latitude: 43.65 }, + advisoryNumber: bulletin, + issuedAt: times[1], + positionAt: times[1], + windKt: 40, + pressureHpa: 1000, + movement: { directionDegrees: 45, speedKt: 10 }, + advisoryUrl: null, + outlookUrl: null, + geometryStatus: forecast ? 'current' : 'pending', + geometryAdvisoryNumber: forecast ? bulletin : null, + track: forecast + ? { + type: 'LineString', + coordinates: [ + [-79.38, 43.65], + [-79.31, 43.68], + ], + } + : null, + cone: forecast + ? { + type: 'Polygon', + coordinates: [ + [ + [-79.35, 43.63], + [-79.28, 43.63], + [-79.28, 43.7], + [-79.35, 43.63], + ], + ], + } + : null, + forecastPoints: forecast + ? [ + { + tauHours: 24, + position: { longitude: -79.31, latitude: 43.68 }, + windKt: 50, + gustKt: 60, + }, + ] + : [], + }; + return { + schemaVersion: 1, + source: 'QA NOAA', + attribution: 'NOAA fixture', + coverage: 'Atlantic', + fetchedAt: Date.parse(times[1]), + stale: false, + unavailable: false, + reason: null, + storms: [storm], }; - return {schemaVersion:1,source:'QA NOAA',attribution:'NOAA fixture',coverage:'Atlantic',fetchedAt:Date.parse(times[1]), - stale:false,unavailable:false,reason:null,storms:[storm]}; } await p.setRequestInterception(true); p.on('request', async (req) => { const url = new URL(req.url()); if (url.pathname === '/api/cyclones') { cycloneReads++; - return req.respond({status:200,contentType:'application/json',body:JSON.stringify(advisory())}); + return req.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(advisory()), + }); } if (!url.pathname.startsWith('/api/weather/')) return req.continue(); const product = url.searchParams.get('product'); @@ -121,7 +171,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -188,6 +238,7 @@ try { latest[2] > latest[0] + 50, 'lightning must paint cyan over radar', ); + await strict.passed(p, id, 'latest frame paints in stacking order', latest); console.log('latest', id, latest); } await p.evaluate((ids) => { @@ -200,7 +251,7 @@ try { const polls = [...reads.entries()]; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await waitTime(ids, times[0]); @@ -238,6 +289,12 @@ try { polls, 'A globe change must not refetch the observation manifest', ); + await strict.passed( + p, + ids, + 'same entry, shared history clock on the earlier frame, no manifest refetch, lightning above radar', + rgb, + ); console.log( 'PASS', kind, @@ -262,10 +319,11 @@ try { await p.evaluate(() => window.qaRestoreSurface()); await p.evaluate(async (ids) => { const manager = window.__godsEyeView.dataManager; - for (const id of ids) await manager.setEnabled(id, false, {origin:'user'}); - await manager.setEnabled('weather-cyclones', true, {origin:'user'}); + for (const id of ids) + await manager.setEnabled(id, false, { origin: 'user' }); + await manager.setEnabled('weather-cyclones', true, { origin: 'user' }); window.qaStorm = manager.layers.get('weather-cyclones').module; - },ids); + }, ids); const click = async (lon, lat, insideCone) => { const hit = await p.waitForFunction( ({ lon, lat, base, insideCone }) => { @@ -303,30 +361,56 @@ try { await hit.dispose(); }; const before = cycloneReads; - for (const kind of ['navara','cesium','navara']) { - if (await p.evaluate(()=>window.__godsEyeView.engine.kind) !== kind) { - await p.select('#globe-engine',kind); + for (const kind of ['navara', 'cesium', 'navara']) { + if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { + await p.select('#globe-engine', kind); await ready(kind); } - await p.waitForFunction(() => window.__godsEyeView.dataManager.layers.get('weather-cyclones').module === window.qaStorm && - window.qaStorm.getDiagnostics().selectedId === 'al012026'); - const d = await p.evaluate(()=>window.qaStorm.getDiagnostics()); - assert.deepEqual([d.storms,d.tracks,d.cones,d.forecastPoints],[1,1,1,1],kind); - await click(-79.43,43.59,false); - await p.waitForFunction(()=>window.qaStorm.getDiagnostics().selectedId === null); - await click(-79.30,43.643,true); - await p.waitForFunction(()=>window.qaStorm.getDiagnostics().selectedId === 'al012026'); - assert.equal(cycloneReads,before); - console.log('PASS',kind,'pick advisory cone and retain selected storm'); + await p.waitForFunction( + () => + window.__godsEyeView.dataManager.layers.get('weather-cyclones') + .module === window.qaStorm && + window.qaStorm.getDiagnostics().selectedId === 'al012026', + ); + const d = await p.evaluate(() => window.qaStorm.getDiagnostics()); + assert.deepEqual( + [d.storms, d.tracks, d.cones, d.forecastPoints], + [1, 1, 1, 1], + kind, + ); + await click(-79.43, 43.59, false); + await p.waitForFunction( + () => window.qaStorm.getDiagnostics().selectedId === null, + ); + await click(-79.3, 43.643, true); + await p.waitForFunction( + () => window.qaStorm.getDiagnostics().selectedId === 'al012026', + ); + assert.equal(cycloneReads, before); + await strict.passed( + p, + 'weather-cyclones', + 'storm, track, cone and forecast point; clicks outside and inside the cone deselect and select; no refetch', + ); + console.log('PASS', kind, 'pick advisory cone and retain selected storm'); } bulletin = '11'; const next = await p.evaluate(async () => { await window.qaStorm.update(); return window.qaStorm.getDiagnostics(); }); - assert.deepEqual([next.storms,next.tracks,next.cones,next.forecastPoints,next.selectedId],[1,0,0,0,'al012026']); - console.log('PASS advisory 11 removes expired geometry while retaining the storm'); + assert.deepEqual( + [ + next.storms, + next.tracks, + next.cones, + next.forecastPoints, + next.selectedId, + ], + [1, 0, 0, 0, 'al012026'], + ); + console.log( + 'PASS advisory 11 removes expired geometry while retaining the storm', + ); assert.deepEqual(errors, []); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa-wind-engine-switch.mjs b/scripts/qa-wind-engine-switch.mjs index 7a2eb724d..0c31e1bcc 100644 --- a/scripts/qa-wind-engine-switch.mjs +++ b/scripts/qa-wind-engine-switch.mjs @@ -1,26 +1,16 @@ -import puppeteer from 'puppeteer'; import sharp from 'sharp'; import assert from 'node:assert/strict'; -import { blockPaidQaMaps } from './qa-map-policy.mjs'; +import { createStrict, localPreview } from './qa-strict-harness.mjs'; -const base = new URL(process.argv[2]); -if ( - !['localhost', '127.0.0.1'].includes(base.hostname) || - ['', '4173', '4185', '4186', '4187', '4188', '4189', '4194'].includes( - base.port, - ) -) - throw new Error('Supply an unreserved local keyless preview URL'); +const base = localPreview(process.argv[2]); base.search = '?engine=navara&welcome=0&look=cinematic'; base.hash = 'v=2&lat=43.6500&lon=-79.3800&alt=2000000&heading=0&pitch=-90&roll=0&map=sentinel2&l='; -const b = await puppeteer.launch({ - headless: true, - args: ['--use-angle=metal', '--enable-gpu'], -}); -try { +// The fixture field is uniform, so the GPU seeds a fixed particle count. +const WIND_PATHS = 7200; +const strict = createStrict({ origin: base.origin, layers: ['wind'] }); +await strict.run({ args: ['--use-angle=metal', '--enable-gpu'] }, async (b) => { const p = await b.newPage(); - await blockPaidQaMaps(p); await p.setViewport({ width: 1024, height: 768 }); const errors = []; p.on('pageerror', (e) => { @@ -83,7 +73,7 @@ try { const ready = async (kind) => { await p.waitForFunction( (k) => - window.__godsEyeView?.engine.kind === k && + window.__godsEyeView?.engine?.kind === k && !document.querySelector('#globe-engine')?.disabled && !document.querySelector('.engine-switch-frame') && document.querySelector('#loading-screen.hidden'), @@ -113,7 +103,7 @@ try { const baseline = reads.length; for (const kind of ['navara', 'cesium', 'navara']) { if ((await p.evaluate(() => window.__godsEyeView.engine.kind)) !== kind) { - await p.select('#globe-engine', kind); + await strict.pick(p, kind); await ready(kind); } await p.waitForFunction( @@ -133,11 +123,27 @@ try { reading: mod.getRowControls().summary.reading, }; }); - assert.equal(check.same, true); - assert.equal(check.params.paused, true); - assert.ok(check.gpu > 1000); - assert.equal(check.reading.scalarValue, '40.0 °C'); - assert.equal(check.reading.from, 'SW'); + await strict.check(p, 'wind', 'same wind model', check.same === true); + await strict.check( + p, + 'wind', + 'paused parameter retained', + check.params.paused === true, + ); + await strict.check( + p, + 'wind', + `${WIND_PATHS} GPU paths`, + check.gpu === WIND_PATHS, + check.gpu, + ); + await strict.check( + p, + 'wind', + 'point reading 40.0 °C from SW', + check.reading.scalarValue === '40.0 °C' && check.reading.from === 'SW', + check.reading, + ); const sample = await sharp(await p.screenshot()) .extract({ left: 530, top: 390, width: 24, height: 24 }) .removeAlpha() @@ -149,22 +155,40 @@ try { return sum / (sample.length / 3); }); console.log(kind, check.gpu, check.reading.wind, [r, g, b]); - assert.ok( + await strict.check( + p, + 'wind', + 'temperature field visible', r > g * 1.35 && r > b * 1.35, - kind + ' temperature field visible', + [r, g, b], + ); + await strict.check( + p, + 'wind', + 'no forecast grid read on the switch', + reads.length === baseline, + reads.length - baseline, ); - assert.equal(reads.length, baseline); } await p.evaluate(() => window.qaWind.setParams({ paused: false })); - await p.waitForFunction(() => window.qaWind.getDiagnostics().gpu.phaseSeconds > 0.2); + await p.waitForFunction( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds > 0.2, + ); await p.evaluate(() => window.qaWind.setParams({ paused: true })); - const frozen = await p.evaluate(() => window.qaWind.getDiagnostics().gpu.phaseSeconds); + const frozen = await p.evaluate( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds, + ); await new Promise((resolve) => setTimeout(resolve, 250)); - assert.equal(await p.evaluate(() => window.qaWind.getDiagnostics().gpu.phaseSeconds), frozen); + await strict.check( + p, + 'wind', + 'pause freezes the particle phase', + (await p.evaluate( + () => window.qaWind.getDiagnostics().gpu.phaseSeconds, + )) === frozen, + ); assert.deepEqual(errors, []); console.log( 'PASS retained forecast grid, pause, temperature field and sampled wind on both engines', ); -} finally { - await b.close(); -} +}); diff --git a/scripts/qa/tileCache.mjs b/scripts/qa/tileCache.mjs index ec1aba3ff..86070ade0 100644 --- a/scripts/qa/tileCache.mjs +++ b/scripts/qa/tileCache.mjs @@ -27,7 +27,7 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); * The cache entry for a GET of `url` with `accept` (terrain answers by its * Accept header), or null for a host that is not cached. */ -export function tileCacheFile(url, accept, dir) { +export function tileCacheFile(url, accept, dir, hosts = CACHED_TILE_HOSTS) { let host; try { const parsed = new URL(url); @@ -36,7 +36,7 @@ export function tileCacheFile(url, accept, dir) { } catch { return null; } - if (!CACHED_TILE_HOSTS.includes(host)) return null; + if (!hosts.includes(host)) return null; const key = createHash('sha256') .update(`${url}\n${accept ?? ''}`) .digest('hex') @@ -121,12 +121,15 @@ export function createMissLoader({ } /** - * Serves the page's requests to CACHED_TILE_HOSTS from the cache (filling - * it on a miss). Returns the counts. + * Serves the page's requests to CACHED_TILE_HOSTS (or `hosts`) from the + * cache (filling it on a miss). Returns the counts. */ export async function installTileCache( page, - { dir = path.join(os.tmpdir(), 'gev-qa-tiles') } = {}, + { + dir = path.join(os.tmpdir(), 'gev-qa-tiles'), + hosts = CACHED_TILE_HOSTS, + } = {}, ) { fs.mkdirSync(dir, { recursive: true }); const stats = { hits: 0, fetched: 0, cancelled: 0, failed: 0, errors: {} }; @@ -137,7 +140,9 @@ export async function installTileCache( ([name]) => name.toLowerCase() === 'accept', )?.[1]; const file = - request.method === 'GET' ? tileCacheFile(request.url, accept, dir) : null; + request.method === 'GET' + ? tileCacheFile(request.url, accept, dir, hosts) + : null; try { if (!file) return await client.send('Fetch.continueRequest', { requestId }); @@ -187,7 +192,7 @@ export async function installTileCache( } }); await client.send('Fetch.enable', { - patterns: CACHED_TILE_HOSTS.map((host) => ({ + patterns: hosts.map((host) => ({ urlPattern: `https://${host}/*`, })), });