diff --git a/scripts/qa-app-once.mjs b/scripts/qa-app-once.mjs new file mode 100644 index 000000000..c255502b1 --- /dev/null +++ b/scripts/qa-app-once.mjs @@ -0,0 +1,234 @@ +// The application is created once per page: switching engines detaches and +// attaches only the engine-bound parts. This gate runs three Cesium→Navara→ +// Cesium round trips and checks that every session object keeps its identity, +// an open panel, typed text and the focused field survive, Escape still does +// what it did before the switches, window/document listeners do not grow, +// and a failed candidate rolls back onto the same session objects. +// +// node scripts/qa-app-once.mjs http://127.0.0.1:/ [--report] +// `--report` prints the measurements without asserting them. +import assert from 'node:assert/strict'; +import { + idDifference, + launch, + listenerCounts, + listenerDifference, + openSessionPage, + previewBase, + readSession, + requestEngine, + sessionAddress, + switchEngine, + waitReady, +} from './qa/appSessionHarness.mjs'; + +const base = previewBase(process.argv[2]); +const reportOnly = process.argv.includes('--report'); +const report = { switches: [], failures: [] }; +const check = (condition, message, detail) => { + if (condition) return; + report.failures.push({ message, detail }); + if (!reportOnly) + assert.fail(`${message}${detail ? ` ${JSON.stringify(detail)}` : ''}`); +}; + +const FIELD = '#location-search'; +const PANEL = 'data-panel'; + +// The page's visible UI state the switches must leave alone. +const readUi = (page) => + page.evaluate( + (field, panel) => ({ + value: document.querySelector(field)?.value ?? null, + focused: document.activeElement?.id || null, + panelOpen: !document + .getElementById(panel) + ?.classList.contains('collapsed'), + }), + FIELD, + PANEL, + ); + +// Focus the field, press Escape and read what changed; then put the state +// back. The same press before and after the switches must have the same +// effect. +async function escapeEffect(page) { + await page.focus(FIELD); + const before = await readUi(page); + await page.keyboard.press('Escape'); + await new Promise((resolve) => setTimeout(resolve, 400)); + const after = await readUi(page); + return { before, after }; +} + +async function prepareUi(page) { + await page.evaluate((panel) => { + const style = window.__godsEyeView.styleManager; + style.setPanelCollapsed(panel, false, { explicit: true }); + style.setPanelCollapsed('location-bar', false, { explicit: true }); + }, PANEL); + await page.waitForFunction( + (field) => document.querySelector(field)?.offsetParent !== null, + {}, + FIELD, + ); + await page.evaluate((field) => { + const input = document.querySelector(field); + input.focus(); + input.value = 'Reykjav'; + input.dispatchEvent(new Event('input', { bubbles: true })); + }, FIELD); +} + +const browser = await launch(); +try { + const { page, errors } = await openSessionPage( + browser, + sessionAddress(base, { engine: 'cesium' }), + ); + await waitReady(page, 'cesium'); + // Let the first boot's deferred listeners land before counting. + await new Promise((resolve) => setTimeout(resolve, 2000)); + const boot = await readSession(page); + const bootListeners = await listenerCounts(page); + report.boot = boot; + + await prepareUi(page); + const escapeBefore = await escapeEffect(page); + report.escapeBefore = escapeBefore; + await prepareUi(page); + const uiBefore = await readUi(page); + report.uiBefore = uiBefore; + + const sequence = ['navara', 'cesium', 'navara', 'cesium', 'navara', 'cesium']; + for (const kind of sequence) { + const started = Date.now(); + await switchEngine(page, kind); + const state = await readSession(page); + const ui = await readUi(page); + if (kind === 'navara') + await page.evaluate(() => { + window.__qaParkedNavara = window.__godsEyeView.engine; + }); + if (report.switches.length === 1) + report.firstRoundTripListeners = listenerDifference( + bootListeners, + await listenerCounts(page), + ); + report.switches.push({ + kind, + ms: Date.now() - started, + ids: idDifference(boot.ids, state.ids), + counters: state.counters, + ui, + }); + check( + Object.keys(idDifference(boot.ids, state.ids)).length === 0, + `session ids changed on the switch to ${kind}`, + idDifference(boot.ids, state.ids), + ); + check( + ui.value === uiBefore.value, + `typed text lost on the switch to ${kind}`, + ui, + ); + check(ui.panelOpen, `open panel closed on the switch to ${kind}`, ui); + check( + ui.focused === uiBefore.focused, + `focus moved on the switch to ${kind}`, + ui, + ); + } + await new Promise((resolve) => setTimeout(resolve, 2000)); + const after = await readSession(page); + const afterListeners = await listenerCounts(page); + report.after = after; + report.listeners = listenerDifference(bootListeners, afterListeners); + check( + Object.keys(report.listeners).length === 0, + 'window/document listeners changed after three round trips', + report.listeners, + ); + check( + after.counters?.attachments === 1 + sequence.length && + after.counters?.detaches === sequence.length, + 'each switch detaches once and attaches once', + after.counters, + ); + check( + after.counters?.applications === 1, + 'the application must be created once', + after.counters, + ); + + const escapeAfter = await escapeEffect(page); + report.escapeAfter = escapeAfter; + check( + JSON.stringify(escapeAfter) === JSON.stringify(escapeBefore), + 'Escape behaves differently after the switches', + { escapeBefore, escapeAfter }, + ); + + // A failed candidate: the parked Navara renderer refuses to resume (as in + // qa-layer-handoff.mjs). The working Cesium attaches again on the same + // session objects. + await prepareUi(page); + const beforeRollback = await readSession(page); + const historyLength = await page.evaluate(() => history.length); + await page.evaluate(() => { + window.__qaRollback = false; + const parked = window.__qaParkedNavara; + const request = parked.render.request; + parked.render.request = function (...args) { + parked.render.request = request; + window.__qaRollback = true; + throw new Error('QA parked renderer failure'); + }; + }); + const sabotaged = true; + if (sabotaged) { + await requestEngine(page, 'navara'); + await page.waitForFunction(() => window.__qaRollback === true, { + timeout: 60_000, + }); + await waitReady(page, 'cesium'); + const rolledBack = await readSession(page); + const ui = await readUi(page); + report.rollback = { + ids: idDifference(beforeRollback.ids, rolledBack.ids), + counters: rolledBack.counters, + ui, + history: (await page.evaluate(() => history.length)) - historyLength, + }; + check( + Object.keys(report.rollback.ids).length === 0, + 'a rollback changed session ids', + report.rollback, + ); + check( + report.rollback.history === 0, + 'a rollback added a history entry', + report.rollback, + ); + check( + ui.value === uiBefore.value && ui.panelOpen, + 'a rollback lost the UI state', + ui, + ); + // The page still switches after the rollback. + await switchEngine(page, 'navara'); + await switchEngine(page, 'cesium'); + const final = await readSession(page); + check( + Object.keys(idDifference(boot.ids, final.ids)).length === 0, + 'ids changed after the rollback', + idDifference(boot.ids, final.ids), + ); + } + report.errors = errors; + check(errors.length === 0, 'page errors', errors); + console.log(JSON.stringify(report, null, 2)); + if (!reportOnly) console.log('PASS application created once; session kept'); +} finally { + await browser.close(); +} diff --git a/scripts/qa/appSessionHarness.mjs b/scripts/qa/appSessionHarness.mjs new file mode 100644 index 000000000..fb6d8c70f --- /dev/null +++ b/scripts/qa/appSessionHarness.mjs @@ -0,0 +1,349 @@ +// Shared setup for the application-session switch gates +// (scripts/qa-app-once.mjs, scripts/qa-voice-engine-switch.mjs): a headless +// page on a keyless local preview, with every paid map, model and voice +// endpoint blocked, a fake microphone and Realtime transport, and readers for +// the session ids, listener counts and transport counters. +import puppeteer from 'puppeteer'; +import { blockPaidQaMaps } from '../qa-map-policy.mjs'; + +const RESERVED_PORTS = new Set([ + '', + '4173', + '4174', + '4185', + '4186', + '4187', + '4188', + '4189', + '4194', + '4196', +]); + +/** Paid or live model endpoints no switch gate may reach. */ +export const BLOCKED_URLS = Object.freeze([ + '*://tile.googleapis.com/*', + '*://*.googleapis.com/*3dtiles*', + '*://api.cesium.com/*', + '*://assets.ion.cesium.com/*', + '*://maps.googleapis.com/*', + '*/api/google/*', + '*/api/openai/*', + '*/api/realtime/*', + '*/api/gemini/*', + '*://api.openai.com/*', + '*://generativelanguage.googleapis.com/*', +]); + +/** The preview origin from argv, refusing the ports that belong to the user. */ +export function previewBase(argument) { + const base = new URL(argument ?? ''); + if ( + !['localhost', '127.0.0.1'].includes(base.hostname) || + RESERVED_PORTS.has(base.port) + ) + throw new Error('Supply an unreserved local preview URL'); + return base; +} + +/** The page address: Boston on OSM, no welcome, on the given engine. */ +export function sessionAddress(base, { engine = 'cesium', extra = {} } = {}) { + const url = new URL(base.href); + url.pathname = '/'; + url.search = new URLSearchParams({ + welcome: '0', + look: 'cinematic', + engine, + ...extra, + }).toString(); + url.hash = + 'v=2&lat=42.3554&lon=-71.0605&alt=6000&heading=0&pitch=-90&map=osm&l=e'; + return url.href; +} + +export function launch() { + return puppeteer.launch({ + headless: true, + args: [ + '--use-angle=metal', + '--enable-gpu', + '--use-fake-device-for-media-stream', + '--use-fake-ui-for-media-stream', + '--autoplay-policy=no-user-gesture-required', + ], + }); +} + +// Installed before any page script: counts microphone requests and +// transports, and replaces the Realtime peer connection with a local fake +// whose data channel records what the client sends and lets the script play +// the server's events. No request reaches a model. +function installFakeVoice() { + const evidence = { + getUserMedia: 0, + peerConnections: 0, + peerConnectionsClosed: 0, + channelsClosed: 0, + webSockets: 0, + webSocketsClosed: 0, + tokenRequests: 0, + sent: [], + channel: null, + }; + window.__qaVoice = evidence; + localStorage.setItem('gev:first-run-mission:v1', 'suppressed'); + const media = navigator.mediaDevices; + if (media?.getUserMedia) { + const getUserMedia = media.getUserMedia.bind(media); + media.getUserMedia = (constraints) => { + evidence.getUserMedia += 1; + return getUserMedia(constraints); + }; + } + class FakeChannel extends EventTarget { + constructor(label) { + super(); + this.label = label; + this.readyState = 'connecting'; + this.bufferedAmount = 0; + this.id = 1; + } + send(data) { + if (this.readyState !== 'open') throw new Error('Channel not open'); + try { + evidence.sent.push(JSON.parse(data)); + } catch { + evidence.sent.push(data); + } + } + close() { + if (this.readyState === 'closed') return; + this.readyState = 'closed'; + evidence.channelsClosed += 1; + this.dispatchEvent(new Event('close')); + } + } + class FakePeerConnection extends EventTarget { + constructor() { + super(); + evidence.peerConnections += 1; + this.connectionState = 'new'; + this.iceConnectionState = 'new'; + this.iceGatheringState = 'complete'; + this.signalingState = 'stable'; + this.channels = []; + } + addTrack() { + return {}; + } + getSenders() { + return []; + } + getReceivers() { + return []; + } + getTransceivers() { + return []; + } + createDataChannel(label) { + const channel = new FakeChannel(label); + this.channels.push(channel); + evidence.channel = channel; + return channel; + } + async createOffer() { + return { type: 'offer', sdp: 'v=0\r\n' }; + } + async setLocalDescription(description) { + this.localDescription = description; + } + async setRemoteDescription(description) { + this.remoteDescription = description; + setTimeout(() => { + if (this.connectionState === 'closed') return; + this.connectionState = 'connected'; + this.iceConnectionState = 'connected'; + this.onconnectionstatechange?.(); + this.oniceconnectionstatechange?.(); + for (const channel of this.channels) { + channel.readyState = 'open'; + channel.dispatchEvent(new Event('open')); + } + }, 0); + } + async getStats() { + return new Map(); + } + close() { + if (this.connectionState === 'closed') return; + this.connectionState = 'closed'; + evidence.peerConnectionsClosed += 1; + for (const channel of this.channels) channel.close(); + } + } + window.RTCPeerConnection = FakePeerConnection; + const NativeWebSocket = window.WebSocket; + window.WebSocket = class extends NativeWebSocket { + constructor(...args) { + super(...args); + evidence.webSockets += 1; + this.addEventListener('close', () => { + evidence.webSocketsClosed += 1; + }); + } + }; + const fetch = window.fetch.bind(window); + window.fetch = (input, options) => { + const url = new URL(input?.url ?? input, document.baseURI); + if (url.pathname === '/api/realtime/token') { + evidence.tokenRequests += 1; + return Promise.resolve( + Response.json({ + value: 'qa-local-token', + expires_at: Math.floor(Date.now() / 1000) + 600, + }), + ); + } + if (url.hostname === 'api.openai.com') + return Promise.resolve( + new Response('v=0\r\n', { + status: 201, + headers: { 'Content-Type': 'application/sdp' }, + }), + ); + if (url.pathname.startsWith('/api/openai/')) + return Promise.resolve( + Response.json({ error: 'blocked in QA' }, { status: 503 }), + ); + // Empty feeds keep the gate offline and quick. + const fixtures = { + '/api/flights': () => ({ time: Date.now() / 1000, states: [] }), + '/api/military': () => ({ now: Date.now(), ac: [] }), + '/api/military-installations': () => ({ elements: [], status: 'ok' }), + '/api/vessels': () => ({ status: 'live', rows: [] }), + }; + const fixture = fixtures[url.pathname]; + return fixture + ? Promise.resolve(Response.json(fixture())) + : fetch(input, options); + }; +} + +/** A page with the paid endpoints blocked and the voice transport faked. */ +export async function openSessionPage(browser, address) { + const page = await browser.newPage(); + await browser + .defaultBrowserContext() + .overridePermissions(new URL(address).origin, ['microphone']); + await blockPaidQaMaps(page); + const network = await page.createCDPSession(); + await network.send('Network.enable'); + await network.send('Network.setBlockedURLs', { urls: [...BLOCKED_URLS] }); + await page.evaluateOnNewDocument(installFakeVoice); + await page.setViewport({ width: 1440, height: 900 }); + const errors = []; + page.on('pageerror', (error) => errors.push(error.message)); + await page.goto(address, { waitUntil: 'domcontentloaded', timeout: 60_000 }); + return { page, errors }; +} + +/** The engine runs, its picker is idle and the loading screen is gone. */ +export async function waitReady(page, kind) { + await page.waitForFunction( + (kind) => + window.__godsEyeView?.engine?.kind === kind && + document.querySelector('#globe-engine')?.value === kind && + !document.querySelector('#globe-engine').disabled && + document.getElementById('loading-screen')?.classList.contains('hidden') && + !document.querySelector('.engine-switch-frame'), + { timeout: 90_000, polling: 100 }, + kind, + ); + await page.evaluate( + () => window.__godsEyeView.styleManager.initialRestorePromise, + ); +} + +/** + * Pick an engine the way the picker does, without moving focus to it, so a + * focused field keeps its focus through the switch. + */ +export function requestEngine(page, kind) { + return page.evaluate((kind) => { + const select = document.querySelector('#globe-engine'); + select.value = kind; + select.dispatchEvent(new Event('change', { bubbles: true })); + }, kind); +} + +export async function switchEngine(page, kind) { + await requestEngine(page, kind); + await waitReady(page, kind); +} + +/** Session object ids and counters (src/app/sessionDiagnostics.js). */ +export function readSession(page) { + return page.evaluate(() => ({ + ids: window.__gevSession?.ids ?? null, + counters: window.__gevSession?.counters ?? null, + })); +} + +/** Live listeners on window and document, counted per event type. */ +export async function listenerCounts(page) { + const client = await page.createCDPSession(); + try { + const counts = {}; + for (const name of ['window', 'document']) { + const { result } = await client.send('Runtime.evaluate', { + expression: name, + }); + const { listeners } = await client.send('DOMDebugger.getEventListeners', { + objectId: result.objectId, + }); + for (const listener of listeners) { + const key = `${name}:${listener.type}`; + counts[key] = (counts[key] ?? 0) + 1; + } + await client.send('Runtime.releaseObject', { + objectId: result.objectId, + }); + } + return counts; + } finally { + await client.detach(); + } +} + +/** Keys whose counts differ between two listener snapshots. */ +export function listenerDifference(before, after) { + const keys = new Set([...Object.keys(before), ...Object.keys(after)]); + return Object.fromEntries( + [...keys] + .filter((key) => (before[key] ?? 0) !== (after[key] ?? 0)) + .map((key) => [ + key, + { before: before[key] ?? 0, after: after[key] ?? 0 }, + ]), + ); +} + +/** Ids that differ between two session snapshots. */ +export function idDifference(before, after) { + return Object.fromEntries( + Object.keys({ ...before, ...after }) + .filter((key) => before?.[key] !== after?.[key]) + .map((key) => [key, { before: before?.[key], after: after?.[key] }]), + ); +} + +/** Transport counters of the fake voice transport. */ +export function readVoiceEvidence(page) { + return page.evaluate(() => { + const { sent, channel, ...counts } = window.__qaVoice; + return { + ...counts, + channelState: channel?.readyState ?? null, + sentCount: sent.length, + }; + }); +} diff --git a/src/app/sessionDiagnostics.js b/src/app/sessionDiagnostics.js new file mode 100644 index 000000000..cf50dc67a --- /dev/null +++ b/src/app/sessionDiagnostics.js @@ -0,0 +1,66 @@ +// Identity and attachment counters for the page session, read by the switch +// acceptance scripts (scripts/qa-app-once.mjs, qa-voice-engine-switch.mjs): +// an engine switch keeps every session object and only re-attaches the parts +// bound to an engine, so each id below stays the same across switches. + +const ids = new WeakMap(); +let nextId = 0; + +/** + * A small number naming one object for this page: the same object always + * reads the same id, a different object a different one. Null for values + * that are not objects. + * @param {unknown} value + * @returns {number|null} + */ +export function instanceId(value) { + if ( + value === null || + (typeof value !== 'object' && typeof value !== 'function') + ) + return null; + let id = ids.get(value); + if (id === undefined) { + id = ++nextId; + ids.set(value, id); + } + return id; +} + +/** + * The page's diagnostic handle (`window.__gevSession`). + * `read()` returns the session objects by name; `ids` maps each name to its + * {@link instanceId}. `count(name)` bumps a counter such as `attachments`. + * @param {() => Record} read + */ +export function createSessionDiagnostics(read) { + const counters = { + applications: 0, + attachments: 0, + detaches: 0, + rollbacks: 0, + }; + return Object.freeze({ + get ids() { + return Object.fromEntries( + Object.entries(read() ?? {}).map(([name, value]) => [ + name, + instanceId(value), + ]), + ); + }, + get counters() { + return { ...counters }; + }, + get attachments() { + return counters.attachments; + }, + get detaches() { + return counters.detaches; + }, + count(name) { + counters[name] = (counters[name] ?? 0) + 1; + return counters[name]; + }, + }); +} diff --git a/src/app/sessionDiagnostics.test.mjs b/src/app/sessionDiagnostics.test.mjs new file mode 100644 index 000000000..7c38d93aa --- /dev/null +++ b/src/app/sessionDiagnostics.test.mjs @@ -0,0 +1,43 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createSessionDiagnostics, instanceId } from './sessionDiagnostics.js'; + +test('an object keeps its id and another object gets another one', () => { + const a = {}; + const b = {}; + const fn = () => {}; + assert.equal(instanceId(a), instanceId(a)); + assert.notEqual(instanceId(a), instanceId(b)); + assert.equal(typeof instanceId(fn), 'number'); + assert.equal(instanceId(null), null); + assert.equal(instanceId(undefined), null); + assert.equal(instanceId(3), null); +}); + +test('the diagnostics read ids at the time they are asked for', () => { + let shell = {}; + const voice = {}; + const diagnostics = createSessionDiagnostics(() => ({ shell, voice })); + const first = diagnostics.ids; + assert.equal(first.shell, instanceId(shell)); + assert.deepEqual(diagnostics.ids, first); + shell = {}; + assert.notEqual(diagnostics.ids.shell, first.shell); + assert.equal(diagnostics.ids.voice, first.voice); +}); + +test('counters start at zero and count each event', () => { + const diagnostics = createSessionDiagnostics(() => ({})); + assert.equal(diagnostics.attachments, 0); + diagnostics.count('attachments'); + diagnostics.count('attachments'); + diagnostics.count('detaches'); + assert.equal(diagnostics.attachments, 2); + assert.equal(diagnostics.detaches, 1); + assert.deepEqual(diagnostics.counters, { + applications: 0, + attachments: 2, + detaches: 1, + rollbacks: 0, + }); +}); diff --git a/src/main.js b/src/main.js index 6851b3548..62311182a 100644 --- a/src/main.js +++ b/src/main.js @@ -12,6 +12,7 @@ import { waitForFirstFrame, } from './standalone/engineSwitch.js'; import { ENGINE_STORAGE_KEY } from './app/engineChoice.js'; +import { createSessionDiagnostics } from './app/sessionDiagnostics.js'; import { useDisplayStore } from './ui/stores/display.js'; import { applicationPinia } from './ui/vueRuntime.js'; import { t } from './i18n/index.js'; @@ -31,6 +32,25 @@ let cockpit = null; const renderers = createRendererCache(); const layerSession = { lifetime: new AbortController() }; const { signal } = layerSession.lifetime; +// Which session objects this page holds now (scripts/qa-app-once.mjs). +const session = createSessionDiagnostics(() => { + const components = application?.getComponents?.() ?? {}; + const style = components.controls?.styleManager; + return { + application, + shell: style, + shareLink: style?.shareLinkManager, + hud: style?.hud, + navigation: style?._navigation, + placeSearch: style?.placeSearch, + dataManager: components.data?.dataManager, + voice: components.tools?.voiceCommands, + annotations: components.tools?.annotations, + geoImport: window.__gevGeoImport, + drawTool: window.__gevDrawTool, + }; +}); +window.__gevSession = session; window.addEventListener('pagehide', (event) => { if (event.persisted) return; layerSession.lifetime.abort(); @@ -56,6 +76,7 @@ window.addEventListener('popstate', () => { }); function createApplication(kind, engineSwitch) { + session.count('applications'); layerSession.restoringView = engineSwitch; application = createStandaloneApplication({ googleApiKey: import.meta.env.GOOGLE_MAPS_API_KEY, @@ -91,6 +112,7 @@ async function leave(from) { renderers.retain(from); } await application?.destroy(); + session.count('detaches'); } // Start an engine on the retained session. Success means the engine runs @@ -105,6 +127,7 @@ async function enter(kind) { await style.initialRestorePromise; signal.throwIfAborted(); layerSession.restoringView = false; + session.count('attachments'); resumeSession(layerSession); // Only resume the aircraft that actually survived the handoff. A history // entry may select a different subject, and a stale track may disappear. @@ -215,6 +238,7 @@ const initialKind = pageEngineKind(); createApplication(initialKind, false) .start() .then(() => { + session.count('attachments'); engineSwitch = createEngineSwitch({ kind: initialKind, requested: pageEngineKind,