diff --git a/scripts/qa-traffic-baseline.mjs b/scripts/qa-traffic-baseline.mjs index 4a73d6be2..57b1bea87 100644 --- a/scripts/qa-traffic-baseline.mjs +++ b/scripts/qa-traffic-baseline.mjs @@ -8,13 +8,16 @@ * 1. exact-viewport revisit (traffic module's full-road cache), * 2. adjacent 0.4–2 km pan (new exact bbox with heavy spatial overlap), * 3. cold city (no traffic-module entry for that city), - * 4. degraded upstream (synthetic delay on `/api/overpass` only). + * 4. degraded upstream (synthetic delay on the road requests: OpenFreeMap's + * host, which only traffic reads in this capture, or the Overpass proxy). * - * The proxy's actual `X-Overpass-Cache` result is printed for every network - * pass, so HIT/DISK/MISS/STALE/INFLIGHT is evidence rather than inference. - * The degraded leg delays request release in Puppeteer; it exercises the same - * fetch-start→response boundary without adding a failure switch to the - * application or proxy. Pass `--degraded-delay-ms 0` to skip that leg. + * Keyless roads come from OpenFreeMap vector tiles (`tiles.openfreemap.org`); + * a build that still reads them through the Overpass proxy has its + * `X-Overpass-Cache` result printed for every network pass, so + * HIT/DISK/MISS/STALE/INFLIGHT is evidence rather than inference ("—" for + * tiles). The degraded leg delays the road requests' release in Puppeteer; + * it exercises the same fetch-start→response boundary without adding a + * failure switch to the application or proxy. Pass `--degraded-delay-ms 0` to skip that leg. * TomTom status is intercepted as keyless so no live-flow key is required and * the capture isolates the OSM road-to-visible-dot chain. * @@ -93,9 +96,42 @@ function debugUrl(rawUrl) { return url.toString(); } -/** Position the camera and explicitly raise the boundaries the layer observes. */ +/** + * Let the camera report a move the way a user's does: traffic hears the + * camera port's `changed` at its own 5% threshold (layers/traffic/policy.js + * CAMERA_CHANGED_THRESHOLD), which the port measures on every scene tick + * (the first measurement after a listener subscribes only records its + * baseline pose). Waiting for two drawn frames after the move is the sync + * point that the port has measured it. Cesium's own moveEnd (500 ms after + * stillness) is left to Cesium, as in the app; traffic re-checks the view + * on it. Installed in the page as `window.__qaRenderAfterMove`; throws when + * the frames do not draw within 5 s. + */ +function installRenderAfterMove() { + window.__qaRenderAfterMove = (gev) => new Promise((resolve, reject) => { + const scene = gev.viewer.scene; + let frames = 0; + let remove = null; + const timer = setTimeout(() => { + remove(); + reject(new Error(`the camera move rendered ${frames} frame(s) in 5 s`)); + }, 5000); + remove = scene.postRender.addEventListener(() => { + if (++frames >= 2) { + clearTimeout(timer); + remove(); + resolve(frames); + return; + } + scene.requestRender(); + }); + scene.requestRender(); + }); +} + +/** Position the camera and render, so the layer hears the move as it would a user's. */ async function moveCamera(page, view) { - return page.evaluate((v) => { + return page.evaluate(async (v) => { const gev = window.__godsEyeView; const module = gev.dataManager.layers.get('traffic').module; const beforeLastUpdate = module.getStats().lastUpdate; @@ -115,13 +151,7 @@ async function moveCamera(page, view) { roll: 0, }, }); - // setView normally propagates through Cesium's render loop. Raising these - // explicitly makes the capture boundary deterministic across GPU speeds. - // Traffic hears the synthetic moveEnd through the camera port and loads at - // once; the port measures its own `changed` on the next rendered tick, so - // the synthetic camera.changed reaches only the layers still on Cesium. - gev.viewer.camera.changed.raiseEvent(); - gev.viewer.camera.moveEnd.raiseEvent(); + await window.__qaRenderAfterMove(gev); return { beforeLastUpdate, boundaryTime }; }, view); } @@ -160,14 +190,29 @@ function readTrafficMeasures(page, boundaryTime) { })), boundaryTime); } +/** + * Road requests the page sent, counted by the request interceptor: `sent` is + * every road request that reached the network, `delayed` those it held for + * the degraded leg. `requestRoads()` can be answered from decoded tiles + * already in memory, so a `fetch-to-response` measure alone does not prove + * the network was used. + */ +const roadRequests = { sent: 0, delayed: 0 }; + /** Drive one measured state and return its raw User Timing measures. */ async function captureState(page, name, view, setDelay) { setDelay(name === 'degraded-upstream' ? DEGRADED_DELAY_MS : 0); + const before = { ...roadRequests }; const { beforeLastUpdate, boundaryTime } = await moveCamera(page, view); await waitForTrafficCapture(page, beforeLastUpdate, boundaryTime); const measures = await readTrafficMeasures(page, boundaryTime); setDelay(0); - return { name, measures }; + return { + name, + measures, + sent: roadRequests.sent - before.sent, + delayed: roadRequests.delayed - before.delayed, + }; } /** Format a duration without hiding zero-duration measurements. */ @@ -319,13 +364,14 @@ async function main() { const page = await browser.newPage(); await page.setViewport({ width: 1440, height: 900 }); await page.evaluateOnNewDocument(() => localStorage.clear()); + await page.evaluateOnNewDocument(installRenderAfterMove); page.on('console', (message) => { if (message.type() === 'error') consoleErrors.push(message.text()); }); page.on('pageerror', (error) => consoleErrors.push(error.message)); - // Force keyless traffic and optionally hold Overpass requests to model a - // degraded proxy/upstream. No server key or proxy debug endpoint is needed. + // Force keyless traffic and optionally hold the road requests to model a + // degraded upstream. No server key or proxy debug endpoint is needed. await page.setRequestInterception(true); page.on('request', async (request) => { if (request.url().includes('/api/tomtom/status')) { @@ -336,7 +382,11 @@ async function main() { }); return; } - if (activeProxyDelayMs > 0 && request.url().includes('/api/overpass')) { + const roadRequest = request.url().includes('/api/overpass') + || new URL(request.url()).hostname === 'tiles.openfreemap.org'; + if (roadRequest) roadRequests.sent += 1; + if (activeProxyDelayMs > 0 && roadRequest) { + roadRequests.delayed += 1; await sleep(activeProxyDelayMs); } try { await request.continue(); } catch { /* page closed or already handled */ } @@ -364,8 +414,9 @@ async function main() { orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, }); await gev.dataManager.setEnabled('traffic', true); - gev.viewer.camera.changed.raiseEvent(); - gev.viewer.camera.moveEnd.raiseEvent(); + // Also records the baseline pose traffic's new `changed` listener + // measures the first measured move against. + await window.__qaRenderAfterMove(gev); }, VIEWS.austin); await sleep(500); @@ -387,7 +438,7 @@ async function main() { // Clear the last-bounds gate without clearing the module's road cache, then // revisit the byte-identical camera pose for a true client-cache hit. - await page.evaluate((view) => { + await page.evaluate(async (view) => { const gev = window.__godsEyeView; const ellipsoid = gev.viewer.scene.globe.ellipsoid; const radians = Math.PI / 180; @@ -399,8 +450,7 @@ async function main() { }), orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, }); - gev.viewer.camera.changed.raiseEvent(); - gev.viewer.camera.moveEnd.raiseEvent(); + await window.__qaRenderAfterMove(gev); }, VIEWS.austin); await sleep(500); @@ -429,7 +479,7 @@ async function main() { } for (const capture of captures.slice(1)) { if (!capture.measures.some((entry) => entry.detail?.segment === 'fetch-to-response')) { - throw new Error(`${capture.name} did not issue an Overpass request`); + throw new Error(`${capture.name} did not issue a road request`); } } if (DEGRADED_DELAY_MS > 0) { @@ -439,20 +489,35 @@ async function main() { && entry.duration >= DEGRADED_DELAY_MS * 0.9 )); if (!delayed) throw new Error('degraded-upstream leg did not observe the synthetic delay'); + // The delay has to come from held road requests, not from processing + // that happened to run long. + if (!degraded.delayed) throw new Error('degraded-upstream leg sent no road request to hold'); } + // A city the client has never seen has no tiles in memory: it must reach + // the network. The other legs may legitimately be answered from tiles + // already decoded, so their counts are reported, not asserted. + const cold = captures.find((capture) => capture.name === 'cold city (client)'); + if (!cold?.sent) throw new Error('cold city leg sent no road request to the network'); const rows = captures.flatMap(rowsForCapture); if (!rows.length) throw new Error('no measured traffic renders were captured'); console.log('\nCausal segments (one row per visible major/full pass)'); console.table(rows); + console.log('\nRoad requests sent to the network per leg (held by the degraded leg)'); + console.table(captures.map((capture) => ({ + State: capture.name, + Sent: capture.sent, + Held: capture.delayed, + }))); console.log('\nNotes:'); console.log(' - Render→postRender is the first visible-frame boundary used by Phase 0.'); - console.log(' - Proxy cache is read from X-Overpass-Cache; “—” means the client cache avoided fetch.'); - console.log(' - response.json includes both body transfer and JSON decoding; the production path does not split them.'); - console.log(' - Last change→fetch is approximately the 320 ms FETCH_DEBOUNCE for the first fetch by construction.'); + console.log(' - Proxy cache is read from X-Overpass-Cache; “—” means the client cache avoided fetch, or the roads came from OpenFreeMap tiles.'); + console.log(' - Through the Overpass proxy, Fetch→response is the HTTP response and response.json is body transfer plus JSON decoding.'); + console.log(' - From OpenFreeMap tiles, Fetch→response spans the tile requests, body reads, decoding and composition, and response.json only hands back the built object; a held leg can stack the delay over tiles fetched in turn.'); + console.log(' - Last change→fetch is the 320 ms FETCH_DEBOUNCE for the first fetch by construction, plus timer and main-thread latency (larger under SwiftShader).'); console.log(' - Full-pass Last change→fetch also includes the intentionally sequential major pass.'); - console.log(' - Cesium moveEnd is a diagnostic raised ~500 ms after stillness, typically after fetch starts; fetch never waits for it.'); - console.log(' - Synthetic events bypass camera percentageChanged/inertia but use the same camera.changed schedule-time anchor as natural captures.'); + console.log(' - Cesium moveEnd is raised ~500 ms after stillness, typically after fetch starts; fetch never waits for it, and traffic only re-checks the view on it.'); + console.log(' - Moves are setView plus rendered frames: the camera port measures traffic\'s own 5% changed threshold as for a user move (inertia is bypassed).'); console.log(' - The keyless capture zeroes Flow race and Heat lines work; live-keyed numbers will differ.'); console.log(' - Measurement caveat: timings include instrumentation overhead (observer effect) and browser clock-floor quantization.'); console.log(' - Per-road clock observations inside road-parse-total/waypoint-materialization slightly overstate production work.');