From 32b0304c2ce1bdb71def174cf30ecd2501ec967a Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:23:10 +0100 Subject: [PATCH 01/20] fix(radio): normalize station tags and retain valid mixed rows --- docs/CURRENT-STATE.md | 7 +++++++ server/providers/radio/catalog.js | 17 ++++------------- server/providers/radio/stations.js | 1 + src/data/radio.test.mjs | 18 ++++++++++++++++++ src/data/radioProxy.test.mjs | 8 ++++++++ src/layers/radio/ingestion.js | 12 ++++++------ src/layers/radio/presentation.js | 1 + src/layers/radio/state.js | 1 + src/sources/radioBrowser.js | 18 +++++++++++------- 9 files changed, 57 insertions(+), 26 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..278c4e0bd 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,12 @@ # God's Eye View Current State +Radio tags are lowercased and whitespace-normalized before the final 80-character +cap, including requested-tag coverage matching. Malformed station rows are skipped +individually; valid rows retain their accepted catalog generation. The UI snapshot +reports a bounded `rejectedStationCount` and degraded coverage when rows were +skipped. Malformed envelope/freshness/generation metadata and all-invalid responses +still preserve the previous usable catalog. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js index 36a3eb730..7eed9f01a 100644 --- a/server/providers/radio/catalog.js +++ b/server/providers/radio/catalog.js @@ -5,7 +5,7 @@ import { readResponseTextCapped } from '../common/http.js'; import { normalizeRadioBrowserStation, publicRadioStation, - cleanRadioText, + normalizeRadioTag, } from './stations.js'; import { radioMirrorOrigin, @@ -177,8 +177,8 @@ export function createRadioProxyMiddleware({ const rows = await fetchPath(`/json/stations/search?${params}`); if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - if ( - !rows.every( + const stations = rows + .filter( (row) => row && typeof row === 'object' && @@ -188,18 +188,9 @@ export function createRadioProxyMiddleware({ (typeof row.url_resolved === 'string' || typeof row.url === 'string'), ) - ) - throw new Error( - 'Radio Browser catalog contained a malformed station row', - ); - const stations = rows .map(normalizeRadioBrowserStation) .filter(Boolean); - const requestedTag = cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(); + const requestedTag = normalizeRadioTag(tag); const requestedTagCovered = !requestedTag || stations.some((station) => diff --git a/server/providers/radio/stations.js b/server/providers/radio/stations.js index b97b2d3c6..49279e2d2 100644 --- a/server/providers/radio/stations.js +++ b/server/providers/radio/stations.js @@ -1,5 +1,6 @@ export { cleanRadioText, + normalizeRadioTag, isNonGlobalIpv4, publicRadioHttpsUrl, normalizeRadioBrowserStation, diff --git a/src/data/radio.test.mjs b/src/data/radio.test.mjs index 58a4101a1..6ab8561c6 100644 --- a/src/data/radio.test.mjs +++ b/src/data/radio.test.mjs @@ -712,6 +712,24 @@ test('unusable directory responses preserve warm client state atomically', async assert.equal(getRadioUIState().stale, true); assert.equal(getRadioUIState().degraded, true); } + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement, { id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 2, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioUIState().stationCount, 1); + assert.equal(getRadioUIState().rejectedStationCount, 1); + assert.equal(getRadioUIState().degraded, true); + assert.equal(getRadioUIState().stale, false); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioAcceptedCatalogSnapshot().stations[0].id, replacement.id); + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [{ id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 3, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioUIState().stale, true); } finally { globalThis.fetch = originalFetch; radioLayer.destroy(); diff --git a/src/data/radioProxy.test.mjs b/src/data/radioProxy.test.mjs index 60463205b..d1c4d2660 100644 --- a/src/data/radioProxy.test.mjs +++ b/src/data/radioProxy.test.mjs @@ -533,3 +533,11 @@ test('a mirror answering an invalid reason phrase rejects instead of crashing', const crashed = mockMirrorStatus(t, 200, 'OK\x7f'); await assert.rejects(Promise.race([fetchMirror(), crashed]), TypeError); }); + + +test('case expansion is capped after tag normalization in the portable and server station path', () => { + const row = normalizeRadioBrowserStation(station({tags: 'İ'.repeat(80) + ', NEWS__RADIO ,news-radio'})); + assert.equal(row.tags[0].length, 80); + assert.equal(row.tags[0], 'i\u0307'.repeat(40)); + assert.deepEqual(row.tags.slice(1), ['news radio']); +}); diff --git a/src/layers/radio/ingestion.js b/src/layers/radio/ingestion.js index 869b8d403..5a0ccfa1d 100644 --- a/src/layers/radio/ingestion.js +++ b/src/layers/radio/ingestion.js @@ -70,11 +70,6 @@ export function createIngestion({ ); if (!acceptedRows.length) throw new Error('Radio directory returned no usable stations'); - if (acceptedRows.length !== rows.length) { - throw new Error( - 'Radio directory response contained malformed stations', - ); - } const acceptedGeneration = body.acceptedGeneration; if ( acceptedGeneration !== null && @@ -147,7 +142,12 @@ export function createIngestion({ } if (!repeatingAcceptedGeneration) layerState._updatedAt = updatedAt; } - layerState._degraded = body.degraded; + layerState._rejectedStationCount = Math.min( + 1000000, + rows.length - acceptedRows.length, + ); + layerState._degraded = + body.degraded || layerState._rejectedStationCount > 0; layerState._stale = body.stale; layerState._error = preservingWarmCatalog ? ERROR_REFRESH_DEGRADED diff --git a/src/layers/radio/presentation.js b/src/layers/radio/presentation.js index 5d6763ca4..c30889018 100644 --- a/src/layers/radio/presentation.js +++ b/src/layers/radio/presentation.js @@ -20,6 +20,7 @@ export function createPresentation({ loading: layerState._loading, stale: layerState._stale, degraded: layerState._degraded, + rejectedStationCount: layerState._rejectedStationCount, error: layerState._error, updatedAt: layerState._updatedAt, filter: layerState._filter, diff --git a/src/layers/radio/state.js b/src/layers/radio/state.js index dca461e03..918ebb989 100644 --- a/src/layers/radio/state.js +++ b/src/layers/radio/state.js @@ -25,6 +25,7 @@ export function createState({ services }) { layerState._stale = false; layerState._degraded = false; + layerState._rejectedStationCount = 0; layerState._error = null; diff --git a/src/sources/radioBrowser.js b/src/sources/radioBrowser.js index 8511b6beb..8329f6257 100644 --- a/src/sources/radioBrowser.js +++ b/src/sources/radioBrowser.js @@ -11,6 +11,16 @@ export function cleanRadioText(value, maxLength) { .trim(); } +/** Normalize a station/filter tag and cap its final, case-folded representation. */ +export function normalizeRadioTag(value) { + return cleanRadioText( + String(value ?? '') + .toLowerCase() + .replace(/[_-]+/g, ' '), + 80, + ); +} + export function isNonGlobalIpv4(hostname) { const pieces = hostname.split('.'); if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) @@ -99,13 +109,7 @@ export function normalizeRadioBrowserStation( if (!name) return null; const tags = String(raw?.tags ?? '') .split(',') - .map((tag) => - cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(), - ) + .map(normalizeRadioTag) .filter(Boolean) .filter((tag, index, all) => all.indexOf(tag) === index) .slice(0, 24); From 34507f606ab99ca2dee5ead78a133045da4315b8 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:23:34 +0100 Subject: [PATCH 02/20] docs: record radio fix --- changelog.d/1268-radio-issue-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1268-radio-issue-fix.md diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md new file mode 100644 index 000000000..de6b0bcae --- /dev/null +++ b/changelog.d/1268-radio-issue-fix.md @@ -0,0 +1 @@ +- Normalize station tags and retain valid mixed rows (asturksever, #1268). From 9d3d5164fe42ec3ee25afb34c6817b0cfe4bd0db Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:28:00 +0100 Subject: [PATCH 03/20] fix(space): respect provider cooldowns and bound TLE groups --- docs/CURRENT-STATE.md | 7 ++ scripts/package-boundaries.json | 3 + server/providers/space/celestrak.js | 56 ++++++++++++++-- server/providers/space/cooldown.js | 12 ++++ server/providers/space/launch-library.js | 36 ++++++++++- src/proxyErrorResponses.test.mjs | 16 +++-- src/tooling/spaceProviders.test.mjs | 82 +++++++++++++++++++++++- 7 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 server/providers/space/cooldown.js diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..106e66ad9 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,12 @@ # God's Eye View Current State +CelesTrak accepts only the seven satellite groups used by the app and gates +requests before provider work. CelesTrak and Launch Library failures start a +provider-wide cooldown: Retry-After seconds or HTTP dates are bounded to 5 seconds +through 30 minutes, with a 60-second fallback. During cooldown dated last-good +data is served with stale provenance, otherwise the proxy returns a bounded +error without another upstream attempt. Error bodies are cancelled before reading. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ca5dc0c7f..d7d9ba727 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -96,6 +96,9 @@ "modules": [ "server/providers/space.js", "server/providers/space/celestrak.js", + "server/providers/common/same-site.js", + "src/localRequestGate.mjs", + "server/providers/space/cooldown.js", "server/providers/space/launch-library.js", "server/providers/common/http.js", "src/data/spaceProviderRequests.js", diff --git a/server/providers/space/celestrak.js b/server/providers/space/celestrak.js index 8baf59ba7..315e9b3f0 100644 --- a/server/providers/space/celestrak.js +++ b/server/providers/space/celestrak.js @@ -1,6 +1,8 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js'; +import { providerCooldownMs } from './cooldown.js'; +import { admitSameSite } from '../common/same-site.js'; import { readResponseTextCapped } from '../common/http.js'; /** @@ -24,6 +26,16 @@ import { readResponseTextCapped } from '../common/http.js'; export function celestrakProxy() { const TLE_TTL_MS = 6 * 3600_000; const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); + const groups = new Set([ + 'stations', + 'visual', + 'gps-ops', + 'glo-ops', + 'galileo', + 'geo', + 'starlink', + ]); + let cooldownUntil = 0; const mem = new Map(); // group -> { at: epochMs, body: string } const inflight = new Map(); // group -> Promise<{at, body}|null> @@ -64,7 +76,11 @@ export function celestrakProxy() { 'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)', }, }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); + if (!res.ok) { + cooldownUntil = Date.now() + providerCooldownMs(res); + void res.body?.cancel().catch(() => {}); + throw new Error(`HTTP ${res.status}`); + } const body = await readResponseTextCapped( res, CELESTRAK_MAX_RESPONSE_BYTES, @@ -80,12 +96,18 @@ export function celestrakProxy() { const group = String(req.url || '') .replace(/^\//, '') .split('?')[0]; - if (!/^[a-z0-9-]+$/i.test(group)) { + if (admitSameSite(req, res)) return; + if (req.method !== 'GET') { + res.writeHead(405); + res.end(); + return; + } + if (!groups.has(group)) { res.writeHead(400, { 'Content-Type': 'text/plain' }); res.end('invalid group'); return; } - const send = (status, body, cacheStatus) => { + const send = (status, body, cacheStatus, entry) => { // Guard against a double-send (e.g. a throw AFTER a response already // went out routing into the catch's send): writeHead after headersSent // throws "Cannot set headers after they are sent". @@ -93,6 +115,17 @@ export function celestrakProxy() { res.writeHead(status, { 'Content-Type': 'text/plain', 'x-tle-cache': cacheStatus, + 'Cache-Control': 'no-store', + ...(entry + ? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() } + : {}), + ...(cooldownUntil > 0 && Date.now() < cooldownUntil + ? { + 'Retry-After': String( + Math.ceil((cooldownUntil - Date.now()) / 1000), + ), + } + : {}), }); res.end(body); }; @@ -104,7 +137,16 @@ export function celestrakProxy() { if (entry) mem.set(group, entry); } if (entry && now - entry.at < TLE_TTL_MS) { - send(200, entry.body, 'HIT'); + send(200, entry.body, 'HIT', entry); + return; + } + if (Date.now() < cooldownUntil) { + send( + entry ? 200 : 502, + entry?.body || 'celestrak upstream cooling down', + entry ? 'STALE-ERROR' : 'NONE', + entry, + ); return; } // Stale or missing → refresh, single-flight per group. @@ -118,6 +160,8 @@ export function celestrakProxy() { return fresh; }) .catch((err) => { + if (Date.now() >= cooldownUntil) + cooldownUntil = Date.now() + 60000; console.warn( '[celestrak-proxy] refresh failed — serving cache if any', ); @@ -128,9 +172,9 @@ export function celestrakProxy() { } const fresh = await inflight.get(group); if (fresh) { - send(200, fresh.body, 'MISS'); + send(200, fresh.body, 'MISS', fresh); } else if (entry) { - send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty + send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty } else { send(502, 'celestrak fetch failed and no cache available', 'NONE'); } diff --git a/server/providers/space/cooldown.js b/server/providers/space/cooldown.js new file mode 100644 index 000000000..acb850124 --- /dev/null +++ b/server/providers/space/cooldown.js @@ -0,0 +1,12 @@ +/** Bound an upstream Retry-After delay (seconds or HTTP date). */ +export function providerCooldownMs(response, now = Date.now()) { + const raw = response?.headers?.get('retry-after')?.trim(); + const seconds = raw && /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) : NaN; + const deadline = raw && !Number.isFinite(seconds) ? Date.parse(raw) : NaN; + const delay = Number.isFinite(seconds) + ? seconds * 1000 + : Number.isFinite(deadline) + ? deadline - now + : 60000; + return Math.min(30 * 60000, Math.max(5000, delay)); +} diff --git a/server/providers/space/launch-library.js b/server/providers/space/launch-library.js index 7b56b6220..4e8ac629d 100644 --- a/server/providers/space/launch-library.js +++ b/server/providers/space/launch-library.js @@ -1,4 +1,5 @@ import path from 'node:path'; +import { providerCooldownMs } from './cooldown.js'; import { promises as fsp } from 'node:fs'; import { readResponseTextCapped, @@ -30,6 +31,8 @@ export function rocketLaunchesProxy() { 'launch-library-2-v2.3.json', ); let cache = null; + let cooldownUntil = 0; + let cooldownStatus = 502; let diskLoaded = false; const inFlight = new Map(); @@ -62,7 +65,18 @@ export function rocketLaunchesProxy() { function send(res, status, body, cacheState) { res.writeHead(status, { 'Content-Type': 'application/json', - 'Cache-Control': status === 200 ? 'public, max-age=900' : 'no-store', + 'Cache-Control': + status === 200 && cacheState !== 'STALE-ERROR' + ? 'public, max-age=900' + : 'no-store', + ...(cache ? { 'X-GEV-Cached-At': new Date(cache.at).toISOString() } : {}), + ...(Date.now() < cooldownUntil + ? { + 'Retry-After': String( + Math.ceil((cooldownUntil - Date.now()) / 1000), + ), + } + : {}), 'X-GEV-Cache': cacheState, }); res.end(body); @@ -75,17 +89,21 @@ export function rocketLaunchesProxy() { signal: AbortSignal.timeout(20000), headers: launchLibraryRequestHeaders(), }); - const body = await readResponseTextCapped(upstream, maxResponseBytes); if (!upstream.ok) { + cooldownUntil = Date.now() + providerCooldownMs(upstream); + cooldownStatus = upstream.status; + void upstream.body?.cancel().catch(() => {}); const error = new Error(`upstream HTTP ${upstream.status}`); error.upstreamStatus = upstream.status; throw error; } + const body = await readResponseTextCapped(upstream, maxResponseBytes); const parsed = JSON.parse(body); if (!Array.isArray(parsed?.results)) throw new Error('malformed upstream response'); const fresh = { at: Date.now(), body }; cache = fresh; + cooldownUntil = 0; void saveDiskCache(fresh); return fresh; } @@ -103,6 +121,16 @@ export function rocketLaunchesProxy() { return; } const stale = cache; + if (now < cooldownUntil) { + send( + res, + stale ? 200 : cooldownStatus, + stale?.body || + JSON.stringify({ error: 'Launch Library 2 unavailable' }), + stale ? 'STALE-ERROR' : 'NONE', + ); + return; + } const request = coalesceProxyRequest( inFlight, 'recent-launches', @@ -112,6 +140,10 @@ export function rocketLaunchesProxy() { const fresh = await request.promise; send(res, 200, fresh.body, request.shared ? 'INFLIGHT' : 'MISS'); } catch (error) { + if (Date.now() >= cooldownUntil) { + cooldownUntil = Date.now() + 60000; + cooldownStatus = 502; + } // Log only a bounded status, never upstream bodies, URLs, or credentials. const status = Number.isInteger(error?.upstreamStatus) ? error.upstreamStatus diff --git a/src/proxyErrorResponses.test.mjs b/src/proxyErrorResponses.test.mjs index 78f3f59f0..d215e78a3 100644 --- a/src/proxyErrorResponses.test.mjs +++ b/src/proxyErrorResponses.test.mjs @@ -2,6 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import path from 'node:path'; +import { providerCooldownMs } from '../server/providers/space/cooldown.js'; import { parseTerrainPoints } from './data/terrainHeightsProxy.js'; import { readResponseTextCapped, coalesceProxyRequest } from './sources/httpBody.js'; import { makeRateLimiter, clientKey } from '../server/providers/common/rate-limit.js'; @@ -23,7 +24,8 @@ function extract(name) { function fixture(name, overrides = {}, preview = false) { const logs = []; const deps = { - readResponseTextCapped, coalesceProxyRequest, + readResponseTextCapped, coalesceProxyRequest, providerCooldownMs, + admitSameSite: () => false, // The rate-limited proxies reach for these; give them the real ones so a // fixture request is throttled exactly as production would be. makeRateLimiter, clientKey, @@ -117,7 +119,7 @@ test('Launch Library retains single-flight, fresh cache, stale fallback, and met test('CelesTrak unexpected failures hide details', async () => { const app = fixture('celestrakProxy', { Date: { now() { throw new Error(detail); } } }); - const res = await app.request('/active'); + const res = await app.request('/stations'); assert.equal(res.status, 500); assert.equal(res.body, 'celestrak proxy error'); assert.equal(res.headers['x-tle-cache'], 'ERROR'); @@ -126,7 +128,7 @@ test('CelesTrak unexpected failures hide details', async () => { test('CelesTrak retains invalid-group and unavailable responses', async () => { const app = fixture('celestrakProxy'); assert.equal((await app.request('/../')).status, 400); - const res = await app.request('/active'); + const res = await app.request('/stations'); assert.equal(res.status, 502); assert.equal(res.headers['x-tle-cache'], 'NONE'); }); @@ -135,13 +137,13 @@ test('CelesTrak retains fresh and stale TLE caches', async () => { let now = Date.now(); let calls = 0; const app = fixture('celestrakProxy', { - Date: { now: () => now }, + Date: class extends Date { static now() { return now; } }, fetch: async () => { if (++calls > 1) throw new Error(detail); return new Response('1 valid-fixture-TLE'); }, }); - assert.equal((await app.request('/active')).headers['x-tle-cache'], 'MISS'); - assert.equal((await app.request('/active')).headers['x-tle-cache'], 'HIT'); + assert.equal((await app.request('/stations')).headers['x-tle-cache'], 'MISS'); + assert.equal((await app.request('/stations')).headers['x-tle-cache'], 'HIT'); now += 7 * 3600_000; - const stale = await app.request('/active'); + const stale = await app.request('/stations'); assert.equal(stale.status, 200); assert.equal(stale.body, '1 valid-fixture-TLE'); assert.equal(stale.headers['x-tle-cache'], 'STALE-ERROR'); diff --git a/src/tooling/spaceProviders.test.mjs b/src/tooling/spaceProviders.test.mjs index a5b2cecba..2abc9d5fc 100644 --- a/src/tooling/spaceProviders.test.mjs +++ b/src/tooling/spaceProviders.test.mjs @@ -161,7 +161,11 @@ test('CelesTrak answers 502 to an oversized TLE body without caching it', async assert.ok(bodies[0].pulled <= 17, 'reading stops at the 16 MiB cap'); assert.equal(fsp.writeFile.mock.callCount(), 0); assert.equal((await request('/api/celestrak', '/stations')).status, 502); - assert.equal(bodies.length, 2, 'nothing was cached in memory'); + assert.equal( + bodies.length, + 1, + 'failed bodies are not cached but a cooldown bounds retry work', + ); }); for (const preview of [false, true]) @@ -190,3 +194,79 @@ for (const preview of [false, true]) assert.equal(hit.body, first.body); assert.equal(calls, 1); }); + +test('space providers honor Retry-After, gate groups, preserve stale dates and recover', async (t) => { + isolateDisk(t); + t.mock.method(console, 'warn', () => {}); + let now = Date.now(); + t.mock.method(Date, 'now', () => now); + for (const kind of ['tle', 'launches']) { + for (const retryAfter of ['90', new Date(now + 90000).toUTCString()]) { + let calls = 0; + let fail = false; + let cancelled = false; + t.mock.method(globalThis, 'fetch', async () => { + calls++; + if (fail) + return new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { + status: kind === 'tle' ? 403 : 429, + headers: { 'Retry-After': retryAfter }, + }, + ); + return new Response( + kind === 'tle' + ? 'ISS\n1 25544U fixture\n2 25544 fixture\n' + : '{"results":[]}', + ); + }); + const request = install( + kind === 'tle' ? celestrakProxy() : rocketLaunchesProxy(), + ); + const route = kind === 'tle' ? '/api/celestrak' : '/api/launches'; + const url = kind === 'tle' ? '/stations' : '/'; + if (kind === 'tle') { + assert.equal( + (await request(route, '/not-a-supported-group')).status, + 400, + ); + assert.equal(calls, 0); + } + const first = await request(route, url); + now += 7 * 3600000; + // Recompute the HTTP-date relative to the clock at the failed refresh. + const originalFetch = globalThis.fetch; + t.mock.method(globalThis, 'fetch', async (...args) => { + const response = await originalFetch(...args); + if (fail && !/^\d+$/.test(retryAfter)) + response.headers.set( + 'Retry-After', + new Date(now + 90000).toUTCString(), + ); + return response; + }); + fail = true; + const stale = await request(route, url); + assert.equal(stale.status, 200); + assert.equal(stale.body, first.body); + assert.equal(stale.headers['Retry-After'], '90'); + assert.ok( + stale.headers[kind === 'tle' ? 'X-TLE-Cached-At' : 'X-GEV-Cached-At'], + ); + assert.equal(cancelled, true); + await request(route, url); + if (kind === 'tle') + assert.equal((await request(route, '/visual')).status, 502); + assert.equal(calls, 2, 'cooldown applies across TLE groups'); + now += 91000; + fail = false; + assert.equal((await request(route, url)).status, 200); + assert.equal(calls, 3); + } + } +}); From 8602999056ca09fdd25e26f4ac8c6b24f592a496 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:29:24 +0100 Subject: [PATCH 04/20] docs: record space fix --- changelog.d/1270-space-issue-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1270-space-issue-fix.md diff --git a/changelog.d/1270-space-issue-fix.md b/changelog.d/1270-space-issue-fix.md new file mode 100644 index 000000000..b483c4c01 --- /dev/null +++ b/changelog.d/1270-space-issue-fix.md @@ -0,0 +1 @@ +- Respect provider cooldowns and bound tle groups (asturksever, #1270). From 5f3d1eaa2fcf5d05d11afc948a2b31b3737e1454 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:43:47 +0100 Subject: [PATCH 05/20] fix(military): retire unavailable history and keep local trails --- docs/CURRENT-STATE.md | 2 ++ server/providers/aircraft/tracks.js | 21 +++++++++++++-------- src/layers/military/tracking.js | 2 +- src/sources/live/contract.test.mjs | 9 ++++----- src/sources/live/standalone.js | 22 ++-------------------- src/tooling/liveProviders.test.mjs | 18 ++++++++++++++++++ 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..70c741f79 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,7 @@ # God's Eye View Current State +The standalone military source no longer requests the unavailable, undocumented ADSB.lol trace endpoint. Tracking retains its locally accumulated trail, and custom sources may still supply optional getTrack history. The old HTTP route validates the identifier then reports 503 history_unavailable without any upstream request, rather than forwarding a misleading 403. This does not restore historical coverage or download bulk daily archives. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/server/providers/aircraft/tracks.js b/server/providers/aircraft/tracks.js index 5f800ac59..3b0acd8d2 100644 --- a/server/providers/aircraft/tracks.js +++ b/server/providers/aircraft/tracks.js @@ -7,10 +7,9 @@ import { readCappedResponseText } from '../common/http.js'; * own credit bucket, 4 credits per call on the free tier). OAuth via the * shared coalesced token. 60s per-icao cache; 404/429 forwarded so the * client can fall back to its accumulated trail silently. - * /api/military/track?hex= — adsb.lol tar1090 readsb trace - * (undocumented but live; no browser CORS, hence this proxy). Up to ~24h - * of real history per aircraft. Treat as best-effort; data is ODbL — - * credit "adsb.lol (ODbL)" in the UI. + * /api/military/track?hex= — compatibility refusal for the retired, + * undocumented ADSB.lol trace route. The standalone military source uses + * locally accumulated trails; provider-supplied history remains optional. */ export function trackBackfillProxies() { const TRACK_CACHE_MS = 60000; @@ -107,10 +106,16 @@ export function trackBackfillProxies() { ); return; } - await proxyJson( - res, - `lol:${hex}`, - `https://adsb.lol/data/traces/${hex.slice(-2)}/trace_full_${hex}.json`, + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Retry-After', '1800'); + res.end( + JSON.stringify({ + error: + 'Military history unavailable; locally accumulated trails remain available', + code: 'history_unavailable', + }), ); } catch (error) { res.statusCode = 502; diff --git a/src/layers/military/tracking.js b/src/layers/military/tracking.js index 8a440a54f..3530a4877 100644 --- a/src/layers/military/tracking.js +++ b/src/layers/military/tracking.js @@ -593,7 +593,7 @@ export function createTracking({ } /** - * Fire-and-forget adsb.lol readsb trace backfill (PRD F2). Trace points are + * Fire-and-forget optional source history backfill (PRD F2). Trace points are * stride-thinned to the remaining vertex budget, then spliced strictly older * than the oldest seeded fix AHEAD of the locally accumulated fine segment, * capped at TRAIL_MAX_POINTS (newest kept). Any failure (404/timeout/ diff --git a/src/sources/live/contract.test.mjs b/src/sources/live/contract.test.mjs index 658db1596..b8b1787da 100644 --- a/src/sources/live/contract.test.mjs +++ b/src/sources/live/contract.test.mjs @@ -214,11 +214,10 @@ test('construction is inert; adapters preserve routes, viewport query, cache epo assert.equal(mil.observedAtMs, now - 60000); assert.equal(mil.records[0].positionTimeMs, now - 75000); assert.equal(mil.stale, true); - const track = await military.getTrack('abc123'); - assert.equal(track.complete, false); - assert.equal(track.records[0].observedAtMs, now - 50000); - assert.equal(track.records[0].baroAltitudeM, 304.8); - assert.equal(track.records[0].ellipsoidAltitudeM, null); + assert.equal(military.getTrack, undefined); + assert.ok( + requests.every(({ url }) => !url.startsWith('/api/military/track')), + ); }); test('cancellation after slow body parsing rejects even with a transport that ignores abort', async () => { diff --git a/src/sources/live/standalone.js b/src/sources/live/standalone.js index 55670c629..7311c492d 100644 --- a/src/sources/live/standalone.js +++ b/src/sources/live/standalone.js @@ -159,26 +159,8 @@ export function createMilitarySource({ status: response.status, }; }, - async getTrack(reference, { signal } = {}) { - const { response, payload } = await readResponse( - fetchImpl, - '/api/military/track?hex=' + encodeURIComponent(reference), - { signal }, - 'Military aircraft', - ); - if (!response.ok) throw httpError(response, 'Military aircraft'); - const baseTimeMs = epoch(payload?.timestamp, 1000); - return { - records: - baseTimeMs == null - ? [] - : normalizeAircraftTrack(payload?.trace, { - baseTimeMs, - readsb: true, - }), - complete: false, - }; - }, + // ADSB.lol does not expose a supported live-history endpoint. Omitting + // optional getTrack keeps the tracking layer on its accumulated trail. }; } diff --git a/src/tooling/liveProviders.test.mjs b/src/tooling/liveProviders.test.mjs index a60a37cea..488c2d40c 100644 --- a/src/tooling/liveProviders.test.mjs +++ b/src/tooling/liveProviders.test.mjs @@ -583,3 +583,21 @@ test('track backfill proxy returns 502 on an oversized upstream body and caches assert.equal(res2.statusCode, 502); assert.equal(callCount, 1); }); + +test('retired military history refuses without upstream I/O while validating identifiers', async (t) => { + t.mock.method(globalThis, 'fetch', async () => { + assert.fail('retired history must not contact the upstream'); + }); + const route = install(providers.trackBackfillProxies()); + for (const hex of ['abc123', 'abc456']) { + const response = await route('/api/military/track', '?hex=' + hex); + assert.equal(response.statusCode, 503); + assert.equal(JSON.parse(response.body).code, 'history_unavailable'); + assert.equal(response.headers['retry-after'], '1800'); + assert.equal(response.headers['cache-control'], 'no-store'); + } + assert.equal( + (await route('/api/military/track', '?hex=invalid')).statusCode, + 400, + ); +}); From 5d9eabc9192e22d7e2620445e27a85df956bb895 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:44:15 +0100 Subject: [PATCH 06/20] docs: record history fix --- changelog.d/1273-history-issue-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1273-history-issue-fix.md diff --git a/changelog.d/1273-history-issue-fix.md b/changelog.d/1273-history-issue-fix.md new file mode 100644 index 000000000..f672a5c22 --- /dev/null +++ b/changelog.d/1273-history-issue-fix.md @@ -0,0 +1 @@ +- Retire unavailable history and keep local trails (asturksever, #1273). From 122b511d1ee04b9e135d2fb729254689a5284b4d Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 01:21:53 +0100 Subject: [PATCH 07/20] fix(traffic): hold rendering only for animated content --- docs/CURRENT-STATE.md | 2 ++ src/layers/traffic/animation.js | 26 +++++++++++++- src/layers/traffic/lifecycle.js | 5 ++- src/layers/traffic/motion.test.mjs | 9 ++++- src/layers/traffic/renderDemand.test.mjs | 45 ++++++++++++++++++++++++ src/layers/traffic/rendering.js | 2 ++ src/layers/traffic/retention.js | 1 + src/layers/transit/lifecycle.test.mjs | 2 +- 8 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 src/layers/traffic/renderDemand.test.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..109015c4d 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,7 @@ # God's Eye View Current State +Traffic holds continuous rendering only while enabled with animated dots or a pulsing jam heat-line. Empty, failed, and high-altitude views release the hold; population admission reacquires it and resets animation time. Clear, disable, and destroy also release it. Retained fading dots keep animation alive until they are removed. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/src/layers/traffic/animation.js b/src/layers/traffic/animation.js index 05afde1c1..4d9f6c4b3 100644 --- a/src/layers/traffic/animation.js +++ b/src/layers/traffic/animation.js @@ -20,6 +20,20 @@ export function createAnimation({ parts, source, }) { + function syncRenderDemand() { + const needed = Boolean( + layerState._enabled && + (layerState._dots.length || layerState._heatJamPrim), + ); + if (needed === Boolean(layerState._renderHeld)) return; + layerState._renderHeld = needed; + if (needed) { + layerState._lastAnimTime = 0; + services.render.holdContinuousRender('traffic'); + layerState._viewer?.scene?.requestRender?.(); + } else services.render.releaseContinuousRender('traffic'); + } + /** * Spawn animated dot primitives along a single road. * @@ -188,6 +202,7 @@ export function createAnimation({ : null, }); } + syncRenderDemand(); } /** @@ -206,6 +221,8 @@ export function createAnimation({ */ function animate() { + syncRenderDemand(); + if (!layerState._renderHeld) return; const now = Date.now(); // Delta time in seconds, capped to avoid jumps when returning from background tab const dt = layerState._lastAnimTime @@ -339,6 +356,13 @@ export function createAnimation({ layerState._count = 0; layerState._bucketCounts = { free: 0, slow: 0, jam: 0, sim: 0 }; layerState._closedRoads = 0; + syncRenderDemand(); } - return { spawnDotsForRoad, animate, maybeStopLight, clearDots }; + return { + spawnDotsForRoad, + animate, + maybeStopLight, + clearDots, + syncRenderDemand, + }; } diff --git a/src/layers/traffic/lifecycle.js b/src/layers/traffic/lifecycle.js index 94c40d935..fc13e205d 100644 --- a/src/layers/traffic/lifecycle.js +++ b/src/layers/traffic/lifecycle.js @@ -12,7 +12,6 @@ export function createLifecycle({ parts, source, }) { - const { holdContinuousRender, releaseContinuousRender } = services.render; const { resetFlowTileCache } = source; const methods = { @@ -81,7 +80,7 @@ export function createLifecycle({ layerState._enabled = true; layerState._surfaceFrameRemover = observeTrafficSurface(viewer.scene); if (layerState._roadMode !== 'tomtom') source.prefetch?.(); - holdContinuousRender('traffic'); // per-frame animator (perf wave 2) + parts.animation.syncRenderDemand(); layerState._lastAnimTime = 0; layerState._pointCollection.show = true; @@ -142,7 +141,7 @@ export function createLifecycle({ disable(viewer) { layerState._enabled = false; services.credits?.hideOsmCredit?.(layerState._viewer, 'traffic'); - releaseContinuousRender('traffic'); + parts.animation.syncRenderDemand(); clearTimeout(layerState._fetchTimeout); clearInterval(layerState._enableKickTimer); layerState._enableKickTimer = null; diff --git a/src/layers/traffic/motion.test.mjs b/src/layers/traffic/motion.test.mjs index 3c230d870..6338eb10f 100644 --- a/src/layers/traffic/motion.test.mjs +++ b/src/layers/traffic/motion.test.mjs @@ -37,7 +37,14 @@ function setup(t) { visibleRoadsForAltitude: (roads) => roads, }, }; - const ctx = { state, services: {}, parts, source: {} }; + const ctx = { + state, + services: { + render: { holdContinuousRender() {}, releaseContinuousRender() {} }, + }, + parts, + source: {}, + }; parts.style = createStyle(ctx); parts.model = createModel(ctx); parts.animation = createAnimation(ctx); diff --git a/src/layers/traffic/renderDemand.test.mjs b/src/layers/traffic/renderDemand.test.mjs new file mode 100644 index 000000000..df5965145 --- /dev/null +++ b/src/layers/traffic/renderDemand.test.mjs @@ -0,0 +1,45 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createAnimation } from './animation.js'; + +test('traffic render demand follows animated population and jam lines across empty, disable and re-enable', () => { + const calls = []; + const state = { _enabled: true, _dots: [], _motion: { rebuilds: 0 } }; + const animation = createAnimation({ + state, + services: { + render: { + holdContinuousRender: (id) => calls.push(['hold', id]), + releaseContinuousRender: (id) => calls.push(['release', id]), + }, + }, + parts: { + retention: { clear() {} }, + rendering: { + removeHeatLines() { + state._heatJamPrim = null; + }, + }, + }, + }); + animation.syncRenderDemand(); + assert.deepEqual(calls, []); + state._dots.push({}); + animation.syncRenderDemand(); + animation.syncRenderDemand(); + assert.deepEqual(calls, [['hold', 'traffic']]); + animation.clearDots(); + assert.deepEqual(calls.at(-1), ['release', 'traffic']); + state._heatJamPrim = {}; + animation.syncRenderDemand(); + assert.deepEqual(calls.at(-1), ['hold', 'traffic']); + state._enabled = false; + animation.syncRenderDemand(); + assert.deepEqual(calls.at(-1), ['release', 'traffic']); + state._enabled = true; + animation.clearDots(); + assert.equal(state._renderHeld, false); + state._dots.push({}); + animation.syncRenderDemand(); + assert.equal(state._renderHeld, true); +}); diff --git a/src/layers/traffic/rendering.js b/src/layers/traffic/rendering.js index 7e148148d..bf0132b98 100644 --- a/src/layers/traffic/rendering.js +++ b/src/layers/traffic/rendering.js @@ -50,6 +50,7 @@ export function createRendering({ layerState._heatSlowPrim = null; } layerState._heatLineCount = 0; + parts.animation.syncRenderDemand?.(); } /** @@ -152,6 +153,7 @@ export function createRendering({ } layerState._heatLineCount = kept.length; + parts.animation.syncRenderDemand?.(); if (candidates.length > kept.length) { console.log( `[Data:Traffic] Heat-lines capped at ${HEAT_LINE_CAP} (${candidates.length} congested roads in view)`, diff --git a/src/layers/traffic/retention.js b/src/layers/traffic/retention.js index 4bfe7bfb6..ec0762b5c 100644 --- a/src/layers/traffic/retention.js +++ b/src/layers/traffic/retention.js @@ -125,6 +125,7 @@ export function createRetention({ state, parts }) { count++; } state._count = count; + parts.animation.syncRenderDemand?.(); } function reconcile(roads, height, replace) { altitude = height; diff --git a/src/layers/transit/lifecycle.test.mjs b/src/layers/transit/lifecycle.test.mjs index 8a2d39edf..9bc5cb5b1 100644 --- a/src/layers/transit/lifecycle.test.mjs +++ b/src/layers/transit/lifecycle.test.mjs @@ -3576,7 +3576,7 @@ test('actual Traffic, Bikeshare and Transit lifecycles retain sensitivity in eve source: {}, parts: { flow: { ensureFlowStatus: noop }, - animation: { animate: noop, clearDots: noop }, + animation: { animate: noop, clearDots: noop, syncRenderDemand: noop }, viewport: { onCameraChanged: noop }, ingestion: { cancelActiveFetch: noop }, }, From 34b064fc6229cd044a8d09e68733ac1c13129030 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 01:22:32 +0100 Subject: [PATCH 08/20] docs: record traffic fix --- changelog.d/1281-traffic-issue-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1281-traffic-issue-fix.md diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md new file mode 100644 index 000000000..3c546d333 --- /dev/null +++ b/changelog.d/1281-traffic-issue-fix.md @@ -0,0 +1 @@ +- Hold rendering only for animated content (asturksever, #1281). From 9fd434438b052f6cd5b4c09a30ffb85244a762d5 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 13:50:42 +0100 Subject: [PATCH 09/20] test: target mounted voice input in radio takeover gate --- changelog.d/1268-radio-issue-fix.md | 1 + scripts/qa-radio.mjs | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md index de6b0bcae..7a5be28b7 100644 --- a/changelog.d/1268-radio-issue-fix.md +++ b/changelog.d/1268-radio-issue-fix.md @@ -1 +1,2 @@ - Normalize station tags and retain valid mixed rows (asturksever, #1268). +- Correct Radio QA push-to-talk setup to target the mounted voice controller (asturksever, #1268). diff --git a/scripts/qa-radio.mjs b/scripts/qa-radio.mjs index 648075bd2..cfc601ca3 100644 --- a/scripts/qa-radio.mjs +++ b/scripts/qa-radio.mjs @@ -2761,13 +2761,16 @@ async function main() { const beforeSpace = radio.getUIState().audioState; // Simulate an already-started hold-Space session; click-started open mic // deliberately ignores Space takeover, and short taps never claim voice. - const priorPushToTalkMode = voice.pushToTalkMode; - voice.pushToTalkMode = true; + // The provider facade forwards reads, not writes. Set the mounted + // controller's input owner; assigning on the facade only shadows it. + const controller = voice.session.adapter.controller; + const priorPushToTalkMode = controller.pushToTalkMode; + controller.pushToTalkMode = true; document.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, cancelable: true, code: 'Space', key: ' ' })); await new Promise((resolve) => setTimeout(resolve, 700)); const afterSpace = radio.getUIState().audioState; document.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, cancelable: true, code: 'Space', key: ' ' })); - voice.pushToTalkMode = priorPushToTalkMode; + controller.pushToTalkMode = priorPushToTalkMode; await radio.play(); voice.setVoiceSpeaker('ai'); return { beforeSpace, afterSpace, afterAi: radio.getUIState().audioState }; From f16fda23506d70329dc73bbc3ec925737ea5c498 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 14:18:23 +0100 Subject: [PATCH 10/20] test: exercise traffic with current vector and terrain fixtures --- changelog.d/1281-traffic-issue-fix.md | 1 + scripts/qa-traffic.mjs | 39 +++---- scripts/traffic-fixtures.mjs | 144 ++++++++++++++++++-------- src/tooling/trafficFixtures.test.mjs | 92 ++++++++++++++++ 4 files changed, 214 insertions(+), 62 deletions(-) create mode 100644 src/tooling/trafficFixtures.test.mjs diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md index 3c546d333..ade62efe4 100644 --- a/changelog.d/1281-traffic-issue-fix.md +++ b/changelog.d/1281-traffic-issue-fix.md @@ -1 +1,2 @@ - Hold rendering only for animated content (asturksever, #1281). +- Exercise current vector road sources and published camera bounds in Traffic fixture QA (asturksever, #1281). diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index 424f3486e..5da4b7db7 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -25,8 +25,8 @@ * Visual proof saved to qa-shots/ (gitignored). * * Run: node scripts/qa-traffic.mjs --url http://localhost:4410 - * Add --fixtures for synthetic roads and the recorded flow tile without a key; - * that mode does not qualify live source access or quota accounting. + * Add --fixtures for synthetic road/flow tiles and a flat terrain surface without a key; + * that mode does not qualify geographic terrain, live source access or quota accounting. * Exits non-zero on any FAIL. Does not commit anything. */ @@ -44,7 +44,9 @@ const getOpt = (name, dflt) => { const i = argv.indexOf(name); return i >= 0 && argv[i + 1] ? argv[i + 1] : dflt; }; -const APP_URL = getOpt('--url', 'http://localhost:4410'); +const appUrl = new URL(getOpt('--url', 'http://localhost:4410')); +appUrl.searchParams.set('welcome', '0'); +const APP_URL = appUrl.href; const HEADFUL = argv.includes('--headful'); const FIXTURES = argv.includes('--fixtures'); const fixtureResponse = FIXTURES ? (await import('./traffic-fixtures.mjs')).trafficFixtureResponse : () => null; @@ -122,8 +124,8 @@ async function main() { return; } - const statusBefore = FIXTURES ? {hasKey:true,dailyCount:0} : await fetch(`${APP_URL}/api/tomtom/status`).then((r) => r.json()).catch(() => null); - if (FIXTURES) console.log('Source mode: synthetic roads and recorded flow tiles; live quota is not qualified.'); + const statusBefore = FIXTURES ? {hasKey:true,dailyCount:0} : await fetch(`${appUrl.origin}/api/tomtom/status`).then((r) => r.json()).catch(() => null); + if (FIXTURES) console.log('Source mode: synthetic road/flow tiles and a flat terrain surface; live quota is not qualified.'); if (!statusBefore?.hasKey) { console.error('\x1b[31mServer has no TomTom key — run against the keyed dev server (:4410).\x1b[0m'); process.exitCode = 2; @@ -135,7 +137,9 @@ async function main() { headless: HEADFUL ? false : 'new', ...(findChromeExecutable() ? { executablePath: findChromeExecutable() } : {}), args: [ - '--no-sandbox', '--disable-setuid-sandbox', '--use-gl=angle', '--use-angle=swiftshader', + '--no-sandbox', '--disable-setuid-sandbox', + ...(!HEADFUL ? ['--use-gl=angle', '--use-angle=swiftshader'] : []), + '--disable-backgrounding-occluded-windows', '--disable-dev-shm-usage', '--disable-web-security', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', '--window-size=1440,900', @@ -236,19 +240,16 @@ async function main() { // ── (ii) C4 oblique bounds ─────────────────────────────────────────────── console.log('\n(ii) C4 — oblique fetch bounds land at the look-at point...'); trafficLogs.length = 0; - await settleTraffic(page, { lon: -97.72, lat: 30.245, height: 2500, heading: 315, pitch: -20 }, { minCount: 100 }); + const oblique = await settleTraffic(page, { lon: -97.72, lat: 30.245, height: 2500, heading: 315, pitch: -20 }, { minCount: 100 }); { - const fetchLine = trafficLogs.find((t) => t.includes('fetch') && t.includes('[')); - let ok = false; let detail = 'no fetch log captured'; - if (fetchLine) { - const m = fetchLine.match(/\[(-?[\d.]+),(-?[\d.]+),(-?[\d.]+),(-?[\d.]+)\]/); - if (m) { - const [s, w, n, e] = m.slice(1).map(Number); - const cLat = (s + n) / 2; const cLon = (w + e) / 2; - const dKm = Math.sqrt(((cLat - 30.245) * 111) ** 2 + ((cLon - (-97.72)) * 111 * Math.cos(30.27 * Math.PI / 180)) ** 2); - ok = dKm > 1 && dKm <= 13; // look-at ~6.9 km ahead; clamp allows ≤12 (+margin) - detail = `box center (${cLat.toFixed(4)},${cLon.toFixed(4)}) is ${dKm.toFixed(1)} km from camera (want 1–13 km; pre-fix bug: 25+ km)`; - } + const bounds = oblique?.roadBounds; + let ok = false; let detail = 'no road bounds published'; + if (bounds) { + const cLat = (bounds.south + bounds.north) / 2; + const cLon = (bounds.west + bounds.east) / 2; + const dKm = Math.hypot((cLat - 30.245) * 111, (cLon + 97.72) * 111 * Math.cos(30.27 * Math.PI / 180)); + ok = dKm > 1 && dKm <= 13; + detail = `road box center is ${dKm.toFixed(1)} km from camera (want 1–13 km)`; } record('C4: oblique fetch box centers on the look-at point', ok, detail); if (!ok) exitCode = 1; @@ -258,7 +259,7 @@ async function main() { // ── (iii) budget honesty ───────────────────────────────────────────────── console.log('\n(iii) Budget — dailyCount grew by ≤ requests this run...'); { - const statusAfter = FIXTURES ? statusBefore : await fetch(`${APP_URL}/api/tomtom/status`).then((r) => r.json()); + const statusAfter = FIXTURES ? statusBefore : await fetch(`${appUrl.origin}/api/tomtom/status`).then((r) => r.json()); const grew = statusAfter.dailyCount - statusBefore.dailyCount; const ok = grew >= 0 && grew <= flowRequests.length; record('BUDGET: /api/tomtom/status growth ≤ page tile requests', FIXTURES ? null : ok, diff --git a/scripts/traffic-fixtures.mjs b/scripts/traffic-fixtures.mjs index 54ad48358..af51487db 100644 --- a/scripts/traffic-fixtures.mjs +++ b/scripts/traffic-fixtures.mjs @@ -1,56 +1,114 @@ -import { readFileSync } from 'node:fs'; -import { decodeFlowTile } from '../src/layers/traffic/flowDecode.js'; -import { tilesForBounds } from '../src/data/tomtomTiles.js'; +import { PbfWriter } from 'pbf'; -const tile = readFileSync( - new URL( - '../src/data/fixtures/tomtom-flow-austin-12-935-1686.pbf', - import.meta.url, - ), -); const json = (body) => ({ status: 200, contentType: 'application/json', body: JSON.stringify(body), }); -/** Reuse the recorded flow tile for an explicitly synthetic browser exercise. */ +// Cesium heightmap-1.0: 65x65 little-endian uint16 heights, scale .2, +// offset -1000, followed by child availability and a uniform land mask. +const flatTerrain = Buffer.alloc(65 * 65 * 2 + 2); +for (let i = 0; i < 65 * 65; i++) flatTerrain.writeUInt16LE(5000, i * 2); + +/** Synthetic, geographically aligned motor roads at both source zooms. */ +export function trafficFixtureTile(zoom, flow = false) { + const writer = new PbfWriter(); + writer.writeMessage( + 3, + (_, layer) => { + layer.writeVarintField(15, 2); + layer.writeStringField(1, flow ? 'Traffic flow' : 'transportation'); + layer.writeVarintField(5, 4096); + layer.writeStringField(3, flow ? 'traffic_level' : 'class'); + layer.writeMessage( + 4, + (_, value) => { + if (flow) value.writeDoubleField(3, 0.6); + else value.writeStringField(1, 'primary'); + }, + null, + ); + if (flow) { + layer.writeStringField(3, 'road_type'); + layer.writeMessage( + 4, + (_, value) => value.writeStringField(1, 'Major road'), + null, + ); + } + // z12 has four times the span of z14. Keep the same world spacing, + // and cover only alternate roads with flow to exercise hide/sim controls. + const step = 256 * 2 ** (zoom - 14); + for (let y = step; y < 4096; y += step) { + if (flow && (y / step) % 2) continue; + layer.writeMessage( + 2, + (_, feature) => { + feature.writeVarintField(3, 2); + feature.writePackedVarint(2, flow ? [0, 0, 1, 1] : [0, 0]); + feature.writePackedVarint(4, [9, 0, y * 2, 10, 8192, 0]); + }, + null, + ); + } + }, + null, + ); + return Buffer.from(writer.finish()); +} + +/** Intercept the current vector sources; never fall through to live roads. */ export function trafficFixtureResponse(request) { const url = new URL(request.url()); + if (url.hostname === 'terrain.reearth.land') { + if (url.pathname.endsWith('/layer.json')) + return json({ + tilejson: '2.1.0', + format: 'heightmap-1.0', + version: 'qa-flat', + scheme: 'tms', + projection: 'EPSG:4326', + minzoom: 0, + maxzoom: 0, + tiles: ['{z}/{x}/{y}.terrain?v={version}'], + available: [[{ startX: 0, startY: 0, endX: 1, endY: 0 }]], + attribution: + 'Synthetic flat terrain for Traffic QA; not live elevation data', + }); + if (/\/0\/[01]\/0\.terrain$/.test(url.pathname)) + return { + status: 200, + contentType: 'application/octet-stream', + body: flatTerrain, + }; + throw new Error(`Unexpected terrain fixture request: ${url.pathname}`); + } if (url.pathname === '/api/tomtom/status') return json({ hasKey: true, dailyCount: 0 }); - if (/^\/api\/tomtom\/flow\/\d+\/\d+\/\d+\.pbf$/.test(url.pathname)) - return { status: 200, contentType: 'application/x-protobuf', body: tile }; - if (url.pathname !== '/api/overpass') return null; - const query = new URLSearchParams(request.postData()).get('data') || ''; - if (!query.includes('highway')) return null; - const match = query.match(/\((-?[\d.]+),(-?[\d.]+),(-?[\d.]+),(-?[\d.]+)\)/); - if (!match) throw new Error('Missing road fixture viewport'); - const [south, west, north, east] = match.slice(1).map(Number); - const inside = ([lon, lat]) => - lat >= south && lat <= north && lon >= west && lon <= east; - const segments = tilesForBounds({ south, west, north, east }, 12).flatMap( - ({ z, x, y }) => decodeFlowTile(tile, z, x, y), + const flow = url.pathname.match( + /^\/api\/tomtom\/flow\/(\d+)\/\d+\/\d+\.pbf$/, ); - const elements = segments - .filter((segment) => segment.coords.some(inside)) - .slice(0, 240) - .map((segment, index) => ({ - type: 'way', - id: index + 1, - tags: { highway: 'primary', oneway: 'yes' }, - geometry: segment.coords.map(([lon, lat]) => ({ lon, lat })), - })); - const latitude = (south + north) / 2, - longitude = (west + east) / 2; - elements.push({ - type: 'way', - id: 99999, - tags: { highway: 'residential' }, - geometry: [ - { lat: latitude, lon: longitude }, - { lat: latitude + 0.003, lon: longitude + 0.003 }, - ], - }); - return json({ elements }); + if (flow) + return { + status: 200, + contentType: 'application/x-protobuf', + body: trafficFixtureTile(Number(flow[1]), true), + }; + if (url.hostname !== 'tiles.openfreemap.org') return null; + if (url.pathname === '/planet') + return json({ + tilejson: '3.0.0', + minzoom: 0, + maxzoom: 14, + tiles: ['https://tiles.openfreemap.org/qa-traffic/{z}/{x}/{y}.pbf'], + }); + const road = url.pathname.match(/^\/qa-traffic\/(\d+)\/\d+\/\d+\.pbf$/); + if (!road) + throw new Error(`Unrecognized road fixture request: ${url.pathname}`); + return { + status: 200, + contentType: 'application/x-protobuf', + body: trafficFixtureTile(Number(road[1])), + }; } diff --git a/src/tooling/trafficFixtures.test.mjs b/src/tooling/trafficFixtures.test.mjs new file mode 100644 index 000000000..308c0aaa6 --- /dev/null +++ b/src/tooling/trafficFixtures.test.mjs @@ -0,0 +1,92 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + trafficFixtureResponse, + trafficFixtureTile, +} from '../../scripts/traffic-fixtures.mjs'; +import { decodeOpenFreeMapTile } from '../sources/openFreeMap.js'; +import { isDrivableFlowSegment } from '../layers/traffic/roadModes.js'; +import { decodeFlowTile } from '../layers/traffic/flowDecode.js'; + +test('traffic fixtures provide aligned current road tiles and partial flow coverage', () => { + for (const z of [12, 14]) { + const roads = decodeOpenFreeMapTile( + trafficFixtureTile(z), + z, + 935, + 1686, + ).roads; + const flow = decodeFlowTile(trafficFixtureTile(z, true), z, 935, 1686); + assert.ok(roads.length > flow.length && flow.length > 0); + for (const segment of flow) { + assert.equal(segment.trafficLevel, 0.6); + assert.equal(isDrivableFlowSegment(segment), true); + assert.ok( + roads.some( + (road) => + JSON.stringify(road.coordinates) === JSON.stringify(segment.coords), + ), + ); + } + } +}); + +test('fixture TileJSON and every road/flow request stay on intercepted sources', () => { + const request = (url) => trafficFixtureResponse({ url: () => url }); + const metadata = JSON.parse( + request('https://tiles.openfreemap.org/planet').body, + ); + const tileUrl = metadata.tiles[0] + .replace('{z}', '14') + .replace('{x}', '935') + .replace('{y}', '1686'); + assert.ok( + decodeOpenFreeMapTile(request(tileUrl).body, 14, 935, 1686).roads.length, + ); + assert.ok( + decodeFlowTile( + request('http://localhost/api/tomtom/flow/12/935/1686.pbf').body, + 12, + 935, + 1686, + ).length, + ); + assert.throws( + () => request('https://tiles.openfreemap.org/unexpected'), + /Unrecognized/, + ); + assert.equal(request('http://localhost/api/unrelated'), null); +}); + +test('flat terrain fixture decodes through Cesium and advertises no network children', async () => { + const { createServer } = await import('node:http'); + const { CesiumTerrainProvider } = await import('cesium'); + const server = createServer((req, res) => { + const response = trafficFixtureResponse({ + url: () => `https://terrain.reearth.land${req.url}`, + }); + res.writeHead(response.status, { 'content-type': response.contentType }); + res.end(response.body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const provider = await CesiumTerrainProvider.fromUrl( + `http://127.0.0.1:${server.address().port}`, + ); + for (const x of [0, 1]) { + const data = await provider.requestTileGeometry(x, 0, 0); + const rect = provider.tilingScheme.tileXYToRectangle(x, 0, 0); + assert.equal( + data.interpolateHeight(rect, (rect.west + rect.east) / 2, 0), + 0, + ); + for (const dx of [0, 1]) + for (const dy of [0, 1]) { + assert.equal(data.isChildAvailable(x, 0, x * 2 + dx, dy), false); + } + } + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + } +}); From 77ed0a7c31633d8c8b8bef8321f733e3c8f2d4a8 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 14:53:08 +0100 Subject: [PATCH 11/20] test: run Traffic fixtures with normal browser security --- changelog.d/1281-traffic-issue-fix.md | 2 ++ scripts/qa-traffic.mjs | 15 ++++++++++++++- scripts/traffic-fixtures.mjs | 5 +++++ src/tooling/trafficFixtures.test.mjs | 6 +++--- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md index ade62efe4..2c4d1900c 100644 --- a/changelog.d/1281-traffic-issue-fix.md +++ b/changelog.d/1281-traffic-issue-fix.md @@ -1,2 +1,4 @@ - Hold rendering only for animated content (asturksever, #1281). - Exercise current vector road sources and published camera bounds in Traffic fixture QA (asturksever, #1281). +- Establish browser-level foreground visibility for Traffic QA and reject hidden-page samples (asturksever, #1281). +- Supply CORS headers on cross-origin traffic fixtures and report browser exceptions without disabling browser security (asturksever, #1281). diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index 5da4b7db7..bb00596ce 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -82,6 +82,10 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); /** Enable traffic + teleport, then poll the layer until settled. */ async function settleTraffic(page, view, { minCount = 1, timeoutS = 30 } = {}) { + // The app deliberately stops rendering hidden pages. Programmatic camera + // changes do not activate a headful window the way a real UI click does. + await page.bringToFront(); + await page.waitForFunction(() => document.visibilityState === 'visible'); return page.evaluate(async (v, minC, tS) => { const gev = window.__godsEyeView; const dm = gev.dataManager; @@ -99,6 +103,7 @@ async function settleTraffic(page, view, { minCount = 1, timeoutS = 30 } = {}) { }); let s = null; for (let i = 0; i < tS; i++) { + if (document.visibilityState !== 'visible') throw new Error('Traffic QA page became hidden; rendering is intentionally paused'); await new Promise((r) => setTimeout(r, 1000)); s = mod.getStats(); // Retained roads from the previous city can satisfy count/loading before @@ -140,7 +145,7 @@ async function main() { '--no-sandbox', '--disable-setuid-sandbox', ...(!HEADFUL ? ['--use-gl=angle', '--use-angle=swiftshader'] : []), '--disable-backgrounding-occluded-windows', - '--disable-dev-shm-usage', '--disable-web-security', + '--disable-dev-shm-usage', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', '--window-size=1440,900', ], @@ -149,6 +154,10 @@ async function main() { let exitCode = 0; try { const page = await browser.newPage(); + // Keep this visible-page regression deterministic even if the OS starts + // Chrome minimized. Use browser focus emulation, not patched DOM getters. + const focusSession = await page.createCDPSession(); + await focusSession.send('Emulation.setFocusEmulationEnabled', { enabled: true }); const onFixture = (request) => { const response = fixtureResponse(request); if (response) void request.respond(response); else void request.continue(); @@ -162,6 +171,9 @@ async function main() { if (req.url().includes('/api/tomtom/flow/')) flowRequests.push(req.url()); }); const trafficLogs = []; + const pageErrors = []; + page.on('pageerror', error => { pageErrors.push(error.message); console.error('PAGE ERROR:', error.message); }); + page.on('console', message => { if (message.type() === 'error') console.error('BROWSER ERROR:', message.text()); }); page.on('console', (msg) => { const t = msg.text(); if (t.includes('[Data:Traffic]')) trafficLogs.push(t); @@ -319,6 +331,7 @@ async function main() { await sleep(1000); await page.screenshot({ path: path.join(SHOTS_DIR, 'traffic-sim-keyless.png') }); } + record('BROWSER: no uncaught page exceptions', pageErrors.length === 0, pageErrors.join('; ')); } catch (e) { console.error('\x1b[31mHarness error:\x1b[0m', e); exitCode = 3; diff --git a/scripts/traffic-fixtures.mjs b/scripts/traffic-fixtures.mjs index af51487db..2f94993cc 100644 --- a/scripts/traffic-fixtures.mjs +++ b/scripts/traffic-fixtures.mjs @@ -1,7 +1,9 @@ import { PbfWriter } from 'pbf'; +const corsHeaders = { 'access-control-allow-origin': '*' }; const json = (body) => ({ status: 200, + headers: corsHeaders, contentType: 'application/json', body: JSON.stringify(body), }); @@ -80,6 +82,7 @@ export function trafficFixtureResponse(request) { return { status: 200, contentType: 'application/octet-stream', + headers: corsHeaders, body: flatTerrain, }; throw new Error(`Unexpected terrain fixture request: ${url.pathname}`); @@ -93,6 +96,7 @@ export function trafficFixtureResponse(request) { return { status: 200, contentType: 'application/x-protobuf', + headers: corsHeaders, body: trafficFixtureTile(Number(flow[1]), true), }; if (url.hostname !== 'tiles.openfreemap.org') return null; @@ -109,6 +113,7 @@ export function trafficFixtureResponse(request) { return { status: 200, contentType: 'application/x-protobuf', + headers: corsHeaders, body: trafficFixtureTile(Number(road[1])), }; } diff --git a/src/tooling/trafficFixtures.test.mjs b/src/tooling/trafficFixtures.test.mjs index 308c0aaa6..e02b959e7 100644 --- a/src/tooling/trafficFixtures.test.mjs +++ b/src/tooling/trafficFixtures.test.mjs @@ -33,9 +33,9 @@ test('traffic fixtures provide aligned current road tiles and partial flow cover test('fixture TileJSON and every road/flow request stay on intercepted sources', () => { const request = (url) => trafficFixtureResponse({ url: () => url }); - const metadata = JSON.parse( - request('https://tiles.openfreemap.org/planet').body, - ); + const tileJson = request('https://tiles.openfreemap.org/planet'); + assert.equal(tileJson.headers['access-control-allow-origin'], '*'); + const metadata = JSON.parse(tileJson.body); const tileUrl = metadata.tiles[0] .replace('{z}', '14') .replace('{x}', '935') From 116f28a5256078e8e76281cbf1c96075b594d7f1 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 14:55:18 +0100 Subject: [PATCH 12/20] test: isolate keyed and keyless Traffic browser sessions --- changelog.d/1281-traffic-issue-fix.md | 1 + scripts/qa-traffic.mjs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md index 2c4d1900c..10dc045d0 100644 --- a/changelog.d/1281-traffic-issue-fix.md +++ b/changelog.d/1281-traffic-issue-fix.md @@ -2,3 +2,4 @@ - Exercise current vector road sources and published camera bounds in Traffic fixture QA (asturksever, #1281). - Establish browser-level foreground visibility for Traffic QA and reject hidden-page samples (asturksever, #1281). - Supply CORS headers on cross-origin traffic fixtures and report browser exceptions without disabling browser security (asturksever, #1281). +- Isolate the Traffic keyless browser scenario from the keyed page’s workers and intercepted cache (asturksever, #1281). diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index bb00596ce..d07b7196f 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -153,7 +153,7 @@ async function main() { let exitCode = 0; try { - const page = await browser.newPage(); + let page = await browser.newPage(); // Keep this visible-page regression deterministic even if the OS starts // Chrome minimized. Use browser focus emulation, not patched DOM getters. const focusSession = await page.createCDPSession(); @@ -162,7 +162,7 @@ async function main() { const response = fixtureResponse(request); if (response) void request.respond(response); else void request.continue(); }; - if (FIXTURES) { await page.setRequestInterception(true); page.on('request', onFixture); } + if (FIXTURES) { await page.setCacheEnabled(false); await page.setRequestInterception(true); page.on('request', onFixture); } await page.setViewport({ width: 1440, height: 900 }); // Track flow-tile requests + traffic console lines for (ii)/(iii)/(v). @@ -281,7 +281,16 @@ async function main() { // ── (v) KEYLESS fallback (intercepted — server key untouched) ──────────── console.log('\n(v) KEYLESS — intercepted status, expect pure simulation...'); - if (FIXTURES) page.off('request', onFixture); + // Keyless is an independent fresh-session contract. Close the keyed page + // so its workers, terrain work and intercepted cache cannot leak into it. + await page.close(); + page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 900 }); + const keylessFocus = await page.createCDPSession(); + await keylessFocus.send('Emulation.setFocusEmulationEnabled', { enabled: true }); + page.on('pageerror', error => { pageErrors.push(error.message); console.error('PAGE ERROR:', error.message); }); + page.on('console', message => { if (message.type() === 'error') console.error('BROWSER ERROR:', message.text()); }); + await page.setCacheEnabled(false); await page.setRequestInterception(true); const keylessFlowReqs = []; page.on('request', (req) => { @@ -299,7 +308,7 @@ async function main() { if (response) { void req.respond(response); return; } try { req.continue(); } catch { /* already handled */ } }); - await page.reload({ waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForFunction( () => window.__godsEyeView?.viewer && window.__godsEyeView?.dataManager, { timeout: 60000 }, From 168d491038134ce28d25324f5510daa3384b9fb8 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 15:12:32 +0100 Subject: [PATCH 13/20] test: keep Traffic interception off workers and restore screenshot windows --- changelog.d/1281-traffic-issue-fix.md | 2 + scripts/qa-chrome.mjs | 27 ++++++++++ scripts/qa-traffic.mjs | 37 ++++--------- scripts/traffic-fixtures.mjs | 56 +++++++++++++++++++ src/qaChrome.test.mjs | 57 +++++++++++++++++++- src/tooling/trafficFixtures.test.mjs | 78 +++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 27 deletions(-) diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md index 10dc045d0..f9477de48 100644 --- a/changelog.d/1281-traffic-issue-fix.md +++ b/changelog.d/1281-traffic-issue-fix.md @@ -3,3 +3,5 @@ - Establish browser-level foreground visibility for Traffic QA and reject hidden-page samples (asturksever, #1281). - Supply CORS headers on cross-origin traffic fixtures and report browser exceptions without disabling browser security (asturksever, #1281). - Isolate the Traffic keyless browser scenario from the keyed page’s workers and intercepted cache (asturksever, #1281). +- Restore minimized test windows before screenshots so native compositor pauses cannot stall browser acceptance (asturksever, #1281). +- Intercept only Traffic provider URLs, leaving Cesium worker and module loading untouched (asturksever, #1281). diff --git a/scripts/qa-chrome.mjs b/scripts/qa-chrome.mjs index 05c4dbff0..2416a04e3 100644 --- a/scripts/qa-chrome.mjs +++ b/scripts/qa-chrome.mjs @@ -18,3 +18,30 @@ export function chromeExecutablePath({ if (env.PUPPETEER_EXECUTABLE_PATH) return env.PUPPETEER_EXECUTABLE_PATH; return exists(MACOS_CHROME) ? MACOS_CHROME : undefined; } + +/** Restore an owned test window before capture: focus emulation alone does not + * unminimize the native compositor, so CDP screenshots can otherwise stall. */ +export async function captureQaScreenshot( + page, + options, + { headful = true } = {}, +) { + if (headful) { + const session = await page.createCDPSession(); + try { + const { windowId, bounds } = await session.send( + 'Browser.getWindowForTarget', + ); + if (bounds.windowState === 'minimized') { + await session.send('Browser.setWindowBounds', { + windowId, + bounds: { windowState: 'normal' }, + }); + } + await page.bringToFront(); + } finally { + await session.detach(); + } + } + return page.screenshot(options); +} diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index d07b7196f..a6c0bdc53 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -31,6 +31,7 @@ */ import puppeteer from 'puppeteer'; +import { captureQaScreenshot } from './qa-chrome.mjs'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -49,7 +50,7 @@ appUrl.searchParams.set('welcome', '0'); const APP_URL = appUrl.href; const HEADFUL = argv.includes('--headful'); const FIXTURES = argv.includes('--fixtures'); -const fixtureResponse = FIXTURES ? (await import('./traffic-fixtures.mjs')).trafficFixtureResponse : () => null; +const { installTrafficRoutes } = await import('./traffic-fixtures.mjs'); const CHROME_EXECUTABLE_CANDIDATES = [ process.env.PUPPETEER_EXECUTABLE_PATH, @@ -158,11 +159,6 @@ async function main() { // Chrome minimized. Use browser focus emulation, not patched DOM getters. const focusSession = await page.createCDPSession(); await focusSession.send('Emulation.setFocusEmulationEnabled', { enabled: true }); - const onFixture = (request) => { - const response = fixtureResponse(request); - if (response) void request.respond(response); else void request.continue(); - }; - if (FIXTURES) { await page.setCacheEnabled(false); await page.setRequestInterception(true); page.on('request', onFixture); } await page.setViewport({ width: 1440, height: 900 }); // Track flow-tile requests + traffic console lines for (ii)/(iii)/(v). @@ -172,6 +168,7 @@ async function main() { }); const trafficLogs = []; const pageErrors = []; + if (FIXTURES) await installTrafficRoutes(page, { origin: appUrl.origin, fixtures: true, onError: error => pageErrors.push(error.message) }); page.on('pageerror', error => { pageErrors.push(error.message); console.error('PAGE ERROR:', error.message); }); page.on('console', message => { if (message.type() === 'error') console.error('BROWSER ERROR:', message.text()); }); page.on('console', (msg) => { @@ -210,7 +207,7 @@ async function main() { `coverage=${mumbai.flowCoveragePct}% tiles=${mumbai.tilesFetched}`); if (mumbai.mode !== 'live' || !(mumbai.count > 0) || !(colored > 0)) exitCode = 1; await sleep(1200); - await page.screenshot({ path: path.join(SHOTS_DIR, 'traffic-live-flow.png') }); + await captureQaScreenshot(page, { path: path.join(SHOTS_DIR, 'traffic-live-flow.png') }, { headful: HEADFUL }); } // ── (iv) uncovered-roads param — hide vs sim ───────────────────────────── @@ -243,7 +240,7 @@ async function main() { (hid.flowBuckets?.sim || 0) === 0 && hid.count > 0, `count=${hid.count} sim=${hid.flowBuckets?.sim}`); if ((hid.flowBuckets?.sim || 0) !== 0) exitCode = 1; - await page.screenshot({ path: path.join(SHOTS_DIR, 'traffic-live-hide-mode.png') }); + await captureQaScreenshot(page, { path: path.join(SHOTS_DIR, 'traffic-live-hide-mode.png') }, { headful: HEADFUL }); } else { record('PARAM: hide mode renders zero sim (white) dots', null, 'view had 100% coverage (no sim dots to hide) — inconclusive here, covered by unit tests'); @@ -265,7 +262,7 @@ async function main() { } record('C4: oblique fetch box centers on the look-at point', ok, detail); if (!ok) exitCode = 1; - await page.screenshot({ path: path.join(SHOTS_DIR, 'traffic-c4-oblique.png') }); + await captureQaScreenshot(page, { path: path.join(SHOTS_DIR, 'traffic-c4-oblique.png') }, { headful: HEADFUL }); } // ── (iii) budget honesty ───────────────────────────────────────────────── @@ -290,23 +287,11 @@ async function main() { await keylessFocus.send('Emulation.setFocusEmulationEnabled', { enabled: true }); page.on('pageerror', error => { pageErrors.push(error.message); console.error('PAGE ERROR:', error.message); }); page.on('console', message => { if (message.type() === 'error') console.error('BROWSER ERROR:', message.text()); }); - await page.setCacheEnabled(false); - await page.setRequestInterception(true); const keylessFlowReqs = []; - page.on('request', (req) => { - const url = req.url(); - if (url.includes('/api/tomtom/status')) { - req.respond({ status: 200, contentType: 'application/json', body: JSON.stringify({ hasKey: false }) }); - return; - } - if (url.includes('/api/tomtom/flow/')) { - keylessFlowReqs.push(url); - req.respond({ status: 503, contentType: 'application/json', body: JSON.stringify({ error: 'no_key' }) }); - return; - } - const response = fixtureResponse(req); - if (response) { void req.respond(response); return; } - try { req.continue(); } catch { /* already handled */ } + await installTrafficRoutes(page, { + origin: appUrl.origin, fixtures: FIXTURES, keyless: true, + onFlow: url => keylessFlowReqs.push(url), + onError: error => pageErrors.push(error.message), }); await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForFunction( @@ -338,7 +323,7 @@ async function main() { exitCode = 1; } await sleep(1000); - await page.screenshot({ path: path.join(SHOTS_DIR, 'traffic-sim-keyless.png') }); + await captureQaScreenshot(page, { path: path.join(SHOTS_DIR, 'traffic-sim-keyless.png') }, { headful: HEADFUL }); } record('BROWSER: no uncaught page exceptions', pageErrors.length === 0, pageErrors.join('; ')); } catch (e) { diff --git a/scripts/traffic-fixtures.mjs b/scripts/traffic-fixtures.mjs index 2f94993cc..0ec8cdacd 100644 --- a/scripts/traffic-fixtures.mjs +++ b/scripts/traffic-fixtures.mjs @@ -117,3 +117,59 @@ export function trafficFixtureResponse(request) { body: trafficFixtureTile(Number(road[1])), }; } + +/** Intercept provider URLs only; Cesium worker/module loading stays untouched. */ +export async function installTrafficRoutes( + page, + { + origin, + fixtures = false, + keyless = false, + onFlow = () => {}, + onError = () => {}, + }, +) { + const session = await page.createCDPSession(); + const patterns = [ + `${origin}/api/tomtom/status*`, + `${origin}/api/tomtom/flow/*`, + ...(fixtures + ? ['https://tiles.openfreemap.org/*', 'https://terrain.reearth.land/*'] + : []), + ].map((urlPattern) => ({ urlPattern, requestStage: 'Request' })); + session.on('Fetch.requestPaused', async ({ requestId, request }) => { + try { + const url = new URL(request.url); + let response; + if (keyless && url.pathname === '/api/tomtom/status') { + response = json({ hasKey: false }); + } else if (keyless && url.pathname.startsWith('/api/tomtom/flow/')) { + onFlow(request.url); + response = { ...json({ error: 'no_key' }), status: 503 }; + } else if (fixtures) { + response = trafficFixtureResponse({ url: () => request.url }); + } + if (!response) { + await session.send('Fetch.continueRequest', { requestId }); + return; + } + await session.send('Fetch.fulfillRequest', { + requestId, + responseCode: response.status, + responseHeaders: Object.entries({ + 'content-type': response.contentType, + ...response.headers, + }).map(([name, value]) => ({ name, value })), + body: Buffer.from(response.body).toString('base64'), + }); + } catch (error) { + if (page.isClosed()) return; + onError(error); + await session + .send('Fetch.failRequest', { requestId, errorReason: 'Failed' }) + .catch(() => {}); + } + }); + await session.send('Fetch.enable', { patterns }); + return session; +} diff --git a/src/qaChrome.test.mjs b/src/qaChrome.test.mjs index 4414f3c93..398d23ab3 100644 --- a/src/qaChrome.test.mjs +++ b/src/qaChrome.test.mjs @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { MACOS_CHROME, chromeExecutablePath } from '../scripts/qa-chrome.mjs'; +import { + MACOS_CHROME, + chromeExecutablePath, + captureQaScreenshot, +} from '../scripts/qa-chrome.mjs'; test('gates keep macOS Chrome where it is installed', () => { assert.equal( @@ -25,3 +29,54 @@ test('an explicit executable path wins', () => { 'C:\\Chrome\\chrome.exe', ); }); + +test('screenshots restore minimized test windows before capture and detach their session', async () => { + const calls = []; + const session = { + send: async (method, params) => { + calls.push([method, params]); + return { windowId: 7, bounds: { windowState: 'minimized' } }; + }, + detach: async () => calls.push(['detach']), + }; + const page = { + createCDPSession: async () => session, + bringToFront: async () => calls.push(['front']), + screenshot: async (options) => { + calls.push(['capture', options]); + return 'pixels'; + }, + }; + assert.equal( + await captureQaScreenshot(page, { path: 'proof.png' }), + 'pixels', + ); + assert.deepEqual( + calls.map((c) => c[0]), + [ + 'Browser.getWindowForTarget', + 'Browser.setWindowBounds', + 'front', + 'detach', + 'capture', + ], + ); + assert.deepEqual(calls[1][1], { + windowId: 7, + bounds: { windowState: 'normal' }, + }); +}); + +test('headless capture does not manipulate a native window', async () => { + const options = { path: 'proof.png' }; + const page = { + screenshot: async (supplied) => { + assert.equal(supplied, options); + return 'pixels'; + }, + }; + assert.equal( + await captureQaScreenshot(page, options, { headful: false }), + 'pixels', + ); +}); diff --git a/src/tooling/trafficFixtures.test.mjs b/src/tooling/trafficFixtures.test.mjs index e02b959e7..b5f6c6de4 100644 --- a/src/tooling/trafficFixtures.test.mjs +++ b/src/tooling/trafficFixtures.test.mjs @@ -90,3 +90,81 @@ test('flat terrain fixture decodes through Cesium and advertises no network chil await new Promise((resolve) => server.close(resolve)); } }); + +test('fixture interception targets provider URLs and leaves app workers alone', async () => { + const { installTrafficRoutes } = + await import('../../scripts/traffic-fixtures.mjs'); + const sent = []; + let paused; + const session = { + on: (_, handler) => { + paused = handler; + }, + send: async (name, args) => { + sent.push({ name, args }); + }, + }; + await installTrafficRoutes( + { createCDPSession: async () => session, isClosed: () => false }, + { origin: 'http://localhost:4173', fixtures: true }, + ); + const patterns = sent[0].args.patterns.map((p) => p.urlPattern); + assert.deepEqual(patterns, [ + 'http://localhost:4173/api/tomtom/status*', + 'http://localhost:4173/api/tomtom/flow/*', + 'https://tiles.openfreemap.org/*', + 'https://terrain.reearth.land/*', + ]); + await paused({ + requestId: 'terrain', + request: { url: 'https://terrain.reearth.land/layer.json' }, + }); + const reply = sent.at(-1); + assert.equal(reply.name, 'Fetch.fulfillRequest'); + assert.equal( + JSON.parse(Buffer.from(reply.args.body, 'base64')).format, + 'heightmap-1.0', + ); + assert.ok( + reply.args.responseHeaders.some( + (h) => h.name === 'access-control-allow-origin' && h.value === '*', + ), + ); +}); + +test('keyless interception refuses and records unexpected flow requests', async () => { + const { installTrafficRoutes } = + await import('../../scripts/traffic-fixtures.mjs'); + const sent = [], + flows = []; + let paused; + const session = { + on: (_, handler) => { + paused = handler; + }, + send: async (name, args) => { + sent.push({ name, args }); + }, + }; + await installTrafficRoutes( + { createCDPSession: async () => session, isClosed: () => false }, + { + origin: 'http://localhost:4173', + keyless: true, + onFlow: (url) => flows.push(url), + }, + ); + await paused({ + requestId: 'status', + request: { url: 'http://localhost:4173/api/tomtom/status' }, + }); + assert.deepEqual(JSON.parse(Buffer.from(sent.at(-1).args.body, 'base64')), { + hasKey: false, + }); + await paused({ + requestId: 'flow', + request: { url: 'http://localhost:4173/api/tomtom/flow/12/1/1.pbf' }, + }); + assert.equal(sent.at(-1).args.responseCode, 503); + assert.equal(flows.length, 1); +}); From 090eeb9b115ed05f498a47f372d70049e4fada4a Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:12:27 +0100 Subject: [PATCH 14/20] Settle pending Traffic height work when the population empties A closure that removes the last animated points mid height correction released the render hold, and animate() then returned before easeHeights() could run, leaving _heightPending (and getStats().loading) true. Settle every retained road on its corrected heights before the hold is released. Adds a regression combining height correction, a late closure and an empty population. Co-Authored-By: Claude Opus 5.5 --- changelog.d/1281-traffic-issue-fix.md | 1 + src/layers/traffic/animation.js | 3 +++ src/layers/traffic/motion.test.mjs | 31 +++++++++++++++++++++++++++ src/layers/traffic/retention.js | 25 ++++++++++++++++++++- 4 files changed, 59 insertions(+), 1 deletion(-) diff --git a/changelog.d/1281-traffic-issue-fix.md b/changelog.d/1281-traffic-issue-fix.md index f9477de48..3ce673038 100644 --- a/changelog.d/1281-traffic-issue-fix.md +++ b/changelog.d/1281-traffic-issue-fix.md @@ -5,3 +5,4 @@ - Isolate the Traffic keyless browser scenario from the keyed page’s workers and intercepted cache (asturksever, #1281). - Restore minimized test windows before screenshots so native compositor pauses cannot stall browser acceptance (asturksever, #1281). - Intercept only Traffic provider URLs, leaving Cesium worker and module loading untouched (asturksever, #1281). +- Settle an unfinished Traffic height correction when the last animated point leaves, so the layer stops reporting loading once its render hold is released (asturksever, #1281). diff --git a/src/layers/traffic/animation.js b/src/layers/traffic/animation.js index 4d9f6c4b3..6c58a57a4 100644 --- a/src/layers/traffic/animation.js +++ b/src/layers/traffic/animation.js @@ -25,6 +25,9 @@ export function createAnimation({ layerState._enabled && (layerState._dots.length || layerState._heatJamPrim), ); + // Height easing only runs in a held frame. Settle it before releasing, so + // a population emptied mid-correction cannot report loading forever. + if (!needed && layerState._heightPending) parts.retention.finishHeights(); if (needed === Boolean(layerState._renderHeld)) return; layerState._renderHeld = needed; if (needed) { diff --git a/src/layers/traffic/motion.test.mjs b/src/layers/traffic/motion.test.mjs index 6338eb10f..ef60e478f 100644 --- a/src/layers/traffic/motion.test.mjs +++ b/src/layers/traffic/motion.test.mjs @@ -565,6 +565,37 @@ test('provisional opposite-direction heights cannot overwrite a measured sibling assert.notEqual(forward.waypoints, reverse.waypoints); }); +test('a closure that empties the population mid-correction settles heights and clears loading before release', (t) => { + const h = setup(t); + h.parts.retention.reconcile(h.roads(), 450, true); + h.settle(); + assert.equal(h.state._renderHeld, true); + const road = h.state._roads[0]; + // Republishing the sole road starts a height correction. + const corrected = h.roads(0, 120); + h.parts.retention.reconcile(corrected, 450, true); + assert.equal(h.state._heightPending, true); + assert.equal(road.heightMoving, true); + // The closure arrives before the next population pass, which then removes + // the final points after their fade, long before the correction finishes. + corrected[0].flow = { level: 0, closure: true }; + for (let i = 0; i < 20 && h.state._dots.length; i++) h.frame(); + assert.equal(h.state._dots.length, 0); + assert.equal(h.state._heatJamPrim, null); + assert.equal(h.state._renderHeld, false); + assert.equal(h.state._heightPending, false); + assert.equal(h.state._populationPending, false); + assert.equal(road.heightMoving, false); + assert.ok( + Math.abs(C.Cartographic.fromCartesian(road.waypoints[0]).height - 120) < + 0.01, + ); + // Frames driven by other layers stay idle and cannot revive the work. + h.frame(); + assert.equal(h.state._heightPending, false); + assert.equal(h.state._renderHeld, false); +}); + test('large height corrections obey a screen-pixel bound and reuse projection scratch objects', (t) => { const h = setup(t); h.state._viewer.scene = { diff --git a/src/layers/traffic/retention.js b/src/layers/traffic/retention.js index ec0762b5c..a987f64b0 100644 --- a/src/layers/traffic/retention.js +++ b/src/layers/traffic/retention.js @@ -242,6 +242,29 @@ export function createRetention({ state, parts }) { road.heightMoving = moving; } } + /** + * Settle every retained road on its corrected heights at once. Easing only + * runs inside the held animation frame, so once nothing is left to animate + * the remaining correction would otherwise stay pending indefinitely. + * Published roads whose record was just dropped are settled too, so a later + * held frame cannot revive their correction. + */ + function finishHeights() { + const roads = new Set(state._roads); + for (const record of records.values()) roads.add(record.road); + for (const road of roads) { + if (!road.heightMoving) continue; + for (let i = 0; i < road.waypoints.length; i++) + Cesium.Cartesian3.clone(road.targetWaypoints[i], road.waypoints[i]); + for (let i = 0; i < road.segmentDist.length; i++) + road.segmentDist[i] = Cesium.Cartesian3.distance( + road.waypoints[i], + road.waypoints[i + 1], + ); + road.heightMoving = false; + } + state._heightPending = false; + } function clear() { if (timer !== null) { if (typeof cancelAnimationFrame === 'function') @@ -253,5 +276,5 @@ export function createRetention({ state, parts }) { state._populationPending = false; state._heightPending = false; } - return { reconcile, easeHeights, clear, refreshCounts }; + return { reconcile, easeHeights, finishHeights, clear, refreshCounts }; } From 1b7498820ffca1ce1418f94f6da7124226016fb4 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:12:27 +0100 Subject: [PATCH 15/20] Remove the unused trafficLogs collector from Traffic QA Co-Authored-By: Claude Opus 5.5 --- scripts/qa-traffic.mjs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index a6c0bdc53..9770999df 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -161,20 +161,15 @@ async function main() { await focusSession.send('Emulation.setFocusEmulationEnabled', { enabled: true }); await page.setViewport({ width: 1440, height: 900 }); - // Track flow-tile requests + traffic console lines for (ii)/(iii)/(v). + // Track flow-tile requests for the (iii) budget check. const flowRequests = []; page.on('request', (req) => { if (req.url().includes('/api/tomtom/flow/')) flowRequests.push(req.url()); }); - const trafficLogs = []; const pageErrors = []; if (FIXTURES) await installTrafficRoutes(page, { origin: appUrl.origin, fixtures: true, onError: error => pageErrors.push(error.message) }); page.on('pageerror', error => { pageErrors.push(error.message); console.error('PAGE ERROR:', error.message); }); page.on('console', message => { if (message.type() === 'error') console.error('BROWSER ERROR:', message.text()); }); - page.on('console', (msg) => { - const t = msg.text(); - if (t.includes('[Data:Traffic]')) trafficLogs.push(t); - }); console.log('Loading app...'); await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); @@ -248,7 +243,6 @@ async function main() { // ── (ii) C4 oblique bounds ─────────────────────────────────────────────── console.log('\n(ii) C4 — oblique fetch bounds land at the look-at point...'); - trafficLogs.length = 0; const oblique = await settleTraffic(page, { lon: -97.72, lat: 30.245, height: 2500, heading: 315, pitch: -20 }, { minCount: 100 }); { const bounds = oblique?.roadBounds; From e974ce096a750462df54bc0890344b0b48571cec Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:12:27 +0100 Subject: [PATCH 16/20] Qualify the Traffic render-hold note for failed loads A failed load leaves no hold only when no animated content is retained. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1851702dd..109c72d73 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -14,7 +14,7 @@ existing batch size and one bounded upstream chunk. Larger requests return HTTP 413 before disk-cache loading or upstream work; malformed coordinates remain 400. The browser continues to split larger workloads into sequential 64-point requests. -Traffic holds continuous rendering only while enabled with animated dots or a pulsing jam heat-line. Empty, failed, and high-altitude views release the hold; population admission reacquires it and resets animation time. Clear, disable, and destroy also release it. Retained fading dots keep animation alive until they are removed. +Traffic holds continuous rendering only while enabled with animated dots or a pulsing jam heat-line. Empty and high-altitude views release the hold. A failed load leaves no hold only when no animated content is retained: if an earlier load succeeded, ingestion keeps its animated points and the hold stays. Population admission reacquires the hold and resets animation time. Clear, disable, and destroy also release it. Retained fading dots keep animation alive until they are removed. When the last point leaves mid height correction, the correction is settled at once, before the hold is released, so it cannot keep the layer reporting loading. Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is From 730ba570738658d111d336477973902751fdd895 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:26:02 +0100 Subject: [PATCH 17/20] Report unsupported military track history in get_aircraft_track The standalone military source no longer has getTrack, so the MCP get_aircraft_track tool threw a TypeError and MCP advised a retry. It now returns an `unsupported` ToolError when the selected feed has no getTrack, while custom sources that supply history still work. Tests cover the default composition through the catalog and MCP. Also drop the unused epoch import and update the CURRENT-STATE track sections to describe local trails and 503 history_unavailable. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 6 +-- src/sources/live/standalone.js | 1 - src/tools/queries/aviation.js | 19 +++++++--- src/tools/queries/aviation.test.mjs | 57 +++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 48e44c016..739d76cc5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -9,7 +9,7 @@ broader suffix variants. Follow-ups retain boundary provenance, and cancelled lookups preserve the previous analyst cohort and boundary source. Navigation search keeps a successful geocode, with no search or view-based region recovery. -The standalone military source no longer requests the unavailable, undocumented ADSB.lol trace endpoint. Tracking retains its locally accumulated trail, and custom sources may still supply optional getTrack history. The old HTTP route validates the identifier then reports 503 history_unavailable without any upstream request, rather than forwarding a misleading 403. This does not restore historical coverage or download bulk daily archives. +The standalone military source no longer requests the unavailable, undocumented ADSB.lol trace endpoint. Tracking retains its locally accumulated trail, and custom sources may still supply optional getTrack history. The old HTTP route validates the identifier then reports 503 history_unavailable without any upstream request, rather than forwarding a misleading 403. The `get_aircraft_track` tool reports `unsupported` for a military feed without `getTrack` instead of a retry-later failure. This does not restore historical coverage or download bulk daily archives. Terrain height requests accept at most 64 points, matching the browser service's existing batch size and one bounded upstream chunk. Larger requests return HTTP @@ -4713,7 +4713,7 @@ silently demoting every later lookup for the session. A selected mission renders its orbit as four repeating tactical sectors, each containing one prominent cyan dot followed by one hundred thin translucent dashes. The bright dots act as orbit anchors while the subdued dash field remains depth-tested against the globe and is shown only for the selected mission. Close selected-pad views add one static 500 m-radius cyan launch-zone ring with a low-opacity translucent fill over the sampled photoreal launch-site surface. The single scene primitive is created only for the visible selected site and is otherwise dormant. It appears during Focus, sufficiently close manual zoom, and the replay countdown, but is suppressed above 120 km camera altitude, beyond 180 km direct camera-to-pad range, for unselected missions, and whenever Space Missions is inactive. Focus establishes a launch-site-centered camera transform once; subsequent manual heading and pitch changes remain centered on that site without an automated per-frame correction. Surface mission markers and labels use an additional conservative globe-limb margin before the exact ellipsoid occluder boundary, preventing near-horizon visibility from alternating between frames. - **AIS vessels**: chevron symbology (naval cyan base, type tints), world-space headings, MMSI-keyed reconciliation (selection survives refreshes; pinned 3 refreshes with STALE marker when absent), detection-overlay integration (`type: 'SEA'`), contextStore registration for voice Q&A. Empty-space clicks, id-less photorealistic-tile picks, and Escape dismiss the vessel card/HUD/context and clear its trail; picks owned by another layer (including `gev-trail:*`) and raw vessel-record picks without a live MMSI key are no-ops for vessel selection. Click and key handlers detach while the layer is disabled and reinstall on enable. Selecting another vessel replaces the selection and trail, and reconciliation clears a trail if its owning vessel is evicted. -- **Track trails**: server accumulates per-MMSI ring buffers (`/api/vessels/track?mmsi=`, Float32+Uint32, 64 samples, 30s/25m thinning); aircraft backfill proxies `/api/flights/track` (OAuth, own credit bucket) and `/api/military/track` (tar1090 readsb, ~24h history, ODbL — credit adsb.lol). +- **Track trails**: server accumulates per-MMSI ring buffers (`/api/vessels/track?mmsi=`, Float32+Uint32, 64 samples, 30s/25m thinning); aircraft backfill proxies `/api/flights/track` (OAuth, own credit bucket). Military aircraft keep only their locally accumulated trails: `/api/military/track` validates the hex, then answers `503 history_unavailable` without an upstream request. - Shared `src/data/pickRegistry.js` stops the two flight layers' click handlers from fighting over the camera. ### Optional Overpass configuration @@ -5277,7 +5277,7 @@ pins and share links already supply complete boxes. - `/api/overpass` is bounded by body/response caps, per-client/global rate limits, concurrency limits, operator-configured upstreams, in-flight dedupe, cache bounds, and static validation that every selector is spatially bounded. - `/api/military-installations` uses an independent limiter with the same 90-per-client/300-global one-minute bounds, so viewport installation refreshes never consume `/api/overpass` annotation capacity. - `/api/route` proxies bounded OSRM route requests for annotation routes, with profile allowlisting, distance caps, response caps, caching, and sanitized "no route found" errors. -- Track endpoints: `/api/vessels/track?mmsi=` (server-accumulated ring buffers; sub-route handled before the rows snapshot), `/api/flights/track?icao24=` (OAuth, 60s cache, sanitized errors, independent OpenSky credit bucket), `/api/military/track?hex=` (60s cache, 5MB cap, ODbL attribution required in UI). +- Track endpoints: `/api/vessels/track?mmsi=` (server-accumulated ring buffers; sub-route handled before the rows snapshot), `/api/flights/track?icao24=` (OAuth, 60s cache, sanitized errors, independent OpenSky credit bucket), `/api/military/track?hex=` (400 for an invalid hex, otherwise `503 history_unavailable` with `Retry-After: 1800` and `Cache-Control: no-store`, no upstream request; military tracking uses its locally accumulated trail). - Realtime debug logs redact API keys, bearer tokens, client secrets, and image data URLs before writing to disk; request bodies are size-capped. - `/api/realtime/debug-log` enforces an always-on 120-per-client/400-global one-minute limiter (not the opt-in `GEV_RATELIMIT_OPENAI_PER_MIN` bucket the cost-bearing OpenAI routes share), appends asynchronously through a serialized queue, and rotates `realtime-conversations.jsonl` at 32 MB keeping one prior generation, so the sink is bounded at twice that regardless of session length. A malformed record answers 400 and a failed write 500, both with a fixed message. diff --git a/src/sources/live/standalone.js b/src/sources/live/standalone.js index 7311c492d..3f2c73b3b 100644 --- a/src/sources/live/standalone.js +++ b/src/sources/live/standalone.js @@ -1,5 +1,4 @@ import { - epoch, finite, httpError, LiveSourceError, diff --git a/src/tools/queries/aviation.js b/src/tools/queries/aviation.js index a5f4babf0..153e36e03 100644 --- a/src/tools/queries/aviation.js +++ b/src/tools/queries/aviation.js @@ -333,13 +333,20 @@ export const getAircraftTrack = defineTool({ requires: ['aircraft'], async run(args, { services, signal }) { const icao24 = args.icao24.toLowerCase(); + const source = feedFor(services, args.military); + // History is optional: the standalone military source has none. + if (typeof source.getTrack !== 'function') + throw new ToolError( + 'unsupported', + args.military + ? 'Track history is not available for the military aircraft feed' + : 'Track history is not available for the aircraft feed', + ); // A track server answers 404 for an aircraft it has no positions for. - const track = await feedFor(services, args.military) - .getTrack(icao24, { signal }) - .catch((error) => { - if (error?.status === 404) return { records: [] }; - throw error; - }); + const track = await source.getTrack(icao24, { signal }).catch((error) => { + if (error?.status === 404) return { records: [] }; + throw error; + }); const points = [...track.records].sort( (a, b) => a.observedAtMs - b.observedAtMs, ); diff --git a/src/tools/queries/aviation.test.mjs b/src/tools/queries/aviation.test.mjs index 1af1352c0..484719790 100644 --- a/src/tools/queries/aviation.test.mjs +++ b/src/tools/queries/aviation.test.mjs @@ -3,6 +3,8 @@ import test from 'node:test'; import { composeCatalog, coreTools } from '../index.js'; import { airlineDesignator } from './aviation.js'; import { LiveSourceError } from '../../sources/live/contract.js'; +import { createMcpServer } from '../mcp/index.js'; +import { createToolServices } from '../services.js'; const record = (id, latitude, longitude, extra = {}) => ({ id, @@ -242,6 +244,61 @@ test('tracks are ordered and thinned to 200 points', async () => { ); }); +test('military track history is unsupported by the default composition and kept for custom sources', async () => { + const requests = []; + const services = createToolServices({ + fetchImpl: async (url) => { + requests.push(String(url)); + throw new Error('no request expected'); + }, + appUrl: 'http://localhost:5173', + }); + const catalog = composeCatalog({ tools: coreTools, services }); + await assert.rejects( + catalog.call('get_aircraft_track', { icao24: 'AE0001', military: true }), + (error) => + error.name === 'ToolError' && + error.code === 'unsupported' && + /military/.test(error.message), + ); + const server = createMcpServer({ catalog, name: 'test', version: '1.0.0' }); + const { result } = await server.handle({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'get_aircraft_track', + arguments: { icao24: 'ae0001', military: true }, + }, + }); + assert.equal(result.isError, true); + assert.equal(result.structuredContent.error, 'unsupported'); + assert.doesNotMatch(result.content[0].text, /try again|later/i); + assert.deepEqual(requests, []); + + const track = [ + { + latitude: 37.6, + longitude: -122.3, + observedAtMs: Date.UTC(2026, 0, 1), + baroAltitudeM: 3000, + onGround: false, + }, + ]; + const custom = feed(military, { track }); + const withHistory = composeCatalog({ + tools: coreTools, + services: { aircraft: feed(civil), military: custom }, + }); + const kept = await withHistory.call('get_aircraft_track', { + icao24: 'AE0001', + military: true, + }); + assert.equal(custom.calls[0], 'ae0001'); + assert.equal(kept.data.total, 1); + assert.equal(kept.data.view.follow.kind, 'military_aircraft'); +}); + test('aircraft info combines type and route lookups', async () => { const aircraft = feed(civil, { enrichment: { From 0b5b897c114e5b1ae9bbd1678f28fba6b7c6ebb2 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:26:41 +0100 Subject: [PATCH 18/20] Allow active TLE group, admit proxied same-origin requests, keep longest cooldown - CelesTrak allowlist includes `active`, which Space Missions' getActiveTle() requests; unknown groups still answer 400. - The CelesTrak gate uses admitSameSiteKeyless so the documented reverse proxy keeps working while foreign/opaque Origins and cross-site Sec-Fetch-Site are still refused with 403. - A provider failure never shortens an existing cooldown (CelesTrak and Launch Library keep the later deadline). - Tests: real getActiveTle() through the middleware, proxied app request vs unrelated Origin, concurrent failures finishing in both orders. - docs/CURRENT-STATE.md: eight groups, keyless gate, non-shortening cooldown. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 15 +-- server/providers/space/celestrak.js | 15 ++- server/providers/space/launch-library.js | 8 +- src/proxyErrorResponses.test.mjs | 1 - src/tooling/spaceProviders.test.mjs | 126 ++++++++++++++++++++++- 5 files changed, 151 insertions(+), 14 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 870eb51aa..82fc9201b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -9,12 +9,15 @@ broader suffix variants. Follow-ups retain boundary provenance, and cancelled lookups preserve the previous analyst cohort and boundary source. Navigation search keeps a successful geocode, with no search or view-based region recovery. -CelesTrak accepts only the seven satellite groups used by the app and gates -requests before provider work. CelesTrak and Launch Library failures start a -provider-wide cooldown: Retry-After seconds or HTTP dates are bounded to 5 seconds -through 30 minutes, with a 60-second fallback. During cooldown dated last-good -data is served with stale provenance, otherwise the proxy returns a bounded -error without another upstream attempt. Error bodies are cancelled before reading. +CelesTrak accepts only the eight satellite groups used by the app (the seven +satellite-layer groups plus `active` for Space Missions) and gates cross-site +requests before provider work with the keyless same-site policy, so the +documented reverse proxy still works. CelesTrak and Launch Library failures +start a provider-wide cooldown that a later failure never shortens: Retry-After +seconds or HTTP dates are bounded to 5 seconds through 30 minutes, with a +60-second fallback. During cooldown dated last-good data is served with stale +provenance, otherwise the proxy returns a bounded error without another upstream +attempt. Error bodies are cancelled before reading. Terrain height requests accept at most 64 points, matching the browser service's existing batch size and one bounded upstream chunk. Larger requests return HTTP diff --git a/server/providers/space/celestrak.js b/server/providers/space/celestrak.js index 315e9b3f0..013fce1b1 100644 --- a/server/providers/space/celestrak.js +++ b/server/providers/space/celestrak.js @@ -2,7 +2,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js'; import { providerCooldownMs } from './cooldown.js'; -import { admitSameSite } from '../common/same-site.js'; +import { admitSameSiteKeyless } from '../common/same-site.js'; import { readResponseTextCapped } from '../common/http.js'; /** @@ -34,6 +34,8 @@ export function celestrakProxy() { 'galileo', 'geo', 'starlink', + // Space Missions (`getActiveTle`) reads the full active catalog. + 'active', ]); let cooldownUntil = 0; const mem = new Map(); // group -> { at: epochMs, body: string } @@ -77,7 +79,12 @@ export function celestrakProxy() { }, }); if (!res.ok) { - cooldownUntil = Date.now() + providerCooldownMs(res); + // Group refreshes run in parallel: a later failure without Retry-After + // must never shorten a longer deadline an earlier one already set. + cooldownUntil = Math.max( + cooldownUntil, + Date.now() + providerCooldownMs(res), + ); void res.body?.cancel().catch(() => {}); throw new Error(`HTTP ${res.status}`); } @@ -96,7 +103,9 @@ export function celestrakProxy() { const group = String(req.url || '') .replace(/^\//, '') .split('?')[0]; - if (admitSameSite(req, res)) return; + // Keyless and needed to render the satellites layer: refuse cross-site + // browsers but keep the documented reverse-proxy deployment working. + if (admitSameSiteKeyless(req, res)) return; if (req.method !== 'GET') { res.writeHead(405); res.end(); diff --git a/server/providers/space/launch-library.js b/server/providers/space/launch-library.js index 4e8ac629d..f95be2056 100644 --- a/server/providers/space/launch-library.js +++ b/server/providers/space/launch-library.js @@ -90,8 +90,12 @@ export function rocketLaunchesProxy() { headers: launchLibraryRequestHeaders(), }); if (!upstream.ok) { - cooldownUntil = Date.now() + providerCooldownMs(upstream); - cooldownStatus = upstream.status; + const deadline = Date.now() + providerCooldownMs(upstream); + // Never shorten a longer cooldown that is already in force. + if (deadline >= cooldownUntil) { + cooldownUntil = deadline; + cooldownStatus = upstream.status; + } void upstream.body?.cancel().catch(() => {}); const error = new Error(`upstream HTTP ${upstream.status}`); error.upstreamStatus = upstream.status; diff --git a/src/proxyErrorResponses.test.mjs b/src/proxyErrorResponses.test.mjs index d215e78a3..4f9ef7602 100644 --- a/src/proxyErrorResponses.test.mjs +++ b/src/proxyErrorResponses.test.mjs @@ -25,7 +25,6 @@ function fixture(name, overrides = {}, preview = false) { const logs = []; const deps = { readResponseTextCapped, coalesceProxyRequest, providerCooldownMs, - admitSameSite: () => false, // The rate-limited proxies reach for these; give them the real ones so a // fixture request is throttled exactly as production would be. makeRateLimiter, clientKey, diff --git a/src/tooling/spaceProviders.test.mjs b/src/tooling/spaceProviders.test.mjs index 2abc9d5fc..7aef775b1 100644 --- a/src/tooling/spaceProviders.test.mjs +++ b/src/tooling/spaceProviders.test.mjs @@ -12,6 +12,7 @@ import { launchLibraryRecentUrl, } from 'gods-eye-view/sources/space'; import * as compatibility from '../../server/providers/local.js'; +import { createLaunchSource } from 'gods-eye-view/layers/launches/source'; function install(plugin, preview = false) { const routes = new Map(); @@ -22,17 +23,23 @@ function install(plugin, preview = false) { }, }, }); - return async (route, url = '/', method = 'GET') => { + return async (route, url = '/', method = 'GET', headers = {}) => { const res = { headersSent: false, + headers: {}, writeHead(status, headers) { Object.assign(this, { status, headers, headersSent: true }); }, + // The same-site gates answer 403 through statusCode + setHeader. + setHeader(name, value) { + this.headers[name] = value; + }, end(body) { this.body = body; + this.status ??= this.statusCode; }, }; - await routes.get(route)({ url, method }, res); + await routes.get(route)({ url, method, headers, socket: {} }, res); return res; }; } @@ -270,3 +277,118 @@ test('space providers honor Retry-After, gate groups, preserve stale dates and r } } }); + +test('Space Missions reads the active catalog through the real proxy while unknown groups stay rejected', async (t) => { + isolateDisk(t); + const tle = 'ISS\n1 25544U fixture\n2 25544 fixture\n'; + const fetched = []; + t.mock.method(globalThis, 'fetch', async (url) => { + fetched.push(new URL(url).searchParams.get('GROUP')); + return new Response(tle); + }); + const request = install(celestrakProxy()); + // Route the browser source's real request into the middleware. + const source = createLaunchSource({ + fetchImpl: async (url) => { + const prefix = '/api/celestrak'; + assert.ok(url.startsWith(`${prefix}/`)); + const res = await request(prefix, url.slice(prefix.length)); + return new Response(res.body, { status: res.status }); + }, + }); + assert.equal(await source.getActiveTle(), tle); + assert.deepEqual(fetched, ['active']); + for (const group of ['/not-a-group', '/active-extra', '/../active']) + assert.equal((await request('/api/celestrak', group)).status, 400); + assert.deepEqual(fetched, ['active'], 'rejected groups never reach upstream'); +}); + +test('CelesTrak admits a same-origin request behind the documented reverse proxy and refuses foreign origins', async (t) => { + isolateDisk(t); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls++; + return new Response('ISS\n1 25544U fixture\n2 25544 fixture\n'); + }); + const request = install(celestrakProxy()); + const proxied = { + host: 'gev.example.test', + 'x-forwarded-for': '203.0.113.7', + 'x-forwarded-host': 'gev.example.test', + 'x-forwarded-proto': 'https', + via: '1.1 proxy', + }; + const app = await request('/api/celestrak', '/stations', 'GET', { + ...proxied, + origin: 'https://gev.example.test', + 'sec-fetch-site': 'same-origin', + }); + assert.equal(app.status, 200); + assert.equal(app.headers['x-tle-cache'], 'MISS'); + assert.equal(calls, 1); + for (const headers of [ + { ...proxied, origin: 'https://unrelated.example' }, + { ...proxied, origin: 'null' }, + { ...proxied, 'sec-fetch-site': 'cross-site' }, + ]) { + const refused = await request('/api/celestrak', '/visual', 'GET', headers); + assert.equal(refused.status, 403); + } + assert.equal(calls, 1, 'refused requests do no provider or cache work'); +}); + +for (const longFirst of [true, false]) + test(`CelesTrak keeps the longest cooldown when concurrent failures finish ${longFirst ? 'long then short' : 'short then long'}`, async (t) => { + isolateDisk(t); + t.mock.method(console, 'warn', () => {}); + let now = Date.now(); + t.mock.method(Date, 'now', () => now); + const pending = new Map(); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (url) => { + calls++; + const group = new URL(url).searchParams.get('GROUP'); + await new Promise((resolve) => pending.set(group, resolve)); + return group === 'stations' + ? new Response('slow down', { + status: 429, + headers: { 'Retry-After': '1800' }, + }) + : new Response('unavailable', { status: 503 }); + }); + const request = install(celestrakProxy()); + const long = request('/api/celestrak', '/stations'); + const short = request('/api/celestrak', '/visual'); + while (pending.size < 2) await new Promise(setImmediate); + const order = longFirst + ? [ + ['stations', long], + ['visual', short], + ] + : [ + ['visual', short], + ['stations', long], + ]; + for (const [group, response] of order) { + pending.get(group)(); + assert.equal((await response).status, 502); + } + assert.equal(calls, 2); + const cooling = await request('/api/celestrak', '/geo'); + assert.equal(cooling.status, 502); + assert.equal(cooling.headers['Retry-After'], '1800'); + now += 61_000; + assert.equal((await request('/api/celestrak', '/geo')).status, 502); + assert.equal( + calls, + 2, + 'the 60 s fallback did not replace the 30 min deadline', + ); + now += 1800_000; + t.mock.method( + globalThis, + 'fetch', + async () => new Response('ISS\n1 25544U fixture\n2 25544 fixture\n'), + ); + assert.equal((await request('/api/celestrak', '/geo')).status, 200); + }); From 82190d531a99c507685b2d10e0330bb7f7f7a5b4 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:39:56 +0100 Subject: [PATCH 19/20] Reset the Radio rejection counter on destroy Clear _rejectedStationCount alongside _degraded so a destroyed and re-initialized layer does not report the previous catalog's count. Co-Authored-By: Claude Opus 5.5 --- src/layers/radio/lifecycle.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/layers/radio/lifecycle.js b/src/layers/radio/lifecycle.js index 414cfa5c0..45d66892e 100644 --- a/src/layers/radio/lifecycle.js +++ b/src/layers/radio/lifecycle.js @@ -108,6 +108,7 @@ export function createLifecycle({ layerState._loading = false; layerState._stale = false; layerState._degraded = false; + layerState._rejectedStationCount = 0; layerState._error = null; layerState._updatedAt = null; layerState._managerLifecyclePresentation = null; From f93982f55e4b15885def69e2e69273adc627abd2 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:39:56 +0100 Subject: [PATCH 20/20] Carry server structural rejections into Radio diagnostics The proxy now skips malformed upstream rows individually but counts them, capped at 10,000, in the catalog metadata and the /stations envelope (rejectedStationCount). The count never fails health policy, so an otherwise healthy generation is still accepted. The client validates the field, adds its own rejected rows after admission, and reports degraded coverage when either side skipped rows. Adds an end-to-end proxy-to-client test and aligns DATA_SOURCES.md and CURRENT-STATE.md. Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 2 +- changelog.d/1268-radio-issue-fix.md | 2 +- docs/CURRENT-STATE.md | 14 +++++-- server/providers/radio/catalog.js | 39 +++++++++++++------ server/providers/radio/constants.js | 3 ++ src/data/radio.test.mjs | 38 +++++++++++++++++- src/data/radioProxy.test.mjs | 60 ++++++++++++++++++++++++++++- src/layers/radio/ingestion.js | 20 +++++++++- src/layers/radio/policy.js | 4 ++ 9 files changed, 159 insertions(+), 23 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 6d2ee498b..3557c4137 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -177,7 +177,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All sixteen registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Hamilton, Guelph, Burlington and YRT publish route ids that are not the number on the vehicle, so those entries carry a route-name table taken from each operator’s own published GTFS; any id missing from a table renders unchanged rather than blank. A label is never invented: YRT runs a route `9101` that appears nowhere in its published routes, so it stays as the feed sends it. Burlington’s is the one that cannot be shortcut: `351` is route 1 while `3510` is route 10, so the table is a real join rather than a trimmed prefix. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Barrie and YRT/Viva each invite acceptance of their licence through a form, and each licence also states that using the data is itself acceptance; this deployment relies on that clause, which is recorded in both entries. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. Ottawa’s OC Transpo falls to the same keyless admission rule as GO Transit — its realtime feed requires an API key. London Transit and Thunder Bay Transit publish vehicle positions over plain HTTP with no HTTPS endpoint available. Every other Ontario system — Brampton, Oakville, Niagara, Milton, Windsor, Greater Sudbury, UP Express and a dozen smaller operators — publishes static GTFS schedules only, with no realtime vehicle positions to register. Between the nine included and these exclusions, every Ontario realtime vehicle-position feed catalogued in the Mobility Database is accounted for. -- **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. +- **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and non-array or empty successful payloads are rejected atomically. Individual structurally malformed rows are skipped rather than failing the whole query; the response carries their bounded `rejectedStationCount` (at most 10,000), which the client adds to its own rejected rows and reports as degraded coverage without discarding the accepted generation. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. - **Re:Earth Terrain.** Keyless (no API key). Used two ways: (1) `src/mapStackController.js` swaps in a `Cesium.CesiumTerrainProvider` pointed at Re:Earth's `cesium-mesh/ellipsoid` quantized-mesh endpoint for globe stacks without a Cesium ion token (e.g. OSM), replacing a flat `EllipsoidTerrainProvider`; falls back to the flat provider if the endpoint can't be reached. (2) The server-side `/api/terrain/heights` proxy (disk-cached, serve-stale) resolves per-point ellipsoidal ground height for entity placement. Both are best-effort with a keyless-safe fallback (bundled EGM96 geoid math) if Re:Earth is unreachable. diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md index 7a5be28b7..ab68243d6 100644 --- a/changelog.d/1268-radio-issue-fix.md +++ b/changelog.d/1268-radio-issue-fix.md @@ -1,2 +1,2 @@ -- Normalize station tags and retain valid mixed rows (asturksever, #1268). +- Normalize station tags and retain valid mixed rows; skipped malformed rows still report degraded coverage (asturksever, #1268). - Correct Radio QA push-to-talk setup to target the mounted voice controller (asturksever, #1268). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 12edc4f97..0001b850f 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -11,10 +11,16 @@ search keeps a successful geocode, with no search or view-based region recovery. Radio tags are lowercased and whitespace-normalized before the final 80-character cap, including requested-tag coverage matching. Malformed station rows are skipped -individually; valid rows retain their accepted catalog generation. The UI snapshot -reports a bounded `rejectedStationCount` and degraded coverage when rows were -skipped. Malformed envelope/freshness/generation metadata and all-invalid responses -still preserve the previous usable catalog. +individually on the server and the client; valid rows retain their accepted +catalog generation. The server's `/stations` envelope carries +`rejectedStationCount`, the structurally malformed upstream rows it skipped +(capped at 10,000; policy-filtered rows such as offline or non-HTTPS streams are +not counted). It is diagnostic only and never fails catalog health. The client +adds its own rejected rows after admission, so the UI snapshot reports the +combined, capped `rejectedStationCount` and degraded coverage when either side +skipped rows. Destroy resets the count. Malformed envelope, freshness, generation +or rejection-count metadata and all-invalid responses still preserve the previous +usable catalog. Terrain height requests accept at most 64 points, matching the browser service's existing batch size and one bounded upstream chunk. Larger requests return HTTP diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js index de66ccbfb..30e6ed4ec 100644 --- a/server/providers/radio/catalog.js +++ b/server/providers/radio/catalog.js @@ -25,6 +25,7 @@ import { RADIO_USER_AGENT, RADIO_UUID_RE, RADIO_FALLBACK_MIRRORS, + RADIO_REJECTED_STATION_COUNT_MAX, } from './constants.js'; async function mapRadioConcurrent(values, concurrency, mapper) { const results = new Array(values.length); @@ -177,17 +178,22 @@ export function createRadioProxyMiddleware({ const rows = await fetchPath(`/json/stations/search?${params}`); if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - const stations = rows - .filter( - (row) => - row && - typeof row === 'object' && - !Array.isArray(row) && - typeof row.stationuuid === 'string' && - typeof row.name === 'string' && - (typeof row.url_resolved === 'string' || - typeof row.url === 'string'), - ) + const structuralRows = rows.filter( + (row) => + row && + typeof row === 'object' && + !Array.isArray(row) && + typeof row.stationuuid === 'string' && + typeof row.name === 'string' && + (typeof row.url_resolved === 'string' || + typeof row.url === 'string'), + ); + // Malformed rows are skipped individually, but counted so the + // catalog keeps a diagnostic for upstream schema drift. Policy + // rejections in normalization (offline, non-HTTPS, codec) are normal + // filtering and are not counted. + const rejectedRows = rows.length - structuralRows.length; + const stations = structuralRows .map(normalizeRadioBrowserStation) .filter(Boolean); const requestedTag = normalizeRadioTag(tag); @@ -206,9 +212,10 @@ export function createRadioProxyMiddleware({ // also contain an accepted station tagged for the requested category. succeeded: stations.length > 0 && requestedTagCovered, stations, + rejectedRows, }; } catch { - return { succeeded: false, stations: [] }; + return { succeeded: false, stations: [], rejectedRows: 0 }; } }, ); @@ -248,6 +255,10 @@ export function createRadioProxyMiddleware({ if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); const degraded = healthReasons.length > 0; + const rejectedStationCount = Math.min( + RADIO_REJECTED_STATION_COUNT_MAX, + outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0), + ); const coverage = { successfulQueries, totalQueries: queries.length, @@ -262,6 +273,9 @@ export function createRadioProxyMiddleware({ degraded, degradedReason: degraded ? healthReasons.join(',') : null, coverage, + // Diagnostic only: it never fails health policy, so a usable generation + // with a few skipped rows is still admitted. The client reports it. + rejectedStationCount, }; if (degraded && catalogCache) { const error = new Error( @@ -351,6 +365,7 @@ export function createRadioProxyMiddleware({ coverage: catalog.coverage || null, acceptedGeneration: catalog.acceptedGeneration ?? null, catalogInstance, + rejectedStationCount: catalog.rejectedStationCount ?? 0, }); } catch (error) { sendJson(res, 503, { diff --git a/server/providers/radio/constants.js b/server/providers/radio/constants.js index 0fee93ac6..b0e24babc 100644 --- a/server/providers/radio/constants.js +++ b/server/providers/radio/constants.js @@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil( export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js'; +// Upper bound for the structurally malformed upstream rows reported in the +// /stations envelope. The client layer validates against the same bound. +export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000; export const RADIO_FALLBACK_MIRRORS = Object.freeze([ 'https://de1.api.radio-browser.info', 'https://de2.api.radio-browser.info', diff --git a/src/data/radio.test.mjs b/src/data/radio.test.mjs index 8d24adc19..99963df16 100644 --- a/src/data/radio.test.mjs +++ b/src/data/radio.test.mjs @@ -723,13 +723,47 @@ test('unusable directory responses preserve warm client state atomically', async assert.equal(getRadioUIState().stale, false); assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); assert.equal(getRadioAcceptedCatalogSnapshot().stations[0].id, replacement.id); + // Server-side structural rejections are combined with client rejections + // after admission; the healthy generation is still accepted. globalThis.fetch = async () => new Response(JSON.stringify({ - stations: [{ id: 'malformed' }], stale: false, degraded: false, + stations: [replacement, { id: 'malformed' }], stale: false, degraded: false, acceptedGeneration: 3, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount: 2, })); await radioLayer.update(); - assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioUIState().rejectedStationCount, 3); + assert.equal(getRadioUIState().degraded, true); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 3); + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement], stale: false, degraded: false, + acceptedGeneration: 4, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount: 10_000, + })); + await radioLayer.update(); + assert.equal(getRadioUIState().rejectedStationCount, 10_000); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); + for (const rejectedStationCount of [-1, 1.5, '2', null, 10_001]) { + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement], stale: false, degraded: false, + acceptedGeneration: 5, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); + assert.equal(getRadioUIState().stale, true); + } + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [{ id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 5, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); assert.equal(getRadioUIState().stale, true); + assert.equal(getRadioUIState().rejectedStationCount, 10_000); + radioLayer.destroy(); + radioLayer.init(viewer); + assert.equal(getRadioUIState().rejectedStationCount, 0); + assert.equal(getRadioUIState().degraded, false); } finally { globalThis.fetch = originalFetch; radioLayer.destroy(); diff --git a/src/data/radioProxy.test.mjs b/src/data/radioProxy.test.mjs index d1c4d2660..e9df10678 100644 --- a/src/data/radioProxy.test.mjs +++ b/src/data/radioProxy.test.mjs @@ -11,7 +11,14 @@ import { publicRadioStation, publicRadioHttpsUrl, } from '../../vite.config.js'; -import { rankRadioStationsForRequest } from './radio.js'; +import { + getRadioAcceptedCatalogSnapshot, + getRadioUIState, + radioLayer, + rankRadioStationsForRequest, +} from './radio.js'; +import { RADIO_REJECTED_STATION_COUNT_MAX as SERVER_REJECTED_MAX } from '../../server/providers/radio/constants.js'; +import { RADIO_REJECTED_STATION_COUNT_MAX as CLIENT_REJECTED_MAX } from '../layers/radio/policy.js'; const UUID = '12345678-1234-4234-8234-123456789abc'; const publicLookup = async () => [{ address: '93.184.216.34', family: 4 }]; @@ -541,3 +548,54 @@ test('case expansion is capped after tag normalization in the portable and serve assert.equal(row.tags[0], 'i\u0307'.repeat(40)); assert.deepEqual(row.tags.slice(1), ['news radio']); }); + +test('server structural rejections reach the client diagnostic without discarding a healthy generation', async () => { + const originalFetch = globalThis.fetch; + const middleware = createProxy({ + now: () => Date.now(), + fetchImpl: async (url) => { + if (String(url).includes('/json/servers')) return responseJson([{ name: 'de1.api.radio-browser.info' }]); + const rows = healthyRowsForQuery(url); + if (queryTag(url)) return responseJson(rows); + // 400 valid broad rows, one row missing its required name, and one + // schema-valid row that normalization filters by policy (not counted). + const { name: _name, ...nameless } = station({ stationuuid: '50000000-0000-4000-8000-000000000001' }); + return responseJson([...rows, nameless, station({ stationuuid: '50000000-0000-4000-8000-000000000002', lastcheckok: 0 })]); + }, + }); + const response = await invoke(middleware, '/stations'); + assert.equal(response.status, 200); + const body = JSON.parse(response.body); + assert.equal(body.degraded, false); + assert.equal(body.degradedReason, null); + assert.equal(body.acceptedGeneration, 1); + assert.equal(body.rejectedStationCount, 1); + assert.equal(SERVER_REJECTED_MAX, CLIENT_REJECTED_MAX); + + const viewer = { + camera: { positionWC: { x: 7_000_000, y: 0, z: 0 } }, + scene: { canvas: { disableRootEvents: true, onwheel: null, addEventListener() {}, removeEventListener() {} } }, + dataSources: { add() {}, remove() {} }, + entities: { add(entity) { return entity; }, remove() {} }, + }; + radioLayer.destroy(); + try { + globalThis.fetch = async () => new Response(response.body, { + status: response.status, + headers: { 'Content-Type': 'application/json' }, + }); + radioLayer.init(viewer); + radioLayer.enable(); + await radioLayer.update(); + const state = getRadioUIState(); + assert.equal(state.stationCount, body.stations.length); + assert.equal(state.rejectedStationCount, 1); + assert.equal(state.degraded, true); + assert.equal(state.stale, false); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 1); + assert.equal(getRadioAcceptedCatalogSnapshot().instance, body.catalogInstance); + } finally { + globalThis.fetch = originalFetch; + radioLayer.destroy(); + } +}); diff --git a/src/layers/radio/ingestion.js b/src/layers/radio/ingestion.js index 5a0ccfa1d..79f348469 100644 --- a/src/layers/radio/ingestion.js +++ b/src/layers/radio/ingestion.js @@ -2,6 +2,7 @@ import { msg } from '../../i18n/index.js'; import { RADIO_DIRECTORY_STALE_MS, RADIO_DIRECTORY_FUTURE_SKEW_MS, + RADIO_REJECTED_STATION_COUNT_MAX, } from './policy.js'; // Directory errors stay English in layer state and stats; the layer row and @@ -64,6 +65,18 @@ export function createIngestion({ typeof body.degraded !== 'boolean' ) throw new Error('Radio directory freshness metadata was malformed'); + // Rows the server skipped as structurally malformed. Older envelopes + // omit the field; a present value must be a bounded count. + const serverRejectedStationCount = + body.rejectedStationCount === undefined + ? 0 + : body.rejectedStationCount; + if ( + !Number.isSafeInteger(serverRejectedStationCount) || + serverRejectedStationCount < 0 || + serverRejectedStationCount > RADIO_REJECTED_STATION_COUNT_MAX + ) + throw new Error('Radio directory diagnostic metadata was malformed'); const rows = body.stations; const acceptedRows = rows.filter( parts.catalogModel.isValidRadioDirectoryStation, @@ -142,9 +155,12 @@ export function createIngestion({ } if (!repeatingAcceptedGeneration) layerState._updatedAt = updatedAt; } + // Combine server-side structural rejections with rows this client + // refused. The count is diagnostic: it marks coverage degraded but does + // not undo the generation admitted above. layerState._rejectedStationCount = Math.min( - 1000000, - rows.length - acceptedRows.length, + RADIO_REJECTED_STATION_COUNT_MAX, + serverRejectedStationCount + rows.length - acceptedRows.length, ); layerState._degraded = body.degraded || layerState._rejectedStationCount > 0; diff --git a/src/layers/radio/policy.js b/src/layers/radio/policy.js index 0d4ad174a..e0438c780 100644 --- a/src/layers/radio/policy.js +++ b/src/layers/radio/policy.js @@ -24,6 +24,10 @@ export const RADIO_DIRECTORY_STALE_MS = 7 * 24 * 60 * 60 * 1000; export const RADIO_DIRECTORY_FUTURE_SKEW_MS = 5 * 60 * 1000; +// Bound for the server's `rejectedStationCount` envelope field and the +// combined client counter; matches server/providers/radio/constants.js. +export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000; + export const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;