From fa9af39e7b3a0b4be845a182062d1eea9ea5238a Mon Sep 17 00:00:00 2001 From: Pedro Lobato <69770518+Lob26@users.noreply.github.com> Date: Tue, 15 Sep 2026 11:21:07 -0500 Subject: [PATCH 01/22] fix(terrain): bound the client terrain-height cache with LRU eviction The in-memory cache in the terrain-height resolver had no eviction, so a session retained every coordinate it ever resolved for the lifetime of the page. The 5-decimal rounding documented in the file header makes repeated visits to one place share a key, but it does not bound how many distinct places a session visits. Cap it at 20 000 entries, overridable per instance so the bound is testable without minting twenty thousand coordinates. Eviction is least-recently-used rather than insertion-ordered because revisits here are spatial: the cell the camera is parked on must not be dropped for having been resolved early. The bound sits far above a city-scale session at the ~111 m floor grid, so normal use never evicts and issues no additional proxy requests. Batch results are assembled from what the call resolved, so eviction during a large batch cannot report a just-resolved point as unresolved. --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 16 ++++ scripts/format-scope.json | 1 + src/services/terrainHeights.js | 96 ++++++++++++++++++-- src/services/terrainHeights.test.mjs | 130 +++++++++++++++++++++++++++ 5 files changed, 236 insertions(+), 9 deletions(-) create mode 100644 src/services/terrainHeights.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3d01413..890bad535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - Preserve Nepal shot camera and map ownership through the public layer lifecycle; passive restoration does not start standalone playback. +- Bound the client terrain-height cache at 20 000 entries with least-recently-used eviction, so a long session no longer retains every coordinate it ever resolved. Consumer reads promote their entry and a batch still reports every point it resolved. + - Separate transit snapshot/history acquisition from the layer and expose its bounded request service independently of Vite. Preserve feed selection, playback, cache policy and compatibility exports. - Widen opaque sensor halos to 3 display pixels with a 30 px fleet core while retaining at least 60% opaque core coverage, adding contrast margin through thermal blur and bloom on bright roofs. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 40e441b25..bf6467972 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -568,6 +568,22 @@ under `gods-eye-view/server/providers/`. Portable terrain mechanics, traffic til math and GBFS source rules are available under `gods-eye-view/sources/`. The browser layers and their rendering remain in their existing modules. +## Terrain height cache bound + +The client terrain-height resolver's in-memory cache is bounded at 20 000 +entries (`DEFAULT_MAX_CACHE_ENTRIES`, overridable per instance through +`createTerrainHeights({ maxCacheEntries })`). Coordinate rounding makes repeated +visits to one place share a key but does not bound how many distinct places a +session visits, and the cache previously had no eviction, so a long session +retained every coordinate it ever resolved. Eviction is least-recently-used: +a consumer read promotes its entry, so the cell being watched is not dropped for +having been resolved early. The bound sits far above a city-scale session, so +normal use never evicts and issues no additional proxy requests. A batch's +results are assembled from what that call resolved, so eviction during a large +batch cannot report a just-resolved point as unresolved. Re:Earth and +geoid-fallback entry semantics, the fallback cooldown and the abort-time flush +are unchanged. + ## Landmark annotation identity When a landmark geocode contains only address components, annotations retain diff --git a/scripts/format-scope.json b/scripts/format-scope.json index b86632108..145739fb1 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -609,6 +609,7 @@ "src/data/surfaceServices.js", "src/annotations/resolver.js", "src/services/terrainHeights.js", + "src/services/terrainHeights.test.mjs", "src/services/groundFloor.js", "src/services/meshFloorSampler.js", "src/services/groundSnap.js", diff --git a/src/services/terrainHeights.js b/src/services/terrainHeights.js index 4366d632a..747eda145 100644 --- a/src/services/terrainHeights.js +++ b/src/services/terrainHeights.js @@ -1,7 +1,19 @@ import { ensureGeoidReady, geoidHeight } from '../data/geoid.js'; -/** Construct an instance-owned terrainHeights service with explicit dependencies. */ -export function createTerrainHeights({ source, signal }) { +/** Default live-entry ceiling for the in-memory cache (see CACHE_MAX_ENTRIES). */ +export const DEFAULT_MAX_CACHE_ENTRIES = 20_000; + +/** + * Construct an instance-owned terrainHeights service with explicit dependencies. + * `maxCacheEntries` overrides the cache bound; it exists so the bound is + * testable without minting twenty thousand coordinates, mirroring the injectable + * `maxBytes` on the GBFS reader. + */ +export function createTerrainHeights({ + source, + signal, + maxCacheEntries = DEFAULT_MAX_CACHE_ENTRIES, +}) { if (typeof source?.getHeights !== 'function') throw new TypeError('Terrain heights require a source'); signal?.throwIfAborted(); @@ -45,15 +57,71 @@ export function createTerrainHeights({ source, signal }) { /** Avoid repeatedly hitting a known-failing proxy from warm fallback reads. */ const GEOID_FALLBACK_COOLDOWN_MS = 60_000; + /** + * Live entry ceiling for the cache below. The rounding in point 2 of the file + * header makes repeated visits to one place share a key; it does nothing to + * bound the number of DISTINCT places a session visits, and this cache had no + * eviction, so a long session kept every coordinate it ever resolved for the + * lifetime of the page. + * + * This is a bound, not a budget. At the ~111 m floor grid a city-scale + * session stays well under it, so eviction is the pathological case rather + * than the normal one and a well-behaved session sees no extra network. Sized + * above the sibling caches (adsb.lol points 80, track history 200, TomTom + * tiles 256) because a miss here costs a proxy round trip, not just RAM. + */ + const CACHE_MAX_ENTRIES = + Number.isFinite(maxCacheEntries) && maxCacheEntries > 0 + ? Math.floor(maxCacheEntries) + : DEFAULT_MAX_CACHE_ENTRIES; + /** * In-memory cache: `"lat.toFixed(5),lon.toFixed(5)"` -> `{ellipsoid, source}`. * Module-scoped (not exported) — the only reads are through * `cachedEllipsoidalGround` and the internal lookup in - * `resolveEllipsoidalGround`. + * `resolveEllipsoidalGround`. Bounded at CACHE_MAX_ENTRIES; write through + * `cacheStore` and read a consumer-facing hit through `cacheTouch` so the + * eviction order stays least-recently-used. * @type {Map} */ const cache = new Map(); + /** + * Store an entry and drop the coldest ones once past the cap. `Map` iterates + * in insertion order, so deleting the first key evicts the least recently + * used provided every write and every consumer read re-inserts its key. + * + * Recency rather than plain insertion order because revisits here are + * spatial: a session comes back to cells near where it already was, so the + * cell it is parked on must not be evicted merely for having been resolved + * early. + * @param {string} key + * @param {{ellipsoid: number, source: 'reearth'|'geoid-fallback', retryAt?: number}} entry + */ + function cacheStore(key, entry) { + cache.delete(key); + cache.set(key, entry); + while (cache.size > CACHE_MAX_ENTRIES) { + const coldest = cache.keys().next().value; + if (coldest === undefined) break; + cache.delete(coldest); + } + } + + /** + * Read an entry and promote it to newest, so a coordinate a consumer keeps + * asking about survives eviction. Returns undefined on a miss. + * @param {string} key + * @returns {{ellipsoid: number, source: 'reearth'|'geoid-fallback', retryAt?: number}|undefined} + */ + function cacheTouch(key) { + const entry = cache.get(key); + if (entry === undefined) return undefined; + cache.delete(key); + cache.set(key, entry); + return entry; + } + /** * Builds the rounded cache key shared between the in-memory cache and the * `points` string sent to the proxy (see file header, point 2). @@ -74,7 +142,7 @@ export function createTerrainHeights({ source, signal }) { * @returns {number|null} */ function cachedEllipsoidalGround(lat, lon) { - const entry = cache.get(cacheKey(lat, lon)); + const entry = cacheTouch(cacheKey(lat, lon)); return entry ? entry.ellipsoid : null; } @@ -91,7 +159,7 @@ export function createTerrainHeights({ source, signal }) { * @returns {number|null} */ function cachedRealEllipsoidalGround(lat, lon) { - const entry = cache.get(cacheKey(lat, lon)); + const entry = cacheTouch(cacheKey(lat, lon)); return entry && entry.source === 'reearth' ? entry.ellipsoid : null; } @@ -213,6 +281,12 @@ export function createTerrainHeights({ source, signal }) { uncached.push(item); } + // What this call resolved, so the assembly below can report a point even + // if a later chunk's writes pushed it past CACHE_MAX_ENTRIES. Without it a + // batch big enough to evict its own earlier chunks would report points as + // unresolved that it had in fact just resolved. + const resolvedThisCall = new Map(); + // Resolve the network path in sequential <=CHUNK_SIZE chunks. Each chunk's // failure is isolated to that chunk's points (geoid fallback), so a single // bad chunk doesn't lose results for the rest of a large batch. @@ -230,7 +304,9 @@ export function createTerrainHeights({ source, signal }) { // one contact frozen at the geoid while its neighbors resolved). // An omitted point now caches nothing and retries on the next warm. if (Number.isFinite(ellipsoid)) { - cache.set(item.key, { ellipsoid, source: 'reearth' }); + const entry = { ellipsoid, source: 'reearth' }; + resolvedThisCall.set(item.key, entry); + cacheStore(item.key, entry); } } } catch { @@ -247,11 +323,13 @@ export function createTerrainHeights({ source, signal }) { item.lon, item.sourceOrthometricM, ); - cache.set(item.key, { + const entry = { ellipsoid, source: 'geoid-fallback', retryAt: Date.now() + GEOID_FALLBACK_COOLDOWN_MS, - }); + }; + resolvedThisCall.set(item.key, entry); + cacheStore(item.key, entry); } } } @@ -260,7 +338,7 @@ export function createTerrainHeights({ source, signal }) { // the upstream omitted has NO entry (round 6 — deliberately uncached so it // retries later): report it unresolved instead of throwing. return work.map((item) => { - const entry = cache.get(item.key); + const entry = cache.get(item.key) || resolvedThisCall.get(item.key); return entry ? { ellipsoid: entry.ellipsoid, source: entry.source } : { ellipsoid: null, source: 'unresolved' }; diff --git a/src/services/terrainHeights.test.mjs b/src/services/terrainHeights.test.mjs new file mode 100644 index 000000000..c9884e89c --- /dev/null +++ b/src/services/terrainHeights.test.mjs @@ -0,0 +1,130 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + createTerrainHeights, + DEFAULT_MAX_CACHE_ENTRIES, +} from './terrainHeights.js'; + +/** + * A terrain source that always resolves, so these tests exercise the cache and + * never the geoid-fallback path. Heights are derived from the coordinate so an + * assertion can name the value it expects. + */ +function stubSource() { + const batches = []; + return { + batches, + async getHeights(chunk) { + batches.push(chunk.map((item) => item.key)); + return chunk.map((item) => ({ ellipsoid: heightAt(item.lon) })); + }, + }; +} + +/** Distinct 5-decimal coordinates along the equator: 0.001, 0.002, … */ +const lonAt = (i) => i / 1000; +const heightAt = (lon) => 1000 + Math.round(lon * 1000); +const pointAt = (i) => ({ lat: 0, lon: lonAt(i) }); + +test('the default cache bound is exported so callers can reason about it', () => { + assert.equal(DEFAULT_MAX_CACHE_ENTRIES, 20_000); +}); + +test('the in-memory cache stops growing once it reaches its entry bound', async () => { + const terrain = createTerrainHeights({ + source: stubSource(), + maxCacheEntries: 4, + }); + + // Six distinct coordinates through a cache that holds four. + for (let i = 0; i < 6; i += 1) { + await terrain.resolveEllipsoidalGround([pointAt(i)]); + } + + // The two coldest are gone. Before the bound existed the cache kept every + // coordinate a session ever resolved, so both of these still answered. + assert.equal(terrain.cachedEllipsoidalGround(0, lonAt(0)), null); + assert.equal(terrain.cachedEllipsoidalGround(0, lonAt(1)), null); + + // The four most recent are untouched — a bound, not a flush. + for (let i = 2; i < 6; i += 1) { + assert.equal( + terrain.cachedEllipsoidalGround(0, lonAt(i)), + heightAt(lonAt(i)), + `coordinate ${i} stayed warm`, + ); + } +}); + +test('a coordinate a consumer keeps reading outlives an equally old one it ignores', async () => { + const terrain = createTerrainHeights({ + source: stubSource(), + maxCacheEntries: 4, + }); + for (let i = 0; i < 4; i += 1) { + await terrain.resolveEllipsoidalGround([pointAt(i)]); + } + + // Read coordinate 0 and ignore coordinate 1. They are otherwise the same age, + // so insertion order alone could not tell them apart. + assert.equal( + terrain.cachedEllipsoidalGround(0, lonAt(0)), + heightAt(lonAt(0)), + ); + + // Two fresh coordinates push the cache two entries past its bound. + for (let i = 4; i < 6; i += 1) { + await terrain.resolveEllipsoidalGround([pointAt(i)]); + } + + assert.equal( + terrain.cachedEllipsoidalGround(0, lonAt(0)), + heightAt(lonAt(0)), + 'the cell being read survived eviction', + ); + assert.equal( + terrain.cachedEllipsoidalGround(0, lonAt(1)), + null, + 'the same-age cell nobody read was evicted instead', + ); +}); + +test('a batch larger than the cache bound still reports every point it resolved', async () => { + const terrain = createTerrainHeights({ + source: stubSource(), + maxCacheEntries: 4, + }); + const batch = Array.from({ length: 10 }, (_, i) => pointAt(i)); + + const results = await terrain.resolveEllipsoidalGround(batch); + + // Eviction inside the call must not turn a point this batch resolved into an + // unresolved one: the results are assembled from what the call produced, not + // from whatever survived in the cache by the time it finished. + assert.equal(results.length, batch.length); + results.forEach((result, i) => { + assert.equal(result.source, 'reearth', `point ${i} reported resolved`); + assert.equal(result.ellipsoid, heightAt(lonAt(i))); + }); + + // The cache still honours the bound afterwards. + const warm = batch.filter( + (point) => terrain.cachedEllipsoidalGround(point.lat, point.lon) !== null, + ); + assert.equal(warm.length, 4); +}); + +test('an out-of-range cache bound falls back to the default instead of disabling the cache', async () => { + for (const maxCacheEntries of [0, -1, Number.NaN, undefined]) { + const terrain = createTerrainHeights({ + source: stubSource(), + maxCacheEntries, + }); + await terrain.resolveEllipsoidalGround([pointAt(1)]); + assert.equal( + terrain.cachedEllipsoidalGround(0, lonAt(1)), + heightAt(lonAt(1)), + `maxCacheEntries=${String(maxCacheEntries)} kept a working cache`, + ); + } +}); From 5f27f6edf4cc7f3ce2b2b593e1890c700da1d40f Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 17 Sep 2026 10:20:26 +0200 Subject: [PATCH 02/22] =?UTF-8?q?fix(cctv):=20surface=20heading=20confiden?= =?UTF-8?q?ce=20=E2=80=94=20estimated=20bearings=20are=20now=20visible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every pack sets headingConfidence on each camera, but nothing consumed it: a bearing synthesized by fallbackHeadingFromId (a hash of the id string, no geographic input) rendered exactly like a surveyed one — 2,827 of 3,997 cameras in a default catalog. Fixes #639. Two consumers, plus a passthrough: - HUD: the heading token is tagged when the bearing is synthetic — `HDG 194° (ESTIMATED)` — so a hash never reads as a measured facing. - Wireframe: a low-confidence camera draws its coverage frustum with dashed polylines instead of solid ones. Colors, widths, and the active/idle emphasis are unchanged; the dash is the only signal. - Public camera state gains `headingConfidence` (the raw pack flag) and `headingEstimated` (the derived bit) so UI consumers never re-derive it. A human-vouched pose always wins over the pack flag: a manually saved calibration (calSource:'manual') or a curated catalog entry (poseSource:'curated') is never presented as estimated, mirroring the CAL badge states in calibration.js. - src/layers/cctv/headingConfidence.js: pure helpers (isHeadingEstimated, headingHudToken) - src/layers/cctv/headingConfidence.test.mjs: confidence normalization, calibration/curation overrides, HUD token formatting - src/layers/cctv/presentation.js: HUD token + public-state passthrough - src/layers/cctv/rendering.js: dashed materials for estimated bearings --- src/layers/cctv/headingConfidence.js | 47 +++++++++++++++++ src/layers/cctv/headingConfidence.test.mjs | 60 ++++++++++++++++++++++ src/layers/cctv/presentation.js | 10 +++- src/layers/cctv/rendering.js | 42 ++++++++++----- 4 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 src/layers/cctv/headingConfidence.js create mode 100644 src/layers/cctv/headingConfidence.test.mjs diff --git a/src/layers/cctv/headingConfidence.js b/src/layers/cctv/headingConfidence.js new file mode 100644 index 000000000..a42297e28 --- /dev/null +++ b/src/layers/cctv/headingConfidence.js @@ -0,0 +1,47 @@ +/** + * @module headingConfidence + * + * Consumers for the catalog's per-camera `headingConfidence` flag (#639). + * + * Packs whose feed publishes no facing get a synthetic bearing from + * `fallbackHeadingFromId` — a hash of the id string, with no geographic + * input — and are marked `headingConfidence: 'low'`. Until now nothing read + * the flag, so a guessed bearing rendered exactly like a surveyed one. These + * helpers make the guess visible: the HUD heading token gains an + * `(ESTIMATED)` tag and the coverage wireframe draws dashed instead of solid. + * + * A human-vouched pose always wins over the pack flag: a manually saved + * calibration (`calSource:'manual'`) or a hand-authored catalog entry + * (`poseSource:'curated'`) is never presented as estimated, mirroring the + * CAL badge states in calibration.js. + */ + +/** + * Whether a camera's bearing is a synthetic guess that should present as + * provisional. + * @param {{headingConfidence?: string|null, calSource?: string|null, + * poseSource?: string|null}} camera - Catalog camera (or public state). + * @returns {boolean} + */ +export function isHeadingEstimated(camera) { + if (!camera) return false; + if (camera.calSource === 'manual') return false; + if (camera.poseSource === 'curated') return false; + return ( + String(camera.headingConfidence || '') + .trim() + .toLowerCase() === 'low' + ); +} + +/** + * HUD heading token: `HDG 194°`, tagged `(ESTIMATED)` when the bearing is + * synthetic — so a hash can never read as a surveyed facing. + * @param {{headingDeg?: number, headingConfidence?: string|null, + * calSource?: string|null, poseSource?: string|null}} camera + * @returns {string} + */ +export function headingHudToken(camera) { + const hdg = Math.round(Number(camera?.headingDeg) || 0); + return `HDG ${hdg}°${isHeadingEstimated(camera) ? ' (ESTIMATED)' : ''}`; +} diff --git a/src/layers/cctv/headingConfidence.test.mjs b/src/layers/cctv/headingConfidence.test.mjs new file mode 100644 index 000000000..513bf0d1b --- /dev/null +++ b/src/layers/cctv/headingConfidence.test.mjs @@ -0,0 +1,60 @@ +// src/layers/cctv/headingConfidence.test.mjs +// Bearing-provenance consumers (#639): a pack's headingConfidence:'low' +// (a hashed synthetic bearing) must present as estimated — unless a human +// vouched for the pose via manual calibration or a curated catalog entry. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { isHeadingEstimated, headingHudToken } from './headingConfidence.js'; + +test('low confidence marks the bearing estimated', () => { + assert.equal(isHeadingEstimated({ headingConfidence: 'low' }), true); +}); + +test('confidence matching is trimmed and case-insensitive', () => { + assert.equal(isHeadingEstimated({ headingConfidence: ' LOW ' }), true); + assert.equal(isHeadingEstimated({ headingConfidence: 'Low' }), true); +}); + +test('medium, high, missing, and absent cameras are not estimated', () => { + assert.equal(isHeadingEstimated({ headingConfidence: 'medium' }), false); + assert.equal(isHeadingEstimated({ headingConfidence: 'high' }), false); + assert.equal(isHeadingEstimated({}), false); + assert.equal(isHeadingEstimated(null), false); + assert.equal(isHeadingEstimated(undefined), false); +}); + +test('a manually saved calibration overrides a low pack flag', () => { + assert.equal( + isHeadingEstimated({ headingConfidence: 'low', calSource: 'manual' }), + false, + ); +}); + +test('a curated catalog entry overrides a low pack flag', () => { + assert.equal( + isHeadingEstimated({ headingConfidence: 'low', poseSource: 'curated' }), + false, + ); +}); + +test('HUD token tags a synthetic bearing and rounds the degrees', () => { + assert.equal( + headingHudToken({ headingDeg: 194.4, headingConfidence: 'low' }), + 'HDG 194° (ESTIMATED)', + ); +}); + +test('HUD token stays untagged for surveyed and calibrated bearings', () => { + assert.equal( + headingHudToken({ headingDeg: 67.5, headingConfidence: 'high' }), + 'HDG 68°', + ); + assert.equal( + headingHudToken({ + headingDeg: 225, + headingConfidence: 'low', + calSource: 'manual', + }), + 'HDG 225°', + ); +}); diff --git a/src/layers/cctv/presentation.js b/src/layers/cctv/presentation.js index 742869877..abda57d02 100644 --- a/src/layers/cctv/presentation.js +++ b/src/layers/cctv/presentation.js @@ -1,5 +1,6 @@ import { CCTV_AMBIENT_CARD_MAX } from '../../data/cctvLod.js'; import { ACTIVE_FRAME_REFRESH_MS, IDLE_FRAME_REFRESH_MS } from './policy.js'; +import { headingHudToken, isHeadingEstimated } from './headingConfidence.js'; export function createPresentation({ state: layerState, @@ -35,7 +36,9 @@ export function createPresentation({ return [ `${active.camera.city.toUpperCase()} CCTV`, `${active.camera.name.toUpperCase()}`, - `HDG ${Math.round(active.camera.headingDeg)}°`, + // A synthetic bearing (headingConfidence 'low', no human calibration) + // is tagged so a hashed guess never reads as a surveyed facing (#639). + headingHudToken(active.camera), `FOV ${Math.round(active.camera.fovDeg)}°`, `COVERAGE ${area.toFixed(2)}km²`, overlapCount > 0 ? `OVERLAP ${overlapCount} cams` : 'ISOLATED VIEW', @@ -76,6 +79,11 @@ export function createPresentation({ lat: camera.lat, lon: camera.lon, headingDeg: camera.headingDeg, + // Bearing provenance (#639): the pack's confidence flag plus the derived + // "is this a synthetic guess" bit (calibration-aware), so UI consumers + // never have to re-derive it. + headingConfidence: camera.headingConfidence || null, + headingEstimated: isHeadingEstimated(camera), pitchDeg: camera.pitchDeg, fovDeg: camera.fovDeg, rangeM: camera.rangeM, diff --git a/src/layers/cctv/rendering.js b/src/layers/cctv/rendering.js index 8c4dbb9be..ad8de36b4 100644 --- a/src/layers/cctv/rendering.js +++ b/src/layers/cctv/rendering.js @@ -1,5 +1,6 @@ import * as Cesium from 'cesium'; import { horizonOccluder } from '../../data/iconOrientation.js'; +import { isHeadingEstimated } from './headingConfidence.js'; import { ACTIVE_CAMERA_COLOR, IDLE_CAMERA_COLOR, @@ -228,6 +229,15 @@ export function createRendering({ ); const inVisibleSet = coverageVisible.has(record.camera.id); + // A synthetic bearing (headingConfidence 'low', no human calibration) + // draws its wireframe dashed so a guessed facing is visibly provisional + // rather than rendering identically to a surveyed one (#639). Colors, + // widths, and the active/idle emphasis are unchanged. + const bearingEstimated = isHeadingEstimated(record.camera); + const lineMaterial = (color) => + bearingEstimated + ? new Cesium.PolylineDashMaterialProperty({ color }) + : color; for (const entity of record.coverageEntities || []) { // The frustum wireframe is part of the projection representation — // force it on for the active camera and let it read through geometry @@ -242,13 +252,15 @@ export function createRendering({ // active camera keeps its width/alpha emphasis in both schemes. const hue = viewshedOn ? record.viewshedColors : null; if (entity._coverageRole === 'cap') { - entity.polyline.material = hue - ? isActive - ? hue.lineActive - : hue.line - : isActive - ? ACTIVE_COVERAGE_CENTER - : IDLE_COVERAGE_CENTER_MUTED; + entity.polyline.material = lineMaterial( + hue + ? isActive + ? hue.lineActive + : hue.line + : isActive + ? ACTIVE_COVERAGE_CENTER + : IDLE_COVERAGE_CENTER_MUTED, + ); entity.polyline.width = isActive ? 2.2 : 1.0; entity.polyline.depthFailMaterial = planeShowing ? hue @@ -256,13 +268,15 @@ export function createRendering({ : ACTIVE_COVERAGE_CENTER_DEPTHFAIL : undefined; } else { - entity.polyline.material = hue - ? isActive - ? hue.lineActive - : hue.line.withAlpha(0.6) - : isActive - ? ACTIVE_COVERAGE_EDGE - : IDLE_COVERAGE_EDGE_MUTED; + entity.polyline.material = lineMaterial( + hue + ? isActive + ? hue.lineActive + : hue.line.withAlpha(0.6) + : isActive + ? ACTIVE_COVERAGE_EDGE + : IDLE_COVERAGE_EDGE_MUTED, + ); entity.polyline.width = isActive ? 1.8 : 0.9; entity.polyline.depthFailMaterial = planeShowing ? hue From 899ca4f56e813ffed4820561bdadb04b0bc87e85 Mon Sep 17 00:00:00 2001 From: Ethan Stoner Date: Sun, 20 Sep 2026 19:56:37 -0700 Subject: [PATCH 03/22] fix: bound CCTV media bodies with an idle deadline The media relay only deadlined the wait for upstream response headers. Once headers arrived the timer stopped, so an upstream that answered and then sent nothing held both the client response and the upstream socket open for as long as the camera host allowed. The declared Content-Length ceiling was the only body bound, and a chunked or length-less body had none. A 30-second idle deadline now measures the gap between upstream chunks and tears the stream down through the existing release path. Because a paused pipe also stops upstream bytes arriving, the deadline reschedules itself whenever the response is still waiting to drain, so a viewer on a slow link is not read as a dead camera. Live feeds that keep delivering frames are unaffected. --- CHANGELOG.md | 10 +++ docs/CURRENT-STATE.md | 10 ++- server/providers/cctv/constants.js | 5 ++ server/providers/cctv/media.js | 46 +++++++++++- src/data/cctvMediaRange.test.mjs | 117 ++++++++++++++++++++++++++++- 5 files changed, 182 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744f4be0a..92946d3a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ while ordinary wheel, line-mode and touch-pinch inputs retain their existing behavior; the listener is removed with the application scene. +- Release CCTV media streams whose upstream falls silent after answering. The + 15-second media deadline covered only the wait for response headers, so a + camera that replied and then stopped sending held both the proxy connection + and its upstream socket open for as long as the camera host allowed; the + declared `Content-Length` ceiling was the only body bound, and a chunked or + length-less body had none. A 30-second idle deadline now bounds the gap + between upstream chunks and releases a body that has gone silent. It is + rescheduled while the response is still waiting to drain, so a viewer on a + slow link is not mistaken for a dead camera; live feeds are unaffected. + - Report AIS speed and course that carry the standard "not available" code as unknown instead of 102.3 knots and 360 degrees. Genuine readings, including a stopped vessel's zero and the highest encodable values, are unchanged. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9c381ac0c..b22d17339 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -587,10 +587,12 @@ their `configured: false` response. CCTV media waits at most 15 seconds for upstream response headers and returns 504 on timeout. Its timer stops when headers arrive, so live bodies can continue -streaming; body idle deadlines are separate from this header deadline. Error -responses are cancelled. Buffered snapshots have a 16 MiB streaming cap; an -oversized image remains an upstream miss and uses the normal fallback chain. -The existing declared media size ceiling remains 64 MiB. +streaming; a separate 30-second idle deadline then bounds the gap between +upstream chunks and releases a body that has gone silent. A response that is +still waiting to drain reschedules that deadline, so a slow client does not read +as a stalled upstream. Error responses are cancelled. Buffered snapshots have a +16 MiB streaming cap; an oversized image remains an upstream miss and uses the +normal fallback chain. The existing declared media size ceiling remains 64 MiB. ## GBFS upstream bounds diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index d66806e45..d6382ad76 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -253,5 +253,10 @@ export const CCTV_FRAME_MAX_BODY_BYTES = 16 * 1024 * 1024; /** Deadline for upstream response headers; live bodies keep streaming afterward. */ export const CCTV_MEDIA_FETCH_TIMEOUT_MS = 15 * 1000; +/** Silence a live body may carry before the relay gives up on it. Twice the + * header deadline, because a camera that is merely slow between frames is far + * more common than one that has died mid-stream, and a viewer would rather + * wait than be dropped. */ +export const CCTV_MEDIA_IDLE_TIMEOUT_MS = 30 * 1000; /** Declared size ceiling for fixed media responses. */ export const CCTV_MEDIA_MAX_BODY_BYTES = 64 * 1024 * 1024; diff --git a/server/providers/cctv/media.js b/server/providers/cctv/media.js index d652f7150..549e6cf06 100644 --- a/server/providers/cctv/media.js +++ b/server/providers/cctv/media.js @@ -4,6 +4,7 @@ import { CCTV_FRAME_FETCH_TIMEOUT_MS, CCTV_FRAME_MAX_BODY_BYTES, CCTV_MEDIA_FETCH_TIMEOUT_MS, + CCTV_MEDIA_IDLE_TIMEOUT_MS, CCTV_MEDIA_MAX_BODY_BYTES, NSW_IMAGE_ORIGIN, NSW_IMAGE_USER_AGENT, @@ -103,11 +104,17 @@ export function toReadable(body) { * @param {Response} upstream - fetch() Response object. * @param {object} [opts] * @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header. + * @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence + * allowed between upstream chunks before the stream is released. Injectable + * only to keep the deadline unit-testable. */ export async function proxyMediaResponse( res, upstream, - { sourceHeader = 'upstream' } = {}, + { + sourceHeader = 'upstream', + idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS, + } = {}, ) { const contentType = upstream.headers.get('content-type') || 'application/octet-stream'; @@ -153,7 +160,20 @@ export async function proxyMediaResponse( return; } + // The header deadline only covers the wait for a response line. Past that an + // upstream can hold the connection open and send nothing at all, and the + // relay would wait on it for as long as the camera host cared to. The + // deadline below measures the gap between upstream chunks rather than the + // life of the stream, so a feed that keeps delivering keeps its connection. + let idleTimer = null; + const clearIdleDeadline = () => { + if (!idleTimer) return; + clearTimeout(idleTimer); + idleTimer = null; + }; + stream.on('error', () => { + clearIdleDeadline(); if (!res.writableEnded) res.end(); }); @@ -164,6 +184,7 @@ export async function proxyMediaResponse( const releaseUpstream = () => { if (released) return; released = true; + clearIdleDeadline(); stream.unpipe(res); // Destroying the Node stream cancels the web body it wraps; the direct // cancel covers a body that was never wrapped, and rejects harmlessly when @@ -176,14 +197,37 @@ export async function proxyMediaResponse( /* already closed */ } }; + const armIdleDeadline = () => { + clearIdleDeadline(); + idleTimer = setTimeout(onIdleDeadline, idleTimeoutMs); + idleTimer.unref?.(); + }; + const onIdleDeadline = () => { + // A viewer who cannot keep up pauses the pipe, and no upstream bytes arrive + // while it is paused. That is a slow client rather than a dead camera, so + // it gets the deadline again instead of a teardown. + if (res.writableNeedDrain) { + armIdleDeadline(); + return; + } + releaseUpstream(); + if (!res.writableEnded) res.end(); + }; res.once('close', () => { + clearIdleDeadline(); if (!res.writableEnded) releaseUpstream(); }); res.once('error', releaseUpstream); stream.once('end', () => { + clearIdleDeadline(); released = true; }); + armIdleDeadline(); stream.pipe(res); + // Attached after the pipe because a data listener resumes the stream, and + // flowing before the destination is attached would spill chunks nobody + // forwards. Only bytes from upstream renew the deadline. + stream.on('data', armIdleDeadline); } /** diff --git a/src/data/cctvMediaRange.test.mjs b/src/data/cctvMediaRange.test.mjs index 3e71d3e17..9f279bdc7 100644 --- a/src/data/cctvMediaRange.test.mjs +++ b/src/data/cctvMediaRange.test.mjs @@ -8,8 +8,9 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; -import { Readable } from 'node:stream'; +import { Readable, Writable } from 'node:stream'; import { cctvProxy } from '../../server/providers/cctv.js'; +import { proxyMediaResponse } from '../../server/providers/cctv/media.js'; import { sanitizeCctvRangeHeader } from '../../server/providers/cctv/range.js'; import { CCTV_MEDIA_MAX_BODY_BYTES as CAP } from '../../server/providers/cctv/constants.js'; @@ -586,3 +587,117 @@ test('two sequential seeks are served as two partial responses', async (t) => { assert.equal(first.body, 'chunk-0'); assert.equal(second.body, 'chunk-100'); }); + +// --------------------------------------------------------------------------- +// The body idle deadline. proxyMediaResponse is driven directly here because +// the deadline is only injectable through its options; the route always passes +// the production value. +// --------------------------------------------------------------------------- + +/** + * A response to pipe into, built on a real Writable so the backpressure the + * proxy reads is the stream's own rather than a flag the test sets. With + * `drains: false` the bytes are accepted and never flushed, which is what a + * viewer on a slow link looks like from the server side. + */ +function pipeTarget({ drains = true } = {}) { + const res = new Writable({ + // One byte is enough to leave a target that never empties its buffer + // permanently in need of a drain. + highWaterMark: drains ? undefined : 1, + write(_chunk, _encoding, callback) { + if (drains) callback(); + }, + }); + res.writeHead = (status, headers) => { + res.statusCode = status; + res.headers = headers || {}; + }; + return res; +} + +/** + * An upstream that answers with headers, sends one chunk, and then either + * keeps producing every `everyMs` or goes silent. `released` reports the body's + * own cleanup: a cancelled fetch body lands in the web stream's cancel hook. + */ +function mediaUpstream({ everyMs = 0 } = {}) { + const state = { produced: 0, released: false }; + let ticker = null; + const body = new ReadableStream({ + start(controller) { + const send = () => { + state.produced += 1; + controller.enqueue(new Uint8Array([0xff, 0xd8, 0xff, 0xe0])); + }; + send(); + if (everyMs > 0) { + ticker = setInterval(send, everyMs); + ticker.unref(); + } + }, + cancel() { + state.released = true; + if (ticker) clearInterval(ticker); + }, + }); + state.upstream = { + ok: true, + status: 200, + headers: { + get: (name) => + String(name).toLowerCase() === 'content-type' ? 'video/mp4' : null, + }, + body, + arrayBuffer: async () => Buffer.alloc(0), + }; + return state; +} + +test('an upstream that goes silent after its headers is released at the idle deadline', async (t) => { + const feed = mediaUpstream(); + const res = pipeTarget(); + t.after(() => res.destroy()); + await proxyMediaResponse(res, feed.upstream, { idleTimeoutMs: 40 }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(feed.released, false, 'released before the deadline was due'); + await new Promise((resolve) => setTimeout(resolve, 120)); + // Without the deadline this stream is held open against the camera host for + // as long as that host will keep the socket, with no bytes ever arriving. + assert.equal(feed.released, true, 'the silent upstream was never released'); + assert.equal(res.writableEnded, true, 'the client was left waiting'); +}); + +test('a live feed that keeps producing is not cut off by the idle deadline', async (t) => { + const feed = mediaUpstream({ everyMs: 10 }); + const res = pipeTarget(); + t.after(() => res.destroy()); + await proxyMediaResponse(res, feed.upstream, { idleTimeoutMs: 40 }); + + await new Promise((resolve) => setTimeout(resolve, 250)); + assert.equal(feed.released, false, 'a healthy feed was torn down'); + assert.equal(res.writableEnded, false, 'a healthy feed was ended early'); + assert.ok( + feed.produced > 5, + `the feed stopped producing (${feed.produced} chunks)`, + ); +}); + +test('a client that cannot keep up is not mistaken for a stalled upstream', async (t) => { + // Nothing arrives from upstream while the pipe is paused, which looks exactly + // like silence unless the response is asked whether it is still draining. + const feed = mediaUpstream({ everyMs: 10 }); + const res = pipeTarget({ drains: false }); + t.after(() => res.destroy()); + await proxyMediaResponse(res, feed.upstream, { idleTimeoutMs: 30 }); + + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + res.writableNeedDrain, + true, + 'the client was not the bottleneck', + ); + assert.equal(feed.released, false, 'a slow viewer tore down a healthy feed'); + assert.equal(res.writableEnded, false, 'a slow viewer ended the response'); +}); From c24399e99d03d3d472490ac29f6b11db59e8d1f7 Mon Sep 17 00:00:00 2001 From: daikaginza Date: Sat, 19 Sep 2026 12:11:55 +0700 Subject: [PATCH 04/22] fix(traffic): say which upstream declined a road load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Street Traffic draws on OpenStreetMap for geometry and TomTom for flow, and the row said "Road data temporarily unavailable" whichever one failed. The reporter of #661 went and verified a TomTom key while every public Overpass mirror was refusing their network with HTTP 406 — a status fetchRoads already had and dropped on the floor. Classify the refusal where the layer talks to Overpass: 429 is "Overpass rate-limited", 504 is "Overpass timed out", anything else is "Overpass refused the road query (HTTP )". The first two are the words alpr's Overpass source already uses; the status is what says which upstream declined, which a single-source layer never has to. The error carries its status as a RoadRequestError, following LiveSourceError in src/sources/live/contract.js, so a caller can branch on the code without parsing English. The catch promotes a classified refusal and keeps the general line for everything else — the branch is on the error type, not on its text, because a dropped socket throws whatever the platform felt like saying and "Failed to fetch" on a panel row helps nobody. Retries, backoff, caching and the mirror rotation are untouched. --- CHANGELOG.md | 7 ++++ docs/CURRENT-STATE.md | 1 + src/layers/traffic/ingestion.js | 15 +++++-- src/layers/traffic/navigation.test.mjs | 36 +++++++++++++++++ src/layers/traffic/source.js | 55 ++++++++++++++++++++++++++ src/layers/traffic/source.test.mjs | 39 +++++++++++++++++- 6 files changed, 149 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 744f4be0a..33174d45b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +- Report which upstream declined a Street Traffic road load. The layer row now + reads `Overpass rate-limited`, `Overpass timed out`, or + `Overpass refused the road query (HTTP 406)` instead of a general + "Road data temporarily unavailable", so a reader is not sent to check a + TomTom key when the public OpenStreetMap mirrors are the side that failed. + Failures the layer cannot classify keep the general line. + - Enable responsive trackpad pinch zoom on the globe. Browser pixel-mode `Ctrl+wheel` pinch gestures now reach Cesium with bounded amplification, while ordinary wheel, line-mode and touch-pinch inputs retain their existing diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9c381ac0c..e63817fd8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3242,6 +3242,7 @@ silently demoting every later lookup for the session. - HTTP refusals such as 406 now rotate alongside the existing network, rate-limit, and runtime-error cases. A refusal from one mirror no longer prevents reaching healthy alternatives or persists under the seven-day road/month-long boundary cache TTLs. Concurrent identical queries share one mirror sequence; if it fails, both the initiating and joined callers can use the same last-good data. - A refusal every mirror agrees on is still reported with the first mirror's status and body, so a genuinely malformed query says what upstream said — but only after every mirror has had the chance to answer it. `fetchOverpassPayload` takes injectable endpoints and fetch so the rotation is tested without a live mirror (`src/overpassProxy.test.mjs`). - Every mirror is asked with a User-Agent that names the application, its version and the project address (`OVERPASS_USER_AGENT` in `server/providers/overpass/constants.js`), which is what the OSM API usage policy asks for. Mirrors may refuse a client they cannot identify; such a refusal is never treated as data and costs the fan-out that mirror, so the header is what keeps the list at full strength. +- When the fan-out has nothing left to offer, the traffic row says which upstream declined and how: `Overpass rate-limited`, `Overpass timed out`, or `Overpass refused the road query (HTTP 406)`. Street Traffic draws on two upstreams — OpenStreetMap for geometry, TomTom for flow — so a row that only said "road data unavailable" sent readers to check a TomTom key that was never the problem (#661). `roadRequestError` in `src/layers/traffic/source.js` owns the mapping and tags the error with its status. A refusal whose code is unreadable — the layer takes its source by injection, so an adapter may not supply one — reports `Overpass temporarily unavailable` rather than printing `HTTP undefined`, and anything the layer did not classify at all (a dropped connection, a malformed snapshot) still reads `Road data temporarily unavailable`, because a platform exception is not a sentence to put on a panel. ### Share-link v2 layer state (August 2026) diff --git a/src/layers/traffic/ingestion.js b/src/layers/traffic/ingestion.js index de5b56d9f..904ceca04 100644 --- a/src/layers/traffic/ingestion.js +++ b/src/layers/traffic/ingestion.js @@ -4,6 +4,7 @@ import { TILE_CACHE_MAX_ENTRIES, FAST_FETCH_ALTITUDE, } from './policy.js'; +import { RoadRequestError, roadRequestError } from './source.js'; export function createIngestion({ state: layerState, @@ -67,7 +68,7 @@ export function createIngestion({ ); if (!response.ok) { - throw new Error(`Overpass API returned ${response.status}`); + throw roadRequestError(response.status); } if (!state) { @@ -307,8 +308,16 @@ export function createIngestion({ renderedSomething = true; } catch (e) { if (e?.name === 'AbortError') return; - if (generation === layerState._loadGeneration && !renderedSomething) - layerState._roadError = 'Road data temporarily unavailable'; + if (generation === layerState._loadGeneration && !renderedSomething) { + // A refusal we classified carries its own line; anything else — a + // dropped connection, a malformed snapshot — keeps the general one, + // because `e.message` from the platform is not a sentence anyone + // should have to read off a panel. + layerState._roadError = + e instanceof RoadRequestError + ? e.message + : 'Road data temporarily unavailable'; + } console.warn('[Data:Traffic] Fetch error:', e); } finally { if (generation === layerState._loadGeneration) { diff --git a/src/layers/traffic/navigation.test.mjs b/src/layers/traffic/navigation.test.mjs index e7bd4cc62..c07ae6ad9 100644 --- a/src/layers/traffic/navigation.test.mjs +++ b/src/layers/traffic/navigation.test.mjs @@ -216,3 +216,39 @@ test('parked failures back off and disabling cancels the scheduled retry', async assert.equal(calls, 2); assert.equal(layer.getStats().error, null); }); + +test('a declined road request reaches the row as the reason, not as a shrug', async (t) => { + // Issue #661: with every public Overpass mirror refusing a network, the row + // read "Road data temporarily unavailable" while the console already held + // "Overpass API returned 406". The reporter went looking for a bad TomTom + // key — the other half of this layer, and never the problem. The status the + // code already has must survive the trip to the panel. + let status = 406; + const { layer, viewer, tick } = setup(t, async () => ({ ok: false, status })); + layer.enable(viewer); + await tick(400); + assert.equal( + layer.getStats().error, + 'Overpass refused the road query (HTTP 406)', + ); + + status = 429; + for (let i = 0; i < 20; i++) await tick(1500); + assert.equal( + layer.getStats().error, + 'Overpass rate-limited', + 'a rate limit is a different instruction to the reader than a refusal', + ); +}); + +test('an unclassified road failure keeps the general line', async (t) => { + // The complement of the test above, and the reason this is not just + // `e.message`: a dropped socket throws whatever the platform felt like + // saying, and "Failed to fetch" on a panel row helps nobody. + const { layer, viewer, tick } = setup(t, async () => { + throw new Error('socket hang up'); + }); + layer.enable(viewer); + await tick(400); + assert.equal(layer.getStats().error, 'Road data temporarily unavailable'); +}); diff --git a/src/layers/traffic/source.js b/src/layers/traffic/source.js index d499a4b7b..dbcbb67c1 100644 --- a/src/layers/traffic/source.js +++ b/src/layers/traffic/source.js @@ -15,6 +15,61 @@ function buildOverpassQuery( return `[out:json][timeout:${timeoutSec}];(way["highway"~"${regex}"](${south},${west},${north},${east}););out geom qt;`; } +/** + * A road request upstream declined, carrying a line fit to show an operator. + * + * The status travels beside the message instead of only inside it, so a caller + * can branch on the code without parsing English — the same shape + * `LiveSourceError` uses in `src/sources/live/contract.js`. + */ +export class RoadRequestError extends Error { + constructor(message, { status = null } = {}) { + super(message); + this.name = 'RoadRequestError'; + this.status = status; + } +} + +/** + * Name an Overpass refusal in the words the traffic row will print. + * + * This layer has two upstreams — OpenStreetMap for road geometry and TomTom + * for flow — and only one of them can be down at a time. A row that says + * nothing more than "road data unavailable" sends the reader to check their + * TomTom key, which is the wrong half of the layer and costs them the + * afternoon. So the status upstream actually returned is reported: 406 from a + * public mirror is not a configuration problem the reader can fix, and saying + * so is the difference between a dead end and a next step. + * + * The first two lines match `alpr`'s Overpass source word for word; this is + * the same vocabulary, not a new one. What the status adds is *which* upstream + * and *how* it declined — the part a single-source layer never needs to say. + * + * Only what the browser can see is claimed. The proxy rotates mirrors behind + * `/api/overpass`, so a refusal arriving here means the proxy had nothing + * better to offer; how many mirrors it tried is not something this side knows, + * and the message does not pretend otherwise. + * @param {number} status - HTTP status the proxy returned; a code that is + * not a finite number is treated as absent rather than printed. + * @returns {RoadRequestError} + */ +export function roadRequestError(status) { + const code = Number.isFinite(status) ? status : null; + const message = + code === 429 + ? 'Overpass rate-limited' + : code === 504 + ? 'Overpass timed out' + : // A source is injected, so a caller's adapter may hand back a + // refusal with no status on it. "HTTP undefined" on a panel row is + // worse than not naming a number, so an unreadable code falls back + // to what `alpr` says when it cannot be more specific either. + code === null + ? 'Overpass temporarily unavailable' + : `Overpass refused the road query (HTTP ${code})`; + return new RoadRequestError(message, { status: code }); +} + /** Supply road responses, flow availability and one decoded flow cache. */ export function createTrafficSource({ fetchImpl = (...args) => globalThis.fetch(...args), diff --git a/src/layers/traffic/source.test.mjs b/src/layers/traffic/source.test.mjs index 8047ddbbd..f7024d81c 100644 --- a/src/layers/traffic/source.test.mjs +++ b/src/layers/traffic/source.test.mjs @@ -1,7 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { readFileSync } from 'node:fs'; -import { createTrafficSource } from './source.js'; +import { + RoadRequestError, + createTrafficSource, + roadRequestError, +} from './source.js'; import { clampBoundsAroundCenter } from '../../data/trafficBounds.js'; const bounds = { south: 30.267, west: -97.744, north: 30.268, east: -97.743 }; const fixture = readFileSync( @@ -188,3 +192,36 @@ test('road sources decode direction and coordinates before scene construction', ], }); }); + +test('an Overpass refusal is named by status, and keeps the status beside the words', () => { + // The panel prints this string verbatim, so the mapping is part of the + // layer's contract with the reader, not an implementation detail. + assert.equal(roadRequestError(429).message, 'Overpass rate-limited'); + assert.equal(roadRequestError(504).message, 'Overpass timed out'); + assert.equal( + roadRequestError(406).message, + 'Overpass refused the road query (HTTP 406)', + ); + assert.equal( + roadRequestError(500).message, + 'Overpass refused the road query (HTTP 500)', + ); + + // An injected source may refuse without a readable code; a row must never + // print "HTTP undefined" at a reader. + for (const missing of [undefined, null, NaN, 'four-oh-six']) { + const vague = roadRequestError(missing); + assert.equal(vague.message, 'Overpass temporarily unavailable'); + assert.equal(vague.status, null, 'an unreadable code is absent, not zero'); + } + + const refused = roadRequestError(406); + assert.ok(refused instanceof RoadRequestError); + assert.ok(refused instanceof Error); + assert.equal(refused.name, 'RoadRequestError'); + assert.equal( + refused.status, + 406, + 'a caller must be able to branch on the code without parsing English', + ); +}); From 2a47bdb8e74e34ae633e5502a88461063f069851 Mon Sep 17 00:00:00 2001 From: Arthur Bogaart <526197+abogaart@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:28:23 +0200 Subject: [PATCH 05/22] fix: spawn Apple /bin/chmod by absolute path when hardening credentials The macOS credential hardener strips ACLs with `chmod -N`, spawning `chmod` by bare name. On machines where Nix home-manager or Homebrew coreutils sit ahead of /bin on PATH, that resolves to GNU chmod, which has no `-N` flag and exits non-zero. The hardener treated that as a failure and refused every save from the Provider Settings panel with "could not restrict the credential file to your account; nothing was saved". Spawn /bin/chmod by absolute path instead, matching how the Windows branch already resolves its native tools without consulting PATH, and add a regression test that pins the absolute path. --- server/standalone/key-setup-hardening.mjs | 6 +++++- src/keySetupHardening.test.mjs | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/standalone/key-setup-hardening.mjs b/server/standalone/key-setup-hardening.mjs index 002ccabdc..e2d11182c 100644 --- a/server/standalone/key-setup-hardening.mjs +++ b/server/standalone/key-setup-hardening.mjs @@ -123,7 +123,11 @@ export function hardenCredentialFile( if (platform !== 'win32') { try { if (platform === 'darwin') { - const aclRemoval = spawn('chmod', ['-N', filepath], { + // `-N` (strip the ACL) is Apple-only. Spawn /bin/chmod by absolute + // path, never by name: a Nix/Homebrew coreutils profile puts GNU chmod + // first on PATH, and GNU chmod rejects `-N` — which read as a + // hardening failure and refused every save on such machines. + const aclRemoval = spawn('/bin/chmod', ['-N', filepath], { stdio: 'ignore', }); if (!commandCompletedSuccessfully(aclRemoval)) return false; diff --git a/src/keySetupHardening.test.mjs b/src/keySetupHardening.test.mjs index 2578be37d..c519576e3 100644 --- a/src/keySetupHardening.test.mjs +++ b/src/keySetupHardening.test.mjs @@ -55,6 +55,21 @@ test('macOS ACL removal failure stops before chmod and fails closed', () => { assert.deepEqual(fileSystem.calls, [], 'mode bits must not disguise an ACL-removal failure'); }); +test('macOS ACL removal spawns Apple /bin/chmod by absolute path, not via PATH', () => { + const spawned = []; + hardenCredentialFile(FILE, { + platform: 'darwin', + fileSystem: fileSystemWithMode(0o100600), + spawn(command, args) { + spawned.push([command, args]); + return { status: 0, signal: null }; + }, + }); + // A coreutils-first PATH (Nix home-manager, Homebrew) resolves bare `chmod` + // to GNU chmod, which has no `-N` and would fail every save. + assert.deepEqual(spawned, [['/bin/chmod', ['-N', FILE]]]); +}); + test('POSIX hardening verifies the resulting 0600 mode', () => { const goodFs = fileSystemWithMode(0o100600); assert.equal(hardenCredentialFile(FILE, { From 34d258b39464a6bc2d323a6503bb87f98c45adef Mon Sep 17 00:00:00 2001 From: Raushankumar0720 Date: Wed, 23 Sep 2026 15:52:21 +0530 Subject: [PATCH 06/22] fix(live): return 502 and prevent caching on oversized track responses (fixes #720) --- server/providers/aircraft/tracks.js | 8 ++++-- src/tooling/liveProviders.test.mjs | 38 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/server/providers/aircraft/tracks.js b/server/providers/aircraft/tracks.js index 7d9501628..d9f2a36d5 100644 --- a/server/providers/aircraft/tracks.js +++ b/server/providers/aircraft/tracks.js @@ -45,7 +45,9 @@ export function trackBackfillProxies() { RESPONSE_CAP_BYTES, ); let body; + let status = upstream.status; if (tooLarge) { + status = 502; body = JSON.stringify({ error: 'Upstream track response too large' }); } else if (!upstream.ok) { // Sanitize upstream error surface; status code is signal enough @@ -53,8 +55,10 @@ export function trackBackfillProxies() { } else { body = text; } - cachePut(key, { at: Date.now(), status: upstream.status, body }); - res.statusCode = upstream.status; + if (!tooLarge) { + cachePut(key, { at: Date.now(), status, body }); + } + res.statusCode = status; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.setHeader('Cache-Control', 'no-store'); res.end(body); diff --git a/src/tooling/liveProviders.test.mjs b/src/tooling/liveProviders.test.mjs index 250924b42..de8681490 100644 --- a/src/tooling/liveProviders.test.mjs +++ b/src/tooling/liveProviders.test.mjs @@ -353,3 +353,41 @@ test('military cooldown bounds untrusted Retry-After and defaults server errors' assert.equal(Number(result.headers['retry-after']), seconds); } }); + +test('track backfill proxy returns 502 and skips cache when upstream response exceeds body cap', async (t) => { + const tracks = install(providers.trackBackfillProxies(), true); + let callCount = 0; + t.mock.method(globalThis, 'fetch', async () => { + callCount++; + if (callCount === 1) { + return { + ok: true, + status: 200, + headers: new Map(), + body: (async function* () { + yield Buffer.alloc(6 * 1024 * 1024, 'x'); + })(), + }; + } + return { + ok: true, + status: 200, + headers: new Map(), + body: (async function* () { + yield Buffer.from(JSON.stringify({ path: [] })); + })(), + }; + }); + + const res1 = await tracks('/api/opensky-track', '?icao24=def456'); + assert.equal(res1.statusCode, 502); + assert.deepEqual(JSON.parse(res1.body), { + error: 'Upstream track response too large', + }); + + // Verify failure was NOT cached: second request invokes upstream again and succeeds + const res2 = await tracks('/api/opensky-track', '?icao24=def456'); + assert.equal(res2.statusCode, 200); + assert.deepEqual(JSON.parse(res2.body), { path: [] }); + assert.equal(callCount, 2); +}); From b456eb8ff1a953dd9bf0f8d7825dff954d39bcdd Mon Sep 17 00:00:00 2001 From: Sean Armstrong Date: Tue, 22 Sep 2026 09:53:05 -0600 Subject: [PATCH 07/22] fix(render): keep Cesium's model atmosphere off Apple Metal iPad and iPhone show "An error occurred while rendering. Rendering has stopped." on load. Cesium's AtmosphereStageVS calls czm_computeGroundAtmosphereScattering with its `out` parameters bound directly to varyings: czm_computeGroundAtmosphereScattering( v_positionWC, lightDirection, v_atmosphereRayleighColor, v_atmosphereMieColor, v_atmosphereOpacity); ANGLE's Metal backend wraps each `out` argument in an ANGLE_out(...) helper taking `thread T &`, but vertex-output members live in `__metal_generic` and cannot bind to `thread`, so the program fails to LINK and Cesium tears down the render loop. This is the only shader in Cesium that binds varyings to `out` params -- SkyAtmosphere (czm_computeScattering) and the ground-atmosphere fragment path both route through locals, so they are unaffected and stay on. ModelSceneGraph.configurePipeline attaches the stage under a single condition, `fog.enabled && fog.renderable`, so clearing `fog.renderable` keeps the broken program from ever being built. `renderable` rather than `enabled` is deliberate: Fog.update returns early when `enabled` is false and forfeits the density that drives 3D Tiles screen-space-error scaling, and visualSettings saves/restores `fog.enabled` around IR boost, which would resurrect the broken shader mid-session. Detection is a WebGL2 link probe of the same out-param/varying pattern, so a future driver fix silently restores the effect with no code change, OR'd with iOS/iPadOS platform detection as a backstop for when no probe context can be created. iPadOS desktop mode is identified by the Macintosh UA token plus maxTouchPoints > 1, since real Macs report zero and navigator.platform is deprecated. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HoRD3yHVgfTHeikTmsUoHf --- CHANGELOG.md | 12 ++ docs/CURRENT-STATE.md | 18 +++ scripts/format-scope.json | 1 + scripts/package-boundaries.json | 3 +- src/app/atmosphereCompat.js | 177 +++++++++++++++++++++++++ src/app/atmosphereCompat.test.mjs | 209 ++++++++++++++++++++++++++++++ src/app/viewer.js | 5 + 7 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 src/app/atmosphereCompat.js create mode 100644 src/app/atmosphereCompat.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bcdebf70..8893a60e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,18 @@ Add feed provenance to analyst/view answers and HUD context while retaining existing response fields and runner ownership (Matt Van Horn, #347). Analyst records for loaded satellites, datacenters and dams, with explicit bounded count/rank coverage (Matt Van Horn, #351). +- Render on iPad and iPhone instead of stopping with "An error occurred while + rendering." Cesium's per-vertex model atmosphere binds shader `out` + parameters directly to varyings, which Apple's Metal/ANGLE backend cannot + link, so the program failed and the render loop was torn down. The stage is + now kept out of the pipeline on affected devices by clearing + `scene.fog.renderable`, which leaves `fog.enabled` — and the fog density that + drives 3D Tiles refinement — untouched. Detection is a WebGL2 link probe of + the same pattern, so a future driver fix restores the effect with no code + change, with iOS/iPadOS detection as a backstop. Sky atmosphere and the + ground-atmosphere fragment path route through locals and are unaffected. + `src/app/atmosphereCompat.test.mjs` pins the probe, the platform matrix and + the `renderable`-not-`enabled` choice. - Remove the spurious scrollbars that appeared on both panel stacks at narrow widths (720px and below) as soon as a panel was expanded. The stacks scroll diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 67acfffce..6592915f9 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -664,6 +664,24 @@ snapshots, lifecycle actions and row descriptors. Remount and teardown remove listeners and row subscriptions; obsolete completions do not repaint old rows. The clear control presents busy state while its existing action owns the transaction. + +## Model atmosphere on Apple Metal + +Cesium's per-vertex model atmosphere is kept out of the pipeline on devices +whose driver cannot link it. `AtmosphereStageVS` binds shader `out` parameters +directly to varyings, which ANGLE's Metal backend rejects at link time, tearing +down the render loop on iPadOS and iOS. `ModelSceneGraph.configurePipeline` +attaches that stage only when `fog.enabled && fog.renderable`, so the viewer +clears `scene.fog.renderable` and leaves `fog.enabled` — and the fog density +that drives 3D Tiles screen-space-error scaling — intact. Sky atmosphere and the +ground-atmosphere fragment path route through locals and are unaffected. + +Detection is a WebGL2 link probe of the same out-parameter/varying pattern, so a +future driver fix restores the effect with no code change; iOS/iPadOS platform +detection is the backstop for when no probe context can be created. Applied +during viewer construction, before any tile builds a draw command. + + ## Map Source control ownership Map Source controls own chip listeners, source-state subscriptions and selection diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 97bb0b8f2..01e429d0d 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -31,6 +31,7 @@ "src/app/mapLifecycle.test.mjs", "src/app/startupCamera.test.mjs", "src/app/viewer.js", + "src/app/atmosphereCompat.test.mjs", "src/standalone/application.js", "src/standalone/controls.js", "src/standalone/data.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 5273abbf8..097ac488e 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -20,7 +20,7 @@ }, "viewer": { "exports": ["./application/viewer"], - "modules": ["src/app/viewer.js"], + "modules": ["src/app/viewer.js", "src/app/atmosphereCompat.js"], "external": ["cesium"] }, "vite-build": { @@ -1287,6 +1287,7 @@ "src/app/surfaceServices.js", "src/app/tools.js", "src/app/viewer.js", + "src/app/atmosphereCompat.js", "src/bloom.js", "src/camera.js", "src/cameraVerbs.js", diff --git a/src/app/atmosphereCompat.js b/src/app/atmosphereCompat.js new file mode 100644 index 000000000..802f2b212 --- /dev/null +++ b/src/app/atmosphereCompat.js @@ -0,0 +1,177 @@ +/** + * Workaround for Apple's Metal/ANGLE shader translator rejecting Cesium's + * per-vertex model atmosphere. + * + * Cesium's `AtmosphereStageVS` calls a function whose `out` parameters are + * bound DIRECTLY to varyings: + * + * czm_computeGroundAtmosphereScattering( + * v_positionWC, lightDirection, + * v_atmosphereRayleighColor, v_atmosphereMieColor, v_atmosphereOpacity); + * + * ANGLE's Metal backend wraps each `out` argument in an `ANGLE_out(...)` + * helper that takes a `thread T &`. Vertex-output struct members live in + * `__metal_generic`, which cannot bind to `thread`, so the program fails to + * LINK (not compile) and Cesium tears the render loop down: + * + * RuntimeError: Program failed to link. ... MSL compilation error: + * reference to type 'thread float3' could not bind to an lvalue of type + * '__metal_generic float3' + * + * This is the only shader in Cesium that binds varyings to `out` params -- + * SkyAtmosphere and the ground-atmosphere fragment path both route through + * locals first, so they are unaffected and stay on. + * + * Cesium attaches that stage in `ModelSceneGraph.configurePipeline` under a + * single condition: + * + * const fogRenderable = frameState.fog.enabled && frameState.fog.renderable; + * if (fogRenderable) modelPipelineStages.push(AtmospherePipelineStage); + * + * so clearing `scene.fog.renderable` keeps the broken stage out of the + * pipeline entirely -- the program is never built, so it cannot fail to link. + * + * We clear `renderable` rather than `enabled` deliberately: + * - `Fog.update` still computes fog density when only `renderable` is off, + * preserving the distance-based screen-space-error scaling that governs + * 3D Tiles refinement. Clearing `enabled` returns early and forfeits it. + * - `visualSettings.js` saves and restores `scene.fog.enabled` around IR + * boost. Parking our fix on `enabled` would let that restore resurrect + * the broken shader mid-session; `renderable` is orthogonal to it. + * + * @module app/atmosphereCompat + */ + +/** Minimal vertex shader reproducing the exact out-param-to-varying bind. */ +const PROBE_VERTEX_SHADER = `#version 300 es +precision highp float; +in vec3 a_position; +out vec3 v_probe; +void writeOut(out vec3 value) { value = vec3(1.0); } +void main() { + writeOut(v_probe); + gl_Position = vec4(a_position, 1.0); +}`; + +const PROBE_FRAGMENT_SHADER = `#version 300 es +precision highp float; +in vec3 v_probe; +out vec4 fragColor; +void main() { fragColor = vec4(v_probe, 1.0); }`; + +/** + * True when the user agent is iOS or iPadOS. + * + * iPadOS 13+ defaults to desktop-mode browsing and reports `MacIntel` with no + * iPad token, so it is identified by touch points instead -- real Macs report + * `maxTouchPoints` of 0, including Apple Silicon ones that also claim + * `MacIntel`. Brave/Chrome/Firefox on iOS are all WebKit underneath and share + * the same Metal backend, so matching the platform rather than the browser is + * what we want. + */ +export function isAppleMobilePlatform(navigatorLike = globalThis.navigator) { + if (!navigatorLike) return false; + const ua = String(navigatorLike.userAgent || ''); + if (/\b(iPad|iPhone|iPod)\b/.test(ua)) return true; + // Desktop-mode iPadOS presents as a Mac. What separates it from a real Mac + // is touch: Macs report 0 touch points, including Apple Silicon ones that + // also claim `MacIntel`. `navigator.platform` is deprecated and may be + // empty, so the UA's Macintosh token is accepted as an equal signal rather + // than relying on platform alone. + const platform = String(navigatorLike.platform || ''); + const macLike = + platform === 'MacIntel' || + platform === 'MacARM' || + /\bMacintosh\b/.test(ua); + const touchPoints = Number(navigatorLike.maxTouchPoints) || 0; + return macLike && touchPoints > 1; +} + +/** + * Link the probe program and report whether the driver rejects the pattern. + * + * Returns `false` when WebGL2 is unavailable so that a missing context is + * never mistaken for a broken driver -- Cesium has its own handling for that, + * and the caller still falls back to platform detection. + */ +export function probeOutParamVaryingLinkFailure(gl) { + if (!gl) return false; + const vs = gl.createShader(gl.VERTEX_SHADER); + const fs = gl.createShader(gl.FRAGMENT_SHADER); + const program = gl.createProgram(); + if (!vs || !fs || !program) return false; + try { + gl.shaderSource(vs, PROBE_VERTEX_SHADER); + gl.compileShader(vs); + gl.shaderSource(fs, PROBE_FRAGMENT_SHADER); + gl.compileShader(fs); + // A compile failure here is not the bug we are probing for: the driver + // rejects this pattern at LINK time, after both stages compile cleanly. + if ( + !gl.getShaderParameter(vs, gl.COMPILE_STATUS) || + !gl.getShaderParameter(fs, gl.COMPILE_STATUS) + ) + return false; + gl.attachShader(program, vs); + gl.attachShader(program, fs); + gl.linkProgram(program); + return !gl.getProgramParameter(program, gl.LINK_STATUS); + } catch { + return false; + } finally { + gl.deleteShader(vs); + gl.deleteShader(fs); + gl.deleteProgram(program); + } +} + +/** Build a throwaway WebGL2 context for the probe, or null if unavailable. */ +function defaultProbeContext() { + try { + const canvas = + typeof OffscreenCanvas === 'function' + ? new OffscreenCanvas(1, 1) + : globalThis.document?.createElement('canvas'); + return canvas?.getContext('webgl2') ?? null; + } catch { + return null; + } +} + +/** + * Decide whether the per-vertex model atmosphere must be kept out of the + * pipeline on this device. + * + * The probe is authoritative when it fires, so a future driver fix silently + * restores the effect without a code change. Platform detection is the + * backstop for the case where no context can be created -- on Apple mobile a + * link failure is fatal and unrecoverable, so we would rather lose fog than + * gamble the whole render loop on the probe having run. + */ +export function shouldDisableModelAtmosphere({ + navigatorLike = globalThis.navigator, + createProbeContext = defaultProbeContext, +} = {}) { + const gl = createProbeContext(); + if (gl) { + if (probeOutParamVaryingLinkFailure(gl)) return true; + // Probe ran and the driver accepted the pattern. Apple mobile is still + // quarantined: the probe program is far simpler than Cesium's, and a + // false negative there costs the entire scene. + return isAppleMobilePlatform(navigatorLike); + } + return isAppleMobilePlatform(navigatorLike); +} + +/** + * Keep Cesium's broken model-atmosphere stage out of the pipeline. + * + * Returns true when the workaround was applied. Safe to call on any scene; + * scenes without a fog object are left untouched. + */ +export function applyModelAtmosphereWorkaround(scene, options = {}) { + if (!scene?.fog) return false; + if (!shouldDisableModelAtmosphere(options)) return false; + scene.fog.renderable = false; + return true; +} diff --git a/src/app/atmosphereCompat.test.mjs b/src/app/atmosphereCompat.test.mjs new file mode 100644 index 000000000..7f601512b --- /dev/null +++ b/src/app/atmosphereCompat.test.mjs @@ -0,0 +1,209 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + isAppleMobilePlatform, + probeOutParamVaryingLinkFailure, + shouldDisableModelAtmosphere, + applyModelAtmosphereWorkaround, +} from './atmosphereCompat.js'; + +const IPAD_OS12 = { + userAgent: + 'Mozilla/5.0 (iPad; CPU OS 12_5 like Mac OS X) AppleWebKit/605.1.15', + platform: 'iPad', + maxTouchPoints: 5, +}; +// iPadOS 13+ desktop mode: no iPad token, reports as a Mac. +const IPAD_DESKTOP_MODE = { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', + platform: 'MacIntel', + maxTouchPoints: 5, +}; +const MAC_DESKTOP = { + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', + platform: 'MacIntel', + maxTouchPoints: 0, +}; +const LINUX_DESKTOP = { + userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/141', + platform: 'Linux x86_64', + maxTouchPoints: 0, +}; + +/** Minimal GL double. `linkOk: false` mimics the Metal translator's refusal. */ +function fakeGl({ linkOk = true, compileOk = true, throwOn = null } = {}) { + const deleted = { shaders: 0, programs: 0 }; + return { + deleted, + VERTEX_SHADER: 1, + FRAGMENT_SHADER: 2, + COMPILE_STATUS: 3, + LINK_STATUS: 4, + createShader: () => ({}), + createProgram: () => ({}), + shaderSource() { + if (throwOn === 'shaderSource') throw new Error('context lost'); + }, + compileShader() {}, + attachShader() {}, + linkProgram() {}, + getShaderParameter: () => compileOk, + getProgramParameter: () => linkOk, + deleteShader: () => (deleted.shaders += 1), + deleteProgram: () => (deleted.programs += 1), + }; +} + +test('identifies iPadOS in both legacy and desktop-mode guises', () => { + assert.equal(isAppleMobilePlatform(IPAD_OS12), true); + assert.equal(isAppleMobilePlatform(IPAD_DESKTOP_MODE), true); +}); + +test('does not mistake a real Mac or a Linux desktop for Apple mobile', () => { + // Regression guard: the desktop-mode check keys on touch points, and a Mac + // reports the same MacIntel platform string with zero of them. + assert.equal(isAppleMobilePlatform(MAC_DESKTOP), false); + assert.equal(isAppleMobilePlatform(LINUX_DESKTOP), false); +}); + +test('treats a missing navigator as not-Apple rather than throwing', () => { + assert.equal(isAppleMobilePlatform(undefined), false); + assert.equal(isAppleMobilePlatform(null), false); +}); + +test('detects desktop-mode iPadOS without navigator.platform', () => { + // `navigator.platform` is deprecated and can be absent or stripped; the UA's + // Macintosh token plus touch points has to carry the detection on its own. + assert.equal( + isAppleMobilePlatform({ + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', + maxTouchPoints: 5, + }), + true, + ); + // Same UA, no touch: a real Mac, which must stay untouched. + assert.equal( + isAppleMobilePlatform({ + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15', + maxTouchPoints: 0, + }), + false, + ); +}); + +test('probe reports failure only when the program fails to link', () => { + assert.equal( + probeOutParamVaryingLinkFailure(fakeGl({ linkOk: false })), + true, + ); + assert.equal( + probeOutParamVaryingLinkFailure(fakeGl({ linkOk: true })), + false, + ); +}); + +test('a compile failure is not reported as the link bug', () => { + // The Metal defect surfaces at link time; both stages compile cleanly. + // Treating a compile error as the bug would disable fog for the wrong reason. + assert.equal( + probeOutParamVaryingLinkFailure( + fakeGl({ compileOk: false, linkOk: false }), + ), + false, + ); +}); + +test('probe releases its GL objects on both paths', () => { + const ok = fakeGl({ linkOk: true }); + probeOutParamVaryingLinkFailure(ok); + assert.equal(ok.deleted.shaders, 2); + assert.equal(ok.deleted.programs, 1); + + const thrown = fakeGl({ throwOn: 'shaderSource' }); + probeOutParamVaryingLinkFailure(thrown); + assert.equal(thrown.deleted.shaders, 2, 'shaders freed even when GL throws'); + assert.equal(thrown.deleted.programs, 1); +}); + +test('probe treats a null context as healthy, not broken', () => { + assert.equal(probeOutParamVaryingLinkFailure(null), false); +}); + +test('a failing probe disables the stage on any platform', () => { + assert.equal( + shouldDisableModelAtmosphere({ + navigatorLike: LINUX_DESKTOP, + createProbeContext: () => fakeGl({ linkOk: false }), + }), + true, + ); +}); + +test('a passing probe still quarantines Apple mobile but frees desktops', () => { + const createProbeContext = () => fakeGl({ linkOk: true }); + assert.equal( + shouldDisableModelAtmosphere({ + navigatorLike: IPAD_DESKTOP_MODE, + createProbeContext, + }), + true, + ); + assert.equal( + shouldDisableModelAtmosphere({ + navigatorLike: LINUX_DESKTOP, + createProbeContext, + }), + false, + ); +}); + +test('falls back to platform detection when no context can be made', () => { + const createProbeContext = () => null; + assert.equal( + shouldDisableModelAtmosphere({ + navigatorLike: IPAD_OS12, + createProbeContext, + }), + true, + ); + assert.equal( + shouldDisableModelAtmosphere({ + navigatorLike: MAC_DESKTOP, + createProbeContext, + }), + false, + ); +}); + +test('applies to the scene by clearing renderable, never enabled', () => { + // `enabled` must survive: Fog.update short-circuits when it is false, which + // forfeits the density used for 3D Tiles screen-space-error scaling, and + // visualSettings' IR boost saves/restores that same flag. + const scene = { fog: { enabled: true, renderable: true } }; + const applied = applyModelAtmosphereWorkaround(scene, { + navigatorLike: IPAD_DESKTOP_MODE, + createProbeContext: () => null, + }); + assert.equal(applied, true); + assert.equal(scene.fog.renderable, false); + assert.equal(scene.fog.enabled, true); +}); + +test('leaves unaffected devices completely untouched', () => { + const scene = { fog: { enabled: true, renderable: true } }; + const applied = applyModelAtmosphereWorkaround(scene, { + navigatorLike: LINUX_DESKTOP, + createProbeContext: () => fakeGl({ linkOk: true }), + }); + assert.equal(applied, false); + assert.equal(scene.fog.renderable, true); +}); + +test('tolerates a scene with no fog object', () => { + assert.equal(applyModelAtmosphereWorkaround({}, {}), false); + assert.equal(applyModelAtmosphereWorkaround(null, {}), false); +}); diff --git a/src/app/viewer.js b/src/app/viewer.js index ec58f52dc..9b6e7f631 100644 --- a/src/app/viewer.js +++ b/src/app/viewer.js @@ -1,4 +1,5 @@ import * as Cesium from 'cesium'; +import { applyModelAtmosphereWorkaround } from './atmosphereCompat.js'; const PINCH_ZOOM_MULTIPLIER = 8; const MAX_PINCH_PIXEL_DELTA = 120; @@ -124,6 +125,10 @@ export function createApplicationViewer({ container, creditContainer }) { }); try { viewer.targetFrameRate = 60; + // Before any tile builds a draw command: Cesium's per-vertex model + // atmosphere fails to LINK on Apple's Metal backend and kills the + // render loop. See app/atmosphereCompat.js. + applyModelAtmosphereWorkaround(viewer.scene); viewer.scene.globe.show = false; viewer.scene.skyAtmosphere.show = true; viewer.scene.skyAtmosphere.atmosphereLightIntensity = 18; From 005254fcd15454cedac2bc5a52f6e4894b78b8c6 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu <106619546+bilawalsidhu@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:49:39 -0500 Subject: [PATCH 08/22] fix(weather): upright N. America cloud tiles; lift shells over coarse 3D tiles (#730) --- src/layers/weather/infraredImage.js | 12 +++++-- src/layers/weather/infraredImage.test.mjs | 12 +++++++ src/layers/weather/rendering.js | 8 ++++- src/layers/weather/shellFixture.mjs | 2 ++ src/layers/weather/shellRendering.js | 25 ++++++++++++++ src/layers/weather/shellRendering.test.mjs | 39 ++++++++++++++++++++++ src/layers/weather/weather.test.mjs | 35 +++++++++++++++++++ 7 files changed, 130 insertions(+), 3 deletions(-) diff --git a/src/layers/weather/infraredImage.js b/src/layers/weather/infraredImage.js index 5b2db50b3..ccbb8e991 100644 --- a/src/layers/weather/infraredImage.js +++ b/src/layers/weather/infraredImage.js @@ -4,13 +4,21 @@ import { infraredAlpha } from './infraredAlpha.js'; export const MAX_MOSAIC_BYTES = 4 * 1024 * 1024; -/** Apply the display transfer once to a decoded image. */ -export function processInfraredImage(image, mode, createCanvas) { +/** Apply the display transfer once to a decoded image; `flipY` draws a + * bottom-up image back upright. */ +export function processInfraredImage( + image, + mode, + createCanvas, + { flipY = false } = {}, +) { const canvas = createCanvas(); canvas.width = image.width; canvas.height = image.height; const context = canvas.getContext('2d'); + if (flipY) context.setTransform(1, 0, 0, -1, 0, image.height); context.drawImage(image, 0, 0); + if (flipY) context.setTransform(1, 0, 0, 1, 0, 0); const pixels = context.getImageData(0, 0, canvas.width, canvas.height); pixels.data.set(infraredAlpha(pixels.data, mode)); context.putImageData(pixels, 0, 0); diff --git a/src/layers/weather/infraredImage.test.mjs b/src/layers/weather/infraredImage.test.mjs index f93a7816a..f3937abc3 100644 --- a/src/layers/weather/infraredImage.test.mjs +++ b/src/layers/weather/infraredImage.test.mjs @@ -10,8 +10,12 @@ import { function canvas() { const output = {}; output.getContext = () => ({ + setTransform: (...matrix) => { + output.transforms = [...(output.transforms ?? []), matrix]; + }, drawImage: (image) => { output.image = image; + output.drawnWith = output.transforms?.at(-1) ?? null; }, getImageData: () => ({ data: new Uint8ClampedArray([194, 0, 0, 200]) }), putImageData: (pixels) => { @@ -34,6 +38,14 @@ test('decoded pixels are processed once on a same-sized canvas', () => { ); }); +test('flipY draws a bottom-up image upright, then restores the transform', () => { + const image = { width: 4, height: 8 }; + assert.equal(processInfraredImage(image, 'full', canvas).drawnWith, null); + const output = processInfraredImage(image, 'full', canvas, { flipY: true }); + assert.deepEqual(output.drawnWith, [1, 0, 0, -1, 0, 8]); + assert.deepEqual(output.transforms.at(-1), [1, 0, 0, 1, 0, 0]); +}); + test('mosaic fetch is capped at 4 MiB, forwards cancellation, and releases decoded images', async () => { const controller = new AbortController(); let closes = 0, diff --git a/src/layers/weather/rendering.js b/src/layers/weather/rendering.js index 34224ecc8..8d195f302 100644 --- a/src/layers/weather/rendering.js +++ b/src/layers/weather/rendering.js @@ -381,8 +381,14 @@ function createGlobeRendering({ if (request) frame.requests.add(request); return Promise.resolve(result) .then((image) => { + // Cesium decodes tiles as ImageBitmaps already flipped, since WebGL + // ignores UNPACK_FLIP_Y for them; a canvas upload flips again. if (!frame.closed && snapshot.product === 'clouds-regional') - image = processInfraredImage(image, infrared, createCanvas); + image = processInfraredImage(image, infrared, createCanvas, { + flipY: + typeof ImageBitmap !== 'undefined' && + image instanceof ImageBitmap, + }); frame.retries.delete(tileKey); frame.loaded++; return image; diff --git a/src/layers/weather/shellFixture.mjs b/src/layers/weather/shellFixture.mjs index 685c72c56..18e5de788 100644 --- a/src/layers/weather/shellFixture.mjs +++ b/src/layers/weather/shellFixture.mjs @@ -71,6 +71,8 @@ export function createShellCesium({ maximumTextureSize = 0 } = {}) { GeometryInstance, BlendingState: Cesium.BlendingState, Cartesian4: Cesium.Cartesian4, + Ellipsoid: Cesium.Ellipsoid, + Matrix4: Cesium.Matrix4, Math: Cesium.Math, Rectangle: Cesium.Rectangle, ContextLimits: { maximumTextureSize }, diff --git a/src/layers/weather/shellRendering.js b/src/layers/weather/shellRendering.js index 42a8ed521..cc8b32035 100644 --- a/src/layers/weather/shellRendering.js +++ b/src/layers/weather/shellRendering.js @@ -25,6 +25,12 @@ const NO_WINDOW = Object.freeze({ west: 0, south: 0, east: -1, north: -1 }); const DETAIL_MIN_WIDTH = 6; const DETAIL_GRID = 0.5; const shellHeights = new WeakMap(); +// From far away Google's coarse 3D tiles rise tens of kilometres above the +// ground and hide the shells, so the shells rise with the camera: 4 m per +// kilometre of camera height, at most 60 km, in 500 m steps. +const LIFT_PER_METRE = 0.004; +const MAX_LIFT = 60_000; +const LIFT_STEP = 500; // Cesium binds a 1×1 white texture to an image uniform until its first upload: // draw nothing until the full-extent image has arrived, and sample the detail // image only once it has arrived. The window is west, south, east, north in the @@ -71,6 +77,13 @@ const sameEdges = (a, b) => a.east === b.east && a.north === b.north; +/** Metres every shell rises for a camera this high above the ellipsoid. */ +export function shellLift(cameraHeight) { + if (!(cameraHeight > 0)) return 0; + const lift = Math.min(MAX_LIFT, cameraHeight * LIFT_PER_METRE); + return Math.round(lift / LIFT_STEP) * LIFT_STEP; +} + /** Keep weather shells first in the primitive collection, in ascending height: * higher shells draw over lower ones and other opaque content draws over both. */ export function orderWeatherShells(primitives, primitive, height) { @@ -135,6 +148,17 @@ export function createShellSurface({ }); primitives.add(primitive); orderWeatherShells(primitives, primitive, height); + // A uniform scale about the Earth's centre lifts every shell alike and keeps + // their order. + let lift = 0; + const offLift = scene.preRender?.addEventListener(() => { + const next = shellLift(scene.camera?.positionCartographic?.height); + if (next === lift) return; + lift = next; + primitive.modelMatrix = cesium.Matrix4.fromUniformScale( + 1 + lift / cesium.Ellipsoid.WGS84.maximumRadius, + ); + }); let image = null; let frames = 0; // The detail image, its own upload count and a window waiting for it to draw. @@ -265,6 +289,7 @@ export function createShellSurface({ destroyed = true; offRender?.(); offRender = null; + offLift?.(); if (!primitives.isDestroyed?.() && primitives.contains(primitive)) primitives.remove(primitive); if (!primitive.isDestroyed()) primitive.destroy(); diff --git a/src/layers/weather/shellRendering.test.mjs b/src/layers/weather/shellRendering.test.mjs index 3fcfdf28e..ad65dcef2 100644 --- a/src/layers/weather/shellRendering.test.mjs +++ b/src/layers/weather/shellRendering.test.mjs @@ -6,6 +6,7 @@ import { createWeatherShell, detailWindow, orderWeatherShells, + shellLift, WEATHER_SHELL_CACHE_BYTES, WEATHER_SHELL_HEIGHTS, } from './shellRendering.js'; @@ -297,6 +298,44 @@ test('a surface applies a new detail window only once its image has drawn; the s assert.equal(scene.postRender.size, 0); }); +test('shells rise with the camera over coarse 3D tiles, in steps, up to 60 km', () => { + assert.equal(shellLift(undefined), 0); + assert.equal(shellLift(1_200), 0, 'city views keep the shell heights'); + assert.equal(shellLift(3_000_000), 12_000); + assert.equal(shellLift(8_000_000), 32_000); + assert.equal(shellLift(18_500_000), 60_000); + const camera = { positionCartographic: { height: 1_200 } }; + const scene = { + primitives: new Cesium.PrimitiveCollection(), + preRender: new Cesium.Event(), + postRender: new Cesium.Event(), + camera, + requestRender() {}, + }; + const surface = createShellSurface({ + viewer: { scene }, + cesium: Cesium, + rectangle: Cesium.Rectangle.fromDegrees(-130, 20, -60, 55), + height: 5_800, + }); + const primitive = scene.primitives.get(0); + const identity = primitive.modelMatrix; + scene.preRender.raiseEvent(); + assert.equal(primitive.modelMatrix, identity, 'no lift, no new matrix'); + camera.positionCartographic.height = 8_000_000; + scene.preRender.raiseEvent(); + const radius = Cesium.Ellipsoid.WGS84.maximumRadius; + assert.ok( + Math.abs(primitive.modelMatrix[0] * radius - (radius + 32_000)) < 1e-3, + ); + const lifted = primitive.modelMatrix; + camera.positionCartographic.height = 8_010_000; + scene.preRender.raiseEvent(); + assert.equal(primitive.modelMatrix, lifted, 'the same step keeps the matrix'); + surface.destroy(); + assert.equal(scene.preRender.numberOfListeners, 0); +}); + test('real Cesium builds the two-texture material, the opaque-pass appearance and the rectangle primitive', () => { const scene = { primitives: new Cesium.PrimitiveCollection(), diff --git a/src/layers/weather/weather.test.mjs b/src/layers/weather/weather.test.mjs index abdc01c7a..a80ad6560 100644 --- a/src/layers/weather/weather.test.mjs +++ b/src/layers/weather/weather.test.mjs @@ -75,9 +75,13 @@ function target(extra = {}) { function mockCanvas() { const canvas = { width: 0, height: 0 }; canvas.getContext = () => ({ + setTransform(...matrix) { + canvas.transforms = [...(canvas.transforms ?? []), matrix]; + }, drawImage(source, ...crop) { canvas.source = source; canvas.crop = crop; + canvas.drawnWith = canvas.transforms?.at(-1) ?? null; }, getImageData: () => ({ data: new Uint8ClampedArray([194, 100, 0, 200]) }), putImageData(pixels) { @@ -1510,6 +1514,37 @@ test('regional transfer preserves deferrals, original Request, rejection and can assert.equal(await stage, false); }); +test('regional tiles decoded as flipped ImageBitmaps are drawn back upright', async () => { + const hadBitmap = Object.hasOwn(globalThis, 'ImageBitmap'); + const previous = globalThis.ImageBitmap; + globalThis.ImageBitmap = class ImageBitmap { + width = 256; + height = 256; + }; + try { + const h = renderingHarness(); + void h.rendering.setFrame( + { ...snapshot, product: 'clouds-regional' }, + times[0], + ); + const provider = h.providers[0]; + // Cesium decodes with imageOrientation 'flipY'; the canvas upload flips again. + provider.response = { promise: Promise.resolve(new ImageBitmap()) }; + const bitmapTile = await provider.requestImage(0, 0, 0, {}); + assert.deepEqual(bitmapTile.drawnWith, [1, 0, 0, -1, 0, 256]); + // An HTMLImageElement fallback is not pre-flipped. + provider.response = { + promise: Promise.resolve({ width: 256, height: 256 }), + }; + const imageTile = await provider.requestImage(1, 0, 0, {}); + assert.equal(imageTile.drawnWith, null); + h.rendering.clear(); + } finally { + if (hadBitmap) globalThis.ImageBitmap = previous; + else delete globalThis.ImageBitmap; + } +}); + for (const kind of ['globe']) { test(`infrared mode restages the same time on ${kind}, retaining the old layer for one rendered frame`, async () => { let host, From 143cffd7ca92eab4f894f41ac4888636d20ee98c Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu <106619546+bilawalsidhu@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:55:41 -0500 Subject: [PATCH 09/22] fix(ui): keep panel scroll position through rail layout passes (#731) The right-rail layout pass lifts the Context, CCTV, Weather and Recent Imagery panel scrollers to measure their natural heights, which clamps their scroll offset to zero, but it only restored scrollers tagged data-rail-scroller. A layer that refreshes its Layers-panel row every second snapped the Context panel back to the top while scrolling. Tag every lifted scroller and pin the pairing with a test that reads the measuring CSS. Co-authored-by: Bilawal Sidhu --- scripts/format-scope.json | 3 ++- src/ui/railScrollers.test.mjs | 30 ++++++++++++++++++++++++++++++ src/ui/templates/context.html | 6 +++--- src/ui/templates/layer-panels.html | 2 +- 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 src/ui/railScrollers.test.mjs diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 97bb0b8f2..c4b1ce72d 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -808,5 +808,6 @@ "src/ui/imageryBoxTool.test.mjs", "src/ui/recentImagery.test.mjs", "src/layers/recentImagery/testDoubles.mjs", - "scripts/qa-recent-imagery.mjs" + "scripts/qa-recent-imagery.mjs", + "src/ui/railScrollers.test.mjs" ] diff --git a/src/ui/railScrollers.test.mjs b/src/ui/railScrollers.test.mjs new file mode 100644 index 000000000..cf549be35 --- /dev/null +++ b/src/ui/railScrollers.test.mjs @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; + +const read = (path) => readFileSync(new URL(path, import.meta.url), 'utf8'); + +// The rail's measuring pass lifts these panel scrollers to their natural +// height, which clamps their scroll offset to zero. Only scrollers tagged +// `data-rail-scroller` get their offset restored afterwards, so every lifted +// scroller must carry the tag or a layout pass snaps it back to the top. +test('every scroller the rail measuring pass lifts restores its scroll offset', () => { + const css = read('./styles/layers.css'); + const templates = [ + read('./templates/context.html'), + read('./templates/layer-panels.html'), + ].join('\n'); + const lifted = [ + ...css.matchAll( + /#right-context-rail\[data-rail-measuring\]\s*>\s*\[data-panel-id\]:not\(\.collapsed\)\s*>\s*\.([a-z-]+)/g, + ), + ].map((match) => match[1]); + assert.ok(lifted.length > 0, 'measuring CSS lists lifted scrollers'); + for (const className of lifted) { + const tag = templates.match( + new RegExp(`
]*>`), + )?.[0]; + if (!tag) continue; + assert.match(tag, /data-rail-scroller/, `${className} restores its scroll`); + } +}); diff --git a/src/ui/templates/context.html b/src/ui/templates/context.html index 6ce5562a6..fbed0cf7c 100644 --- a/src/ui/templates/context.html +++ b/src/ui/templates/context.html @@ -2,7 +2,7 @@