From 499e1168a7090a37bbebd52d40c2b2842260ca7c Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu Date: Sun, 13 Sep 2026 17:08:23 -0500 Subject: [PATCH 1/2] fix(layers): release bundled GeoJSON entities on toggle-off instead of hiding them Toggling datacenters, dams, or submarine cables off only set `show = false` on their data sources. Cesium's DataSourceDisplay still walks every visualizer of a hidden source each frame, and the ~11.5k hidden entities (hundreds of MB) stayed resident, so the whole scene rendered ~2x slower for the rest of the session after a single on/off round trip (owner field test 2026-09-13). disable() now removes the data sources from the viewer and drops the built records; the parsed dataset (NDJSON features / TeleGeography JSON) is cached so enable() rebuilds without a refetch or re-parse. A disable that lands while a load is in flight releases the finished build instead of parking it hidden. destroy() clears the cache. Measured with a headful Chrome harness (flights+military+satellites on, camera rotating, 250-frame windows), before -> after: - entities resident after OFF: 11,537 -> 0 - DataSourceDisplay.update per frame after OFF: 1.09 ms -> 0.02 ms - scene.render per frame after OFF: 13.9 ms (control 6.6 ms) -> 5.2 ms (fresh baseline 5.0 ms in the same run) - JS heap 30 s after OFF: 1.6 GB and climbing -> 342 MB after GC Tests: the GeoJSON lifecycle and cables suites assert disable leaves zero sources/contexts and re-enable rebuilds with fetch called once. --- src/data/localGeojsonCore.js | 78 +++++++++++++++---- src/data/localGeojsonLifecycle.test.mjs | 35 +++++++-- src/data/telegeographySubmarineCables.js | 66 +++++++++++++--- .../telegeographySubmarineCables.test.mjs | 43 +++++++++- 4 files changed, 190 insertions(+), 32 deletions(-) diff --git a/src/data/localGeojsonCore.js b/src/data/localGeojsonCore.js index b90d1b66b..764005568 100644 --- a/src/data/localGeojsonCore.js +++ b/src/data/localGeojsonCore.js @@ -354,6 +354,17 @@ export function createLocalGeoJsonLayer( let _destroyed = false; let _loadPromise = null; let _loadController = null; + /** + * The parsed bundled dataset, kept across disable/enable so a re-enable + * rebuilds entities without refetching. The Cesium entities themselves are + * NOT kept: a hidden data source still costs every frame (DataSourceDisplay + * walks every visualizer over every entity regardless of `show`), and the + * ~11k entities of the three bundled layers hold hundreds of MB, so leaving + * them parked after a toggle-off made the whole scene render ~2× slower for + * the rest of the session (owner field test 2026-09-13). + * @type {Array|null} + */ + let _cachedFeatures = null; /** * Globe-LOD active set: the record ids allowed to carry a live stem right * now. This bounds geometry refreshes and ground-sample work to the @@ -432,6 +443,27 @@ export function createLocalGeoJsonLayer( host: overlayHost, }); + /** + * Take the built entities out of the scene entirely. The parsed features + * stay cached, so the next enable() rebuilds without a fetch; what must not + * survive a disable is the per-frame visualizer walk and the entity memory. + * @param {object} viewer + */ + const releaseDataSource = (viewer) => { + if (!_dataSource) return; + const source = _dataSource; + _dataSource = null; + _stemRecords = []; + _stemGeometryDirty = true; + _lastVisibilityUpdate = Number.NEGATIVE_INFINITY; + removeEntityContextsForLayer(id); + try { + viewer?.dataSources?.remove(source, true); + } catch { + /* already gone */ + } + }; + const disableLayer = (viewer) => { _enabled = false; clearGroundRetryRender(); @@ -440,7 +472,7 @@ export function createLocalGeoJsonLayer( _lastLodBudgetLimit = 0; _lodComputed = false; _lastLodProbeMs = Number.NEGATIVE_INFINITY; - if (_dataSource) _dataSource.show = false; + releaseDataSource(viewer); _overlayPublisher.hide(); clearSelectedEntityContextForLayer(id); if (viewer?.selectedEntity?.__localLayerId === id) { @@ -529,20 +561,26 @@ export function createLocalGeoJsonLayer( // windows (before vs after the add settles) need different cleanup. let addedToScene = false; try { - const response = await fetch(url, { - signal: _loadController.signal, - }); - if (_destroyed) return; - // A 404 returns an HTML body that would otherwise die in JSON.parse - // one line later, reported as a parse error for a missing file. - if (!response.ok) { - throw new Error(`HTTP ${response.status ?? '?'}`); - } - const text = await response.text(); - if (_destroyed) return; - const lines = text.split('\n').filter((l) => l.trim().length > 0); + let features = _cachedFeatures; + if (!features) { + const response = await fetch(url, { + signal: _loadController.signal, + }); + if (_destroyed) return; + // A 404 returns an HTML body that would otherwise die in JSON.parse + // one line later, reported as a parse error for a missing file. + if (!response.ok) { + throw new Error(`HTTP ${response.status ?? '?'}`); + } + const text = await response.text(); + if (_destroyed) return; + const lines = text + .split('\n') + .filter((l) => l.trim().length > 0); - const features = lines.map((line) => JSON.parse(line)); + features = lines.map((line) => JSON.parse(line)); + _cachedFeatures = features; + } const geojson = { type: 'FeatureCollection', @@ -991,9 +1029,14 @@ export function createLocalGeoJsonLayer( } // Honor a disable() that landed while we were awaiting the fetch/parse: - // disable() runs before _dataSource exists, so its show=false is a no-op — - // reading _enabled here (rather than forcing true) respects the toggle-off. - if (_dataSource) _dataSource.show = _enabled; + // disable() runs before _dataSource exists, so its release is a no-op — + // release the finished build here instead of parking it hidden in the + // scene (the parsed features stay cached for the next enable). + if (!_enabled) { + releaseDataSource(viewer); + return; + } + if (_dataSource) _dataSource.show = true; viewer.scene.requestRender?.(); }, @@ -1016,6 +1059,7 @@ export function createLocalGeoJsonLayer( } _overlayPublisher.destroy(); _dataSource = null; + _cachedFeatures = null; _stemRecords = []; _count = 0; _lastUpdate = null; diff --git a/src/data/localGeojsonLifecycle.test.mjs b/src/data/localGeojsonLifecycle.test.mjs index 984e6e4af..245867155 100644 --- a/src/data/localGeojsonLifecycle.test.mjs +++ b/src/data/localGeojsonLifecycle.test.mjs @@ -54,7 +54,8 @@ test('destroy removes context records and permits a fresh replacement', async t const oldEntity = [...getContextStore().entities.values()][0].entity; assert.equal(getContextStore().entities.size, 1); env.layer.disable(env.viewer); - assert.equal(getContextStore().entities.size, 1, 'disabled cached source retains its records'); + assert.equal(getContextStore().entities.size, 0, 'disable releases its records with the source'); + assert.equal(env.sources.size, 0, 'disable removes the built source from the scene'); env.layer.destroy(env.viewer); assert.equal(getContextStore().entities.size, 0); assert.equal(env.sources.size, 0); @@ -126,17 +127,41 @@ test('concurrent enable and disable/re-enable share one pending dataset load', a assert.equal([...env.sources][0].show, true); }); -test('disable during loading keeps the completed source hidden until re-enabled', async t => { +test('disable during loading releases the completed source; re-enable rebuilds without refetching', async t => { const env = harness(t); const release = deferred(); - t.mock.method(globalThis, 'fetch', async () => { await release.promise; return response(); }); + const fetchMock = t.mock.method(globalThis, 'fetch', async () => { await release.promise; return response(); }); const loading = env.layer.enable(env.viewer); env.layer.disable(env.viewer); release.resolve(); await loading; - assert.equal([...env.sources][0].show, false); + assert.equal(env.sources.size, 0, 'a build that finishes after disable() is released, not parked hidden'); assert.equal(env.listeners.size, 0); + assert.equal(getContextStore().entities.size, 0); await env.layer.enable(env.viewer); - assert.equal([...env.sources][0].show, true); assert.equal(env.sources.size, 1); + assert.equal([...env.sources][0].show, true); + assert.equal(fetchMock.mock.callCount(), 1, 'the parsed dataset is cached across the toggle'); +}); + +test('a toggle-off frees the entities and a toggle-on rebuilds them from the cached dataset', async t => { + const env = harness(t); + const fetchMock = t.mock.method(globalThis, 'fetch', async () => response()); + await env.layer.enable(env.viewer); + const first = [...env.sources][0]; + assert.equal(first.entities.values.length, 1); + assert.equal(getContextStore().entities.size, 1); + env.layer.disable(env.viewer); + assert.equal(env.sources.size, 0, 'no data source stays registered while the layer is off'); + assert.equal(getContextStore().entities.size, 0); + assert.equal(env.layer.getLodDiagnostics().total, 0); + assert.equal(env.layer.getStats().count, 1, 'the row keeps reporting the dataset size'); + await env.layer.enable(env.viewer); + const second = [...env.sources][0]; + assert.notEqual(second, first, 're-enable builds a fresh source'); + assert.equal(second.show, true); + assert.equal(second.entities.values.length, 1); + assert.equal(getContextStore().entities.size, 1); + assert.equal(fetchMock.mock.callCount(), 1, 'no second fetch for a bundled dataset'); + assert.equal(env.layer.getLodDiagnostics().total, 1); }); diff --git a/src/data/telegeographySubmarineCables.js b/src/data/telegeographySubmarineCables.js index 53d1a664e..20d6e02ff 100644 --- a/src/data/telegeographySubmarineCables.js +++ b/src/data/telegeographySubmarineCables.js @@ -496,6 +496,14 @@ export function createTeleGeographySubmarineCableLayer({ let _cableDataSource = null; let _landingDataSource = null; let _referenceDataSource = null; + /** + * Fetched cable/landing JSON kept across disable/enable. A disable RELEASES + * the three data sources (see releaseDataSources) instead of hiding them, + * so a re-enable has to rebuild the entities; the network/parse half of + * that rebuild is the expensive part users would notice, so it is cached. + */ + let _cachedCableJson = null; + let _cachedLandingJson = null; let _referenceRecords = []; let _surfaceRecords = []; let _clickHandler = null; @@ -553,11 +561,17 @@ export function createTeleGeographySubmarineCableLayer({ const owns = () => generation === _loadGeneration && !abort.signal.aborted; try { - const [cableJson, landingJson] = await Promise.all([ - fetchJson(cableUrl, abort.signal), - fetchJson(landingPointUrl, abort.signal), - ]); - if (!owns()) return; + let cableJson = _cachedCableJson; + let landingJson = _cachedLandingJson; + if (!cableJson || !landingJson) { + [cableJson, landingJson] = await Promise.all([ + fetchJson(cableUrl, abort.signal), + fetchJson(landingPointUrl, abort.signal), + ]); + if (!owns()) return; + _cachedCableJson = cableJson; + _cachedLandingJson = landingJson; + } const cableFeatures = normalizeFeatures(cableJson, 'cable'); const landingFeatures = normalizeFeatures(landingJson, 'landing'); @@ -706,6 +720,37 @@ export function createTeleGeographySubmarineCableLayer({ } } + /** + * Remove the three data sources from the viewer and forget the built + * entities. Hiding them (`show = false`) is NOT enough: Cesium's + * DataSourceDisplay still walks every visualizer of a hidden source on + * every frame, and the thousands of hidden polylines/billboards keep + * costing CPU and GC pressure long after the user toggled the layer off — + * that is the "everything gets sluggish after I turn layers off" report. + * The fetched JSON stays cached, so the next enable rebuilds without a + * network round trip or a re-parse. + */ + function releaseDataSources(viewer) { + const sources = [_cableDataSource, _landingDataSource, _referenceDataSource]; + _cableDataSource = null; + _landingDataSource = null; + _referenceDataSource = null; + _referenceRecords = []; + _surfaceRecords = []; + _pickByEntity = new WeakMap(); + _referenceLabelCount = 0; + _publishScratch.length = 0; + // The blend pass targets the collections being dropped; the rebuilt + // sources need their own pass. + _markerBlendDone = false; + _loaded = false; + _referenceSweepGate.reset(); + for (const source of sources) { + if (!source) continue; + try { viewer?.dataSources?.remove(source, true); } catch { /* collection gone */ } + } + } + function updateVisibility() { if (_cableDataSource) _cableDataSource.show = _enabled; if (_landingDataSource) _landingDataSource.show = _enabled; @@ -1093,7 +1138,6 @@ export function createTeleGeographySubmarineCableLayer({ disable() { _enabled = false; - updateVisibility(); _overlayPublisher.hide(); resetPublishSignature(); if (_loading && _abort) { @@ -1101,6 +1145,10 @@ export function createTeleGeographySubmarineCableLayer({ _loading = false; _loadingLabel = ''; } + // Free the entities rather than hide them; enable() rebuilds from the + // cached JSON. An in-flight load's own post-await ownership check + // removes whatever that stale generation adds later. + releaseDataSources(_viewer); }, update(viewer) { @@ -1109,9 +1157,9 @@ export function createTeleGeographySubmarineCableLayer({ destroy(viewer) { if (_abort) _abort.abort(); - if (_cableDataSource && viewer) viewer.dataSources.remove(_cableDataSource, true); - if (_landingDataSource && viewer) viewer.dataSources.remove(_landingDataSource, true); - if (_referenceDataSource && viewer) viewer.dataSources.remove(_referenceDataSource, true); + releaseDataSources(viewer || _viewer); + _cachedCableJson = null; + _cachedLandingJson = null; // hide() clears every published host entry and goes invisible while // keeping the publisher reusable — the legacy layer supported re-init // after destroy, and hidden publishers already drop late publishes. diff --git a/src/data/telegeographySubmarineCables.test.mjs b/src/data/telegeographySubmarineCables.test.mjs index 11550561e..58bfaa87d 100644 --- a/src/data/telegeographySubmarineCables.test.mjs +++ b/src/data/telegeographySubmarineCables.test.mjs @@ -743,7 +743,15 @@ test('an abort landing at the post-cable-GeoJson boundary stops the stale load b env.layer.disable(); env.layer.enable(env.viewer); await env.settle(); - assert.equal(geojson.calls.length, 1, 'stale load A must not start the landing GeoJson build'); + // A already cached the fetched JSON, so load B skips the network and goes + // straight to its own CABLE build; the only new GeoJson call must be that + // one — a landing build here would mean stale A ran on past its check. + assert.equal(env.pendingFetches.length, 0, 'load B rebuilds from the cached JSON without refetching'); + assert.equal(geojson.calls.length, 2, 'stale load A must not start the landing GeoJson build'); + assert.ok( + Cesium.Color.equals(geojson.calls[1][1].stroke, geojson.calls[0][1].stroke), + 'the second GeoJson call is load B\'s cable build, not A\'s landing build', + ); env.releaseAll(); await env.settle(); @@ -1254,3 +1262,36 @@ test('an unchanged cohort never republishes, so a parked camera stays governor-i env.layer.destroy(env.viewer); }); + +test('a toggle-off releases every data source and a toggle-on rebuilds them without refetching', async () => { + const env = await createRealCableLayerHarness(); + assert.equal(env.dataSources.length, 3, 'cable + landing + reference sources while on'); + const before = env.layer.getStats().count; + assert.equal(before, 2); + + // Any fetch after this point is a bug: the rebuild must come from the cache. + const originalFetch = globalThis.fetch; + let fetches = 0; + globalThis.fetch = async () => { fetches += 1; throw new Error('unexpected refetch'); }; + try { + env.layer.disable(); + assert.equal(env.dataSources.length, 0, 'hidden entities are not free; disable must remove the sources'); + + env.layer.enable(env.viewer); + const deadline = Date.now() + 2_000; + while (Date.now() < deadline && env.dataSources.length < 3) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.equal(fetches, 0, 'the rebuild must not touch the network'); + assert.equal(env.layer.getStats().error, null); + assert.equal(env.dataSources.length, 3, 're-enable rebuilds the trio'); + assert.equal(env.layer.getStats().count, before); + const referenceSource = env.dataSources.find((ds) => /References/.test(ds.name || '')); + assert.ok(referenceSource?.entities.values.length > 0, 'reference stems are rebuilt'); + + env.layer.destroy(env.viewer); + assert.equal(env.dataSources.length, 0); + } finally { + globalThis.fetch = originalFetch; + } +}); From 9f0159343e6a0f4d8f7547812981dc77ffce1da1 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Sun, 13 Sep 2026 16:26:08 -0700 Subject: [PATCH 2/2] docs: describe infrastructure disable and rebuild lifecycle --- CHANGELOG.md | 6 ++++++ docs/CURRENT-STATE.md | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfa8b533e..49e15a285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Release disabled infrastructure rendering + +- Remove built Data Center, Dam and Submarine Cable entities when their layers + are disabled, avoiding retained visualizer work and entity memory. +- Keep parsed datasets cached for re-enable; rebuild entities without refetching. + ## Camera layer components - Separate camera source requests, placement, frames, projection, cards and calibration. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 5834a063c..0f4c03167 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,12 @@ # God's Eye View Current State +Data Centers, Dams and Submarine Cables release their built Cesium data sources +and record references when disabled. Parsed datasets remain cached for the layer +lifetime, so re-enable rebuilds entities without downloading or parsing again; +this can take longer than simply revealing hidden entities. Destruction clears +the parsed cache as well. A disable during loading cannot leave a completed +build hidden in the scene. + CCTV exposes a factory through `./layers/cctv`. Catalog and health requests, frame/media URLs, camera records, ground placement, geometry queues, playback, projection, cards, calibration and interaction have separate components. The