From 28f3243090fe2aa766d9df2f096e095811f04ac0 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Tue, 29 Sep 2026 00:20:16 -0500 Subject: [PATCH 01/27] feat(cyclones): publish explicit selection to shared context --- scripts/package-boundaries.json | 2 + src/app/constructCatalog.js | 4 +- src/app/layers/cyclones.js | 11 +++++ src/layers/cyclones/index.js | 52 +++++++++++++++++++++ src/layers/cyclones/index.test.mjs | 72 +++++++++++++++++++++++++++++- 5 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 src/app/layers/cyclones.js diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 276b9e0db..93bdcf7e1 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1340,6 +1340,7 @@ "src/app/layers/alprCameras.js", "src/app/layers/bikeshare.js", "src/app/layers/cctv.js", + "src/app/layers/cyclones.js", "src/app/layers/earthquakes.js", "src/app/layers/firms.js", "src/app/layers/flights.js", @@ -2075,6 +2076,7 @@ "src/app/layers/alprCameras.js", "src/app/layers/bikeshare.js", "src/app/layers/cctv.js", + "src/app/layers/cyclones.js", "src/app/layers/earthquakes.js", "src/app/layers/firms.js", "src/app/layers/flights.js", diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 7ed03d0d2..0437e1749 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -1,6 +1,6 @@ import { createWeatherClock } from '../layers/weather/clock.js'; import { createWeatherLayer } from '../layers/weather/index.js'; -import { createCyclonesLayer } from '../layers/cyclones/index.js'; +import { createApplicationCyclones } from './layers/cyclones.js'; import { createWindLayer } from '../layers/wind/index.js'; import { createLayerCatalog } from './catalog.js'; import { LAYER_STATE_REGISTRY } from '../data/layerState.js'; @@ -181,7 +181,7 @@ export function createApplicationCatalog({ id: 'weather-lightning', clock: weatherClock, }), - createCyclonesLayer({ feed: sources.cyclones }), + createApplicationCyclones({ feed: sources.cyclones }), ...createInfrastructureLayers(localGeoJsonServices), createApplicationCables({ source: sources.cables }), createApplicationFirms({ diff --git a/src/app/layers/cyclones.js b/src/app/layers/cyclones.js new file mode 100644 index 000000000..d671a11eb --- /dev/null +++ b/src/app/layers/cyclones.js @@ -0,0 +1,11 @@ +import { createCyclonesLayer } from '../../layers/cyclones/index.js'; +import * as context from '../../data/contextStore.js'; + +/** Wire cyclone selection into the application shared-context owner. */ +export function createApplicationCyclones(options) { + const browserContext = typeof window !== 'undefined' ? { context } : {}; + return createCyclonesLayer({ + ...browserContext, + ...options, + }); +} diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 666614176..2738bebe0 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -37,6 +37,7 @@ export function createCyclonesLayer({ feed, hitTestOverlay = hitTestWorldOverlay, overlayHost, + context = null, cesium = Cesium, createRendering = createCycloneRendering, matchMedia = globalThis.matchMedia?.bind(globalThis), @@ -52,6 +53,8 @@ export function createCyclonesLayer({ selectedId = null, selectionIntent = 'auto', navigationGeneration = 0, + selectedContextStormId = null, + selectedContextEntity = null, clickHandler = null, removeClickCapture = null; let enabled = false, @@ -62,6 +65,46 @@ export function createCyclonesLayer({ const notify = () => listener?.(); const selected = () => snapshot?.storms.find((storm) => storm.id === selectedId) || null; + + function clearSelectedContext() { + context?.removeEntityContextsForLayer?.('weather-cyclones'); + selectedContextStormId = null; + selectedContextEntity = null; + } + + function publishSelectedContext({ announce = false } = {}) { + const storm = selected(); + if (!storm || !context) { + if (!storm) clearSelectedContext(); + return null; + } + selectedContextEntity ||= { show: true }; + const record = context.registerEntityContext?.(selectedContextEntity, { + id: `cyclone:${storm.id}`, + layerId: 'weather-cyclones', + layerName: 'Cyclone advisories', + source: 'NOAA NHC / CPHC', + label: `Cyclone · ${storm.name}`, + latitude: storm.position.latitude, + longitude: storm.position.longitude, + properties: { + classification: storm.classification, + basin: storm.basin, + advisoryNumber: storm.advisoryNumber, + issuedAt: storm.issuedAt, + positionAt: storm.positionAt, + windKt: storm.windKt, + pressureHpa: storm.pressureHpa, + movement: storm.movement, + advisoryUrl: storm.advisoryUrl, + }, + }); + if (!record) return null; + selectedContextStormId = storm.id; + if (announce) context.selectEntityContext?.(selectedContextEntity); + return record; + } + function select(id) { if (selectedId !== id) ++navigationGeneration; selectedId = id; @@ -206,6 +249,7 @@ export function createCyclonesLayer({ selectedId = null; selectionIntent = 'auto'; ++navigationGeneration; + clearSelectedContext(); rendering?.clear(); }, async update(_viewer, { signal } = {}) { @@ -229,6 +273,7 @@ export function createCyclonesLayer({ snapshot = next; selectedId = null; ++navigationGeneration; + clearSelectedContext(); error = next.reason || 'Cyclone advisories unavailable'; return true; } @@ -251,6 +296,10 @@ export function createCyclonesLayer({ : snapshot.storms[0]?.id || null, ); else rendering.setSelection(selectedId); + if (selectedContextStormId) { + if (selectedId === selectedContextStormId) publishSelectedContext(); + else clearSelectedContext(); + } return true; } catch (cause) { if (controller.signal.aborted || request !== controller) return false; @@ -260,6 +309,7 @@ export function createCyclonesLayer({ snapshot = null; selectedId = null; ++navigationGeneration; + clearSelectedContext(); return true; } finally { signal?.removeEventListener('abort', abort); @@ -275,6 +325,7 @@ export function createCyclonesLayer({ if (params.clear === true || params.stormId === null) { selectionIntent = 'cleared'; select(null); + clearSelectedContext(); notify(); } else if ( typeof params.stormId === 'string' && @@ -282,6 +333,7 @@ export function createCyclonesLayer({ ) { selectionIntent = 'user'; select(params.stormId); + publishSelectedContext({ announce: true }); notify(); } const storm = selected(); diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index 4127871f4..7d796bef7 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -338,7 +338,7 @@ test('a click on a storm card or lead-hour label selects that storm without pick }); function harness( feed = { getSnapshot: async () => snapshot() }, - { reducedMotion = false } = {}, + { reducedMotion = false, context = null } = {}, ) { const applied = [], navigation = [], @@ -368,6 +368,7 @@ function harness( }; const layer = createCyclonesLayer({ feed, + context, createRendering: () => rendering, matchMedia: () => ({ matches: reducedMotion }), openLink: (url) => opened.push(url), @@ -401,6 +402,75 @@ function harness( }, }; } +test('explicit cyclone selection publishes shared context without auto-claiming it', async () => { + let next = snapshot([storm(), storm('ep162026')]); + const records = new Map(); + let selectedContext = null; + let selectedEvents = 0; + const context = { + registerEntityContext(entity, metadata) { + entity.__gevContextId = metadata.id; + const record = { ...metadata, entity }; + records.set(metadata.id, record); + return record; + }, + selectEntityContext(entity) { + selectedContext = records.get(entity.__gevContextId) || null; + selectedEvents += 1; + return selectedContext; + }, + removeEntityContextsForLayer(layerId) { + for (const [id, record] of records) { + if (record.layerId === layerId) records.delete(id); + } + if (selectedContext?.layerId === layerId) selectedContext = null; + }, + }; + const h = harness({ getSnapshot: async () => next }, { context }); + h.layer.enable(); + await h.layer.update(); + + assert.equal( + selectedEvents, + 0, + 'the automatic first-storm row selection must not claim shared context', + ); + assert.equal(records.size, 0); + + h.layer.setParams({ stormId: 'ep162026' }); + assert.equal(selectedEvents, 1); + assert.equal(selectedContext?.layerId, 'weather-cyclones'); + assert.equal(selectedContext?.source, 'NOAA NHC / CPHC'); + assert.equal(selectedContext?.label, 'Cyclone · Another system'); + assert.equal(selectedContext?.latitude, 15.5); + assert.equal(selectedContext?.longitude, -125.8); + assert.equal(selectedContext?.properties.advisoryNumber, '10'); + + await h.layer.update(); + assert.equal( + selectedEvents, + 1, + 'refresh updates the selected record without re-announcing the click', + ); + + next = snapshot([storm()]); + await h.layer.update(); + assert.equal( + selectedContext, + null, + 'a vanished user-selected storm must clear instead of transferring context to the fallback row', + ); + assert.equal(records.size, 0); + + h.layer.setParams({ stormId: 'ep152026' }); + assert.equal(selectedEvents, 2); + h.layer.setParams({ clear: true }); + assert.equal(selectedContext, null); + assert.equal(records.size, 0); + + h.layer.destroy(); +}); + test('refresh preserves selection intent, including explicit clears and missing storms', async () => { let next = snapshot([storm(), storm('ep162026')]); const h = harness({ getSnapshot: async () => next }); From 81f49406b80f2f4c8076ed739360f79ab2e2c34f Mon Sep 17 00:00:00 2001 From: Wilmer Alzate Date: Wed, 30 Sep 2026 12:31:59 -0500 Subject: [PATCH 02/27] perf(flights): skip dead-reckoning contacts that stay behind the horizon The fleet tick dead-reckoned, floored and distance-tested every contact on every ~80 ms tick, then hid the ones behind the horizon. The worldwide feed carries ~13k contacts; zoomed in, well under 1% are on screen, so nearly all of that work produced a hidden billboard. A contact that is already hidden and still behind the horizon at its last position now re-reckons only on its turn in a staggered one-in-12-ticks rotation (about once a second, HIDDEN_CONTACT_DR_TICKS). Anything that position puts in view, and any contact with a fleet model, takes the full pass immediately, so on-screen behaviour is unchanged. Hidden-inclusive proximity queries (getNearby includeHidden, getAllPositions) read positions at most about a second old, well inside the 30 s poll's uncertainty. Measured in Chrome on a Radeon 780M, ~12.9k contacts, 56 on screen over Colombia at 4 km, same view before and after: fleet tick 7.66 -> 2.03 ms/frame, longest tick 50.9 -> ~16 ms frame 20.6 -> 14.0 ms, 42.9 -> 57.8 FPS Teleporting the camera to Madrid showed 975 contacts within 100 ms. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++++ docs/CURRENT-STATE.md | 2 +- scripts/format-scope.json | 1 + .../flights/hiddenContactRotation.test.mjs | 42 +++++++++++++++++++ src/layers/flights/policy.js | 31 ++++++++++++++ src/layers/flights/rendering.js | 24 +++++++++++ src/layers/flights/state.js | 4 ++ 7 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 src/layers/flights/hiddenContactRotation.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58103d04d..4cc44d40c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +- Stop dead-reckoning every flight hidden behind the horizon on every fleet + tick. With the worldwide feed (~13k contacts, ~56 on screen over Colombia at + 4 km), the fleet pass drops from 7.7 ms to 2.0 ms per frame and its longest + tick from 51 ms to about 16 ms; the scene goes from 43 to 58 FPS. Hidden + contacts re-reckon about once a second, staggered, and appear as soon as + their position is in view. + - Public Overpass instances are no longer used by default. Street Traffic roads come from TomTom flow tiles, OpenFreeMap vector tiles, or both, chosen on the layer row (TomTom / OSM / Hybrid) or with `?trafficRoads=`. With a diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 22ce1eb3b..ee4269a63 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3469,7 +3469,7 @@ silently demoting every later lookup for the session. ### Motion & Symbology Correctness (June 10, 2026) -- **Flights (commercial + military)** render one poll interval behind real time (30s/15s) and interpolate between two known feed-stamped fixes (OpenSky `time_position`; adsb.lol `receipt − seen_pos`). The display latency is an intentional product decision — do not "fix" it away. The whole fleet dead-reckons at ~12Hz (1m² write gating); aircraft get a 3-poll grace period (faded icon) before removal. When a position epoch pauses, both layers coast for at least 60 seconds of contact grace with an absolute five-minute ceiling. Source backoff marks each contact and the cockpit `STALE`; the cockpit then holds the exact layer position instead of continuing inertial flight. Repeated-position kinematic changes create a forward-only synthetic fix rather than mutating history, and grounded history is lifted only when no owned 3D model already controls its datum. A nominally successful worldwide OpenSky response whose own snapshot epoch is more than two minutes old prefers the existing 250-nm viewport-scoped adsb.lol fallback and labels the source/coverage accordingly when that upstream is available. If the fallback is also unavailable, the layer's freshness/error fields use the source epoch—never the cache receipt time—so the UI reports an old snapshot rather than “just now.” +- **Flights (commercial + military)** render one poll interval behind real time (30s/15s) and interpolate between two known feed-stamped fixes (OpenSky `time_position`; adsb.lol `receipt − seen_pos`). The display latency is an intentional product decision — do not "fix" it away. The fleet dead-reckons at ~12Hz (1m² write gating); a contact already hidden behind the horizon whose last position still is re-reckons on a staggered one-in-12-ticks rotation (about once a second) instead, so the worldwide feed's ~13k contacts no longer cost a full pass when well under 1% are on screen. Anything that position puts in view, and any contact with a fleet model, takes the full pass immediately, and hidden-inclusive proximity queries read positions at most about a second old; aircraft get a 3-poll grace period (faded icon) before removal. When a position epoch pauses, both layers coast for at least 60 seconds of contact grace with an absolute five-minute ceiling. Source backoff marks each contact and the cockpit `STALE`; the cockpit then holds the exact layer position instead of continuing inertial flight. Repeated-position kinematic changes create a forward-only synthetic fix rather than mutating history, and grounded history is lifted only when no owned 3D model already controls its datum. A nominally successful worldwide OpenSky response whose own snapshot epoch is more than two minutes old prefers the existing 250-nm viewport-scoped adsb.lol fallback and labels the source/coverage accordingly when that upstream is available. If the fallback is also unavailable, the layer's freshness/error fields use the source epoch—never the cache receipt time—so the UI reports an old snapshot rather than “just now.” - **World-space headings at every angle** (`src/data/iconOrientation.js`): aircraft/vessel icon rotation uses the camera right/up basis per tick (alignedAxis always ZERO), which is exact at screen center and for orthographic/nadir views and remains stable through >180° tracked orbits. Perspective rays vary across the viewport, so off-center contacts at oblique pitch can diverge from an exact finite-difference window projection; a regression test pins that known regime, and field evidence decides whether to adopt exact projection with the basis method as fallback. Fleet rotations refresh on camera-pose change; tracked entities per frame. Billboards are horizon-culled via a shared EllipsoidalOccluder. - **Military/OpenSky reconciliation** (`src/data/militaryRegistry.js`): known-military ICAOs render amber in the flights layer (60s self-poll of the cached mil endpoint when the military layer is off) and are suppressed there while the military layer renders them. - **Satellites**: 838-sat core catalog (stations/visual/GPS/GLONASS/Galileo/GEO), tracking lands ~726km out via `viewFrom` (tracked entity owns a point graphic so the tracking camera engages), rings realign via primitive modelMatrix (no per-second rebuild flicker), optional `setParams({catalog:'dense'})` Starlink mode. diff --git a/scripts/format-scope.json b/scripts/format-scope.json index cd836a5cb..a12d5124f 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -308,6 +308,7 @@ "src/layers/flights/tracking.js", "src/layers/flights/rendering.js", "src/layers/flights/testing.js", + "src/layers/flights/hiddenContactRotation.test.mjs", "src/layers/flights/evidence.js", "src/layers/flights/ingestion.js", "src/layers/flights/queries.js", diff --git a/src/layers/flights/hiddenContactRotation.test.mjs b/src/layers/flights/hiddenContactRotation.test.mjs new file mode 100644 index 000000000..23e81c45b --- /dev/null +++ b/src/layers/flights/hiddenContactRotation.test.mjs @@ -0,0 +1,42 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + FLEET_DR_INTERVAL_MS, + HIDDEN_CONTACT_DR_INTERVAL_MS, + HIDDEN_CONTACT_DR_TICKS, + hiddenContactDrPhase, +} from './policy.js'; + +// Contacts hidden behind the horizon re-reckon on one fleet tick per rotation +// instead of every tick. Their positions still feed hidden-inclusive proximity +// queries, so the rotation must reach every contact about once a second, and +// it must spread them across ticks so no single tick carries the whole fleet. + +const icao = (n) => n.toString(16).padStart(6, '0'); + +test('the rotation revisits each hidden contact about once a second', () => { + const periodMs = HIDDEN_CONTACT_DR_TICKS * FLEET_DR_INTERVAL_MS; + assert.ok(HIDDEN_CONTACT_DR_TICKS > 1); + assert.ok( + Math.abs(periodMs - HIDDEN_CONTACT_DR_INTERVAL_MS) <= FLEET_DR_INTERVAL_MS, + ); +}); + +test('a contact keeps the same slot, inside the rotation', () => { + for (let n = 0; n < 500; n++) { + const phase = hiddenContactDrPhase(icao(n * 7919)); + assert.ok(Number.isInteger(phase)); + assert.ok(phase >= 0 && phase < HIDDEN_CONTACT_DR_TICKS); + assert.equal(hiddenContactDrPhase(icao(n * 7919)), phase); + } +}); + +test('a worldwide fleet is spread evenly enough that no tick carries a burst', () => { + const fleet = 13000; + const perSlot = new Array(HIDDEN_CONTACT_DR_TICKS).fill(0); + // Real ICAO24 addresses cluster by national block; sample several blocks. + for (let n = 0; n < fleet; n++) + perSlot[hiddenContactDrPhase(icao(0x0a0000 + n * 53))]++; + const fair = fleet / HIDDEN_CONTACT_DR_TICKS; + for (const count of perSlot) assert.ok(count < fair * 1.5, `${count} > 1.5×`); +}); diff --git a/src/layers/flights/policy.js b/src/layers/flights/policy.js index f8358c98f..04313c367 100644 --- a/src/layers/flights/policy.js +++ b/src/layers/flights/policy.js @@ -200,6 +200,37 @@ export const FLEET_DR_INTERVAL_MS = 80; export const ROTATION_REFRESH_MS = 1000; +/** + * @constant {number} Dead-reckoning cadence (ms) for contacts hidden behind the + * horizon. The worldwide feed carries ~13k contacts and a zoomed-in view shows + * well under 1% of them; the rest still need positions for hidden-inclusive + * proximity queries, but ~1 s old (≈250 m for an airliner) is well inside the + * 30 s poll's own uncertainty. Each hidden contact is re-reckoned on one tick in + * every HIDDEN_CONTACT_DR_TICKS, staggered by id so no tick carries them all. + */ + +export const HIDDEN_CONTACT_DR_INTERVAL_MS = 1000; + +export const HIDDEN_CONTACT_DR_TICKS = Math.max( + 1, + Math.round(HIDDEN_CONTACT_DR_INTERVAL_MS / FLEET_DR_INTERVAL_MS), +); + +/** + * Stable slot in [0, HIDDEN_CONTACT_DR_TICKS) for a contact id, so each hidden + * contact re-reckons on the same tick of every rotation. + * @param {string} id Contact id (ICAO24). + * @returns {number} + */ + +export function hiddenContactDrPhase(id) { + const text = String(id); + let hash = 0; + for (let i = 0; i < text.length; i++) + hash = (hash * 31 + text.charCodeAt(i)) >>> 0; + return hash % HIDDEN_CONTACT_DR_TICKS; +} + /** Max course slew (deg/s) — well above real turns (≤4°/s), hides fix-boundary * steps. Scaled down toward COURSE_MIN_DPS at low speed (courseSlewCapDps). */ diff --git a/src/layers/flights/rendering.js b/src/layers/flights/rendering.js index 44e1b02c3..2048128dd 100644 --- a/src/layers/flights/rendering.js +++ b/src/layers/flights/rendering.js @@ -33,6 +33,8 @@ import { COURSE_SLEW_DT_MAX_SEC, ROTATION_REFRESH_MS, COURSE_MAX_DPS, + HIDDEN_CONTACT_DR_TICKS, + hiddenContactDrPhase, } from './policy.js'; export function createRendering({ @@ -811,6 +813,12 @@ export function createRendering({ } } + /** The contact's hidden-rotation slot, cached on its billboard. */ + function _hiddenDrPhase(bb, icao24) { + bb._gevHiddenDrPhase ??= hiddenContactDrPhase(icao24); + return bb._gevHiddenDrPhase; + } + function _fleetTick() { if ( !flightState._viewer || @@ -931,9 +939,25 @@ export function createRendering({ for (const icao of toRelease) _releaseModel(icao); } + const hiddenDrPhase = flightState._fleetTickSeq++ % HIDDEN_CONTACT_DR_TICKS; + for (const [icao24, bb] of flightState._billboards) { if (icao24 === flightState._trackedIcao) continue; // tracked entity owns its own motion + // A contact still behind the horizon at its last position would only be + // re-reckoned to be hidden again, so off its turn it keeps that position + // (see HIDDEN_CONTACT_DR_TICKS). Anything the stale position puts in view, + // and any contact with a fleet model, takes the full pass below at once. + if ( + !bb.show && + _hiddenDrPhase(bb, icao24) !== hiddenDrPhase && + !flightState._models.has(icao24) && + !occluder.isPointVisible( + flightState._cullPositions.get(icao24) || bb.position, + ) + ) + continue; + const info = flightState.records.data.get(icao24); const dr = parts.motion._deadReckon(icao24, flightState._scratchFleetPos); diff --git a/src/layers/flights/state.js b/src/layers/flights/state.js index 2978ceba4..d1adec064 100644 --- a/src/layers/flights/state.js +++ b/src/layers/flights/state.js @@ -204,6 +204,10 @@ export function createFlightState({ source, services }) { flightState._lastFleetTickMs = 0; + /** @type {number} Fleet ticks run; selects which hidden contacts re-reckon */ + + flightState._fleetTickSeq = 0; + /** @type {string} Camera pose signature at the last rotation pass */ flightState._lastCamPoseSig = ''; From aa745bcee5c4f76353c9426fb9055cb5737f0fc0 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:02:05 +0200 Subject: [PATCH 03/27] data: refresh Norway datacenters from Geofabrik extract Add a maintainer script that merges a pinned, MD5-verified Geofabrik Norway extract (osmium tags-filter telecom/building=data_center) into the bundled datacenter layer. Only features inside the Geofabrik Norway boundary are touched; unchanged features keep their original line. norway-261005 (OSM 2026-10-05) adds Troll Housing (w1551029155) and updates tags on four existing features. Provenance is recorded in the dataset README and the bundled count moves from 4,351 to 4,352. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + DATA_SOURCES.md | 2 +- README.md | 2 +- docs/INFRASTRUCTURE-LAYERS.md | 2 +- scripts/build-datacenters-norway.py | 196 ++++++++++++++++++ src/data/infrastructure.test.mjs | 2 +- src/data/local_data/datacenters/README.md | 16 +- .../datacenters/datacenters.geojsonl | 9 +- 8 files changed, 226 insertions(+), 9 deletions(-) create mode 100644 scripts/build-datacenters-norway.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..031e36751 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +- Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract + (2026-10-05). Troll Housing joins the layer (4,352 features) and four + Norwegian sites pick up current OSM tags; no site moved or was removed. + `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, + touching only features inside Geofabrik's Norway boundary, and the dataset + README records the extract, checksum and filter. - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 734e35a69..b4f0dc0cc 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -112,7 +112,7 @@ The [Bhote Koshi event pack](public/events/bhote-koshi-2026/README.md), under `p | Dataset | Folder | License | Commercial use? | Attribution | | --------------------------------------------------------------------------- | --------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | --------------------------------------------------------------------------- | -| **Datacenters** (4,351) | `datacenters/` | **ODbL 1.0** (OpenStreetMap extract) | ✅ (attribution + share-alike on data) | © OpenStreetMap contributors (shared credit) | +| **Datacenters** (4,352) | `datacenters/` | **ODbL 1.0** (OpenStreetMap extract) | ✅ (attribution + share-alike on data) | © OpenStreetMap contributors (shared credit) | | **Dams** (704) | `dams/` | **ODbL 1.0** (OpenInfraMap / OSM extract) | ✅ (attribution + share-alike on data) | © OpenStreetMap contributors (shared credit) + Open Infrastructure Map | | **Military area names** (36,466) | `osm_military_names/` | **ODbL 1.0** (OpenStreetMap, via Overture Maps) | ✅ (attribution + share-alike on data) | © OpenStreetMap contributors (shared credit) + Overture Maps Foundation | | **TeleGeography Submarine Cable Map** (712 cables + 1,917 landing points) | `telegeography_submarine_cables/` | **CC BY-NC-SA 3.0** | ❌ **NonCommercial — remove for commercial use** | "© TeleGeography — submarinecablemap.com" | diff --git a/README.md b/README.md index 724ea18cb..0500645af 100644 --- a/README.md +++ b/README.md @@ -323,7 +323,7 @@ In the app, **Data Layers** groups them as Movement, Cameras, Infrastructure, Ev _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED ESTIMATE`, scrubbable 0.25×–4×._ -**Also on the globe:** neighborhood overlays · an optional cockpit WX cloud effect. **Bundled static infrastructure:** Datacenters (4,351), Dams (704), and Submarine Cables (712). +**Also on the globe:** neighborhood overlays · an optional cockpit WX cloud effect. **Bundled static infrastructure:** Datacenters (4,352), Dams (704), and Submarine Cables (712). ![Diving into the Bahamas and revealing labeled submarine cable routes beneath the globe](docs/media/09-undersea-cables.gif) diff --git a/docs/INFRASTRUCTURE-LAYERS.md b/docs/INFRASTRUCTURE-LAYERS.md index 76567e605..1166a9e69 100644 --- a/docs/INFRASTRUCTURE-LAYERS.md +++ b/docs/INFRASTRUCTURE-LAYERS.md @@ -58,6 +58,6 @@ factory and supplies its existing functions. Other consumers should use the pack exports, which do not import standalone application globals. Dataset files and source/license notices remain under `src/data/local_data/`. -Their input counts are 4,351 datacenter features and 704 dam features. Cesium may +Their input counts are 4,352 datacenter features and 704 dam features. Cesium may expand multipart geometries into multiple entities; entity and stem counts are different measurements. This change does not refresh or relicense the datasets. diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py new file mode 100644 index 000000000..01a891a8b --- /dev/null +++ b/scripts/build-datacenters-norway.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Maintainer-only Norway refresh of the bundled datacenter extract (osmium-tool). + +Downloads a pinned Geofabrik Norway extract, verifies its MD5, filters +telecom=data_center / building=data_center with osmium, and merges the result +into datacenters.geojsonl: features inside the Geofabrik Norway boundary are +added, updated or removed; every other line is kept byte-for-byte. +""" +import argparse +import hashlib +import json +import math +from pathlib import Path +import shutil +import subprocess +import urllib.request + +ROOT = Path(__file__).resolve().parent.parent +EXTRACT = 'norway-261005' +GEOFABRIK = 'https://download.geofabrik.de/europe/' +DATASET = ROOT / 'src/data/local_data/datacenters/datacenters.geojsonl' +CONTACT_TAGS = {'email', 'phone', 'fax', 'mobile', 'whatsapp', 'contact:email', 'contact:phone', + 'contact:fax', 'contact:mobile', 'contact:whatsapp'} +SAME_GEOMETRY_METERS = 0.05 + + +def fetch(url, target): + if not target.exists(): + with urllib.request.urlopen(url) as response, open(str(target) + '.part', 'wb') as out: + shutil.copyfileobj(response, out) + Path(str(target) + '.part').rename(target) + return target + + +def md5(path): + digest = hashlib.md5() + with open(path, 'rb') as handle: + for block in iter(lambda: handle.read(1 << 20), b''): + digest.update(block) + return digest.hexdigest() + + +def read_poly(path): + """Geofabrik .poly -> list of (ring, is_hole).""" + rings, ring, hole = [], None, False + for raw in path.read_text().splitlines()[1:]: + line = raw.strip() + if ring is None: + if line and line != 'END': + ring, hole = [], line.startswith('!') + elif line == 'END': + rings.append((ring, hole)) + ring = None + elif line: + lon, lat = (float(v) for v in line.split()) + ring.append((lon, lat)) + return rings + + +def inside(rings, lon, lat): + def in_ring(ring): + hit = False + for (x1, y1), (x2, y2) in zip(ring, ring[1:] + ring[:1]): + if (y1 > lat) != (y2 > lat) and lon < x1 + (lat - y1) * (x2 - x1) / (y2 - y1): + hit = not hit + return hit + return any(in_ring(r) for r, h in rings if not h) and not any(in_ring(r) for r, h in rings if h) + + +def points(coordinates): + if isinstance(coordinates[0], (int, float)): + return [coordinates] + return [p for part in coordinates for p in points(part)] + + +def osm_key(feature): + osm_id = feature['properties']['osm_id'] + if osm_id < 0: + return 'r%d' % -osm_id + return ('n' if feature['geometry']['type'] == 'Point' else 'w') + str(osm_id) + + +def export(osmium, pbf, cache): + filtered = cache / (EXTRACT + '-datacenters.osm.pbf') + exported = cache / (EXTRACT + '-datacenters.geojsonseq') + subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf), + 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) + subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id', + '-o', str(exported), str(filtered)], check=True) + fresh = {} + for line in exported.read_text().splitlines(): + line = line.lstrip('\x1e').strip() + if not line: + continue + feature = json.loads(line) + tags = {k: v for k, v in feature['properties'].items() if k not in CONTACT_TAGS} + if tags.get('telecom') != 'data_center' and tags.get('building') != 'data_center': + continue # untagged member ways exported only for geometry + key = feature['id'] + geometry = feature['geometry'] + if key[0] == 'a': # osmium area id: way * 2 or relation * 2 + 1 + number = int(key[1:]) + key = ('w%d' % (number // 2)) if number % 2 == 0 else ('r%d' % (number // 2)) + if key[0] == 'r': + tags = {**tags, 'type': 'multipolygon'} + if geometry['type'] == 'MultiPolygon' and len(geometry['coordinates']) == 1: + geometry = {'type': 'Polygon', 'coordinates': geometry['coordinates'][0]} + elif key in fresh or geometry['type'] == 'LineString': + continue # closed ways are taken from their area export + fresh[key] = {'geometry': geometry, 'tags': tags} + return fresh + + +def same_geometry(old, new): + """Same type and vertex set within tolerance; ring start and winding may differ.""" + if old['type'] != new['type']: + return False + a, b = points(old['coordinates']), points(new['coordinates']) + if len(a) != len(b): + return False + + def meters(p, q): + dx = (p[0] - q[0]) * 111320 * math.cos(math.radians(p[1])) + return math.hypot(dx, (p[1] - q[1]) * 110540) + return all(min(meters(p, q) for q in b) <= SAME_GEOMETRY_METERS for p in a) and \ + all(min(meters(q, p) for p in a) <= SAME_GEOMETRY_METERS for q in b) + + +def feature_line(key, sequence, fresh): + osm_id = int(key[1:]) * (-1 if key[0] == 'r' else 1) + feature = {'id': osm_id, 'type': 'Feature', 'geometry': fresh['geometry'], + 'properties': {'id': sequence, 'tags': fresh['tags'], 'type': 'data_center', + 'osm_id': osm_id}} + return json.dumps(feature, separators=(',', ':'), ensure_ascii=False) + + +def merge(lines, fresh, rings): + out, seen, sequence = [], set(), 0 + report = {'kept': 0, 'updated': [], 'removed': [], 'added': []} + for line in lines: + feature = json.loads(line) + sequence = max(sequence, feature['properties']['id']) + key = osm_key(feature) + if key in fresh: + seen.add(key) + old_tags = feature['properties']['tags'] + if old_tags == fresh[key]['tags'] and same_geometry(feature['geometry'], fresh[key]['geometry']): + out.append(line) + report['kept'] += 1 + else: + out.append(feature_line(key, feature['properties']['id'], fresh[key])) + report['updated'].append(key) + continue + lon, lat = points(feature['geometry']['coordinates'])[0][:2] + if inside(rings, lon, lat): + report['removed'].append(key) + continue + out.append(line) + for key in sorted(set(fresh) - seen, key=lambda k: (k[0], int(k[1:]))): + sequence += 1 + out.append(feature_line(key, sequence, fresh[key])) + report['added'].append(key) + if len({osm_key(json.loads(line)) for line in out}) != len(out): + raise ValueError('Duplicate OSM identity') + return out, report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cache', type=Path, required=True) + parser.add_argument('--osmium', default=shutil.which('osmium') or 'osmium') + parser.add_argument('--verify', action='store_true') + args = parser.parse_args() + args.cache.mkdir(parents=True, exist_ok=True) + pbf = fetch(GEOFABRIK + EXTRACT + '.osm.pbf', args.cache / (EXTRACT + '.osm.pbf')) + expected = fetch(GEOFABRIK + EXTRACT + '.osm.pbf.md5', + args.cache / (EXTRACT + '.osm.pbf.md5')).read_text().split()[0] + actual = md5(pbf) + if actual != expected: + raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected)) + rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly')) + fresh = export(args.osmium, pbf, args.cache) + current = [line for line in DATASET.read_text().splitlines() if line.strip()] + merged, report = merge(current, fresh, rings) + payload = '\n'.join(merged) + '\n' + if args.verify: + if payload != DATASET.read_text(): + raise SystemExit('datacenters.geojsonl is not the %s merge' % EXTRACT) + else: + DATASET.write_text(payload) + print(json.dumps({'extract': EXTRACT, 'md5': actual, 'norway': len(fresh), + 'features': len(merged), **report})) + + +if __name__ == '__main__': + main() diff --git a/src/data/infrastructure.test.mjs b/src/data/infrastructure.test.mjs index 45fb3d317..c7f1b5a23 100644 --- a/src/data/infrastructure.test.mjs +++ b/src/data/infrastructure.test.mjs @@ -82,7 +82,7 @@ test('infrastructure factory preserves identity and creates independent state wi test('dataset URLs still name the complete bundled sources', () => { for (const [file, count] of [ - ['datacenters', 4351], + ['datacenters', 4352], ['dams', 704], ]) { const lines = readFileSync( diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index 454a15007..ef0720ae4 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -5,7 +5,7 @@ layer. - Source: OpenStreetMap contributors - License: Open Database License (ODbL) 1.0 -- Feature count: 4,351 +- Feature count: 4,352 - Runtime file: `datacenters.geojsonl` The public-release snapshot removes contact-oriented tags such as email, phone, @@ -16,3 +16,17 @@ ODbL 1.0 with the required OpenStreetMap attribution. The original extraction date and query were not recorded alongside this snapshot. Future refreshes should record both before replacing the file. + +## Norway refresh + +Features inside the Geofabrik Norway boundary (`norway.poly`, which includes +Svalbard) were refreshed separately, so this file mixes two vintages: Norway +reflects OSM as of the extract below, everything else the original snapshot. + +- Extract: Geofabrik `norway-261005.osm.pbf`, OSM data + 2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0` +- Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` +- Contact tags are dropped as above; unchanged features keep their original + line, so geometry is only rewritten when tags or vertices actually changed. +- Rebuild or check: `python3 scripts/build-datacenters-norway.py --cache ` + (add `--verify` to check without writing; requires `osmium-tool`) diff --git a/src/data/local_data/datacenters/datacenters.geojsonl b/src/data/local_data/datacenters/datacenters.geojsonl index f7f5fc89f..b02bd57b3 100644 --- a/src/data/local_data/datacenters/datacenters.geojsonl +++ b/src/data/local_data/datacenters/datacenters.geojsonl @@ -3770,15 +3770,15 @@ {"id":939340882,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[21.021364183,52.315373019],[21.02232433,52.315435883],[21.022363641,52.315209656],[21.021403578,52.315146875],[21.021364183,52.315373019]]]},"properties":{"id":11407,"tags":{"name":"T-Mobile Warszawa-Szlachecka","telecom":"data_center","building":"office","building:levels":"3"},"type":"data_center","osm_id":939340882}} {"id":1005973518,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[21.021325375,52.315595978],[21.022285857,52.315658423],[21.02232433,52.315435883],[21.021364183,52.315373019],[21.021325375,52.315595978]]]},"properties":{"id":11976,"tags":{"telecom":"data_center","building":"office","building:levels":"3"},"type":"data_center","osm_id":1005973518}} {"id":1156823776,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[21.115130857,52.235385942],[21.115189866,52.236349777],[21.115232027,52.236667954],[21.115818174,52.237174556],[21.116175578,52.237468677],[21.117262124,52.236985628],[21.11728157,52.236976576],[21.117262795,52.236960985],[21.117340495,52.236926033],[21.117360276,52.2369433],[21.117447532,52.236904491],[21.11804097,52.236654459],[21.118030996,52.236645742],[21.118041054,52.236641719],[21.116603055,52.235624072],[21.116518984,52.235676375],[21.115986231,52.235696659],[21.115954966,52.235361551],[21.115130857,52.235385942]]]},"properties":{"id":15342,"tags":{"name":"Atman Data Center Warsaw-1","telecom":"data_center","website":"https://www.atman.pl"},"type":"data_center","osm_id":1156823776}} -{"id":7405586142,"type":"Feature","geometry":{"type":"Point","coordinates":[5.598350034,58.860111293]},"properties":{"id":24849,"tags":{"name":"Maan Group","telecom":"data_center","operator":"Privat"},"type":"data_center","osm_id":7405586142}} +{"id":7405586142,"type":"Feature","geometry":{"type":"Point","coordinates":[5.5983501,58.8601113]},"properties":{"id":24849,"tags":{"name":"Maan Group","opening_hours":"24/7","operator":"Privat","telecom":"data_center"},"type":"data_center","osm_id":7405586142}} {"id":481615983,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[9.572510149,56.481777951],[9.574069938,56.484921751],[9.57465357,56.48483349],[9.574639572,56.484805243],[9.574700676,56.484795939],[9.574714925,56.484824857],[9.575298557,56.484736595],[9.57454888,56.483225422],[9.574609229,56.483216369],[9.574510155,56.483016629],[9.574449722,56.483025849],[9.573738769,56.481592795],[9.573155137,56.481680972],[9.573170643,56.481712321],[9.573109539,56.481721625],[9.573093781,56.48168969],[9.572510149,56.481777951]]]},"properties":{"id":7881,"tags":{"telecom":"data_center","building":"data_center","operator":"Apple","wikidata":"Q19721053","wikipedia":"en:Foulum Data Center","description":"Apple Datacenter"},"type":"data_center","osm_id":481615983}} {"id":618756435,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.500360092,57.318668738],[10.500399236,57.318671672],[10.500382891,57.318734871],[10.500552038,57.318744175],[10.500545249,57.318787677],[10.500501495,57.318785666],[10.500458999,57.318957746],[10.500970295,57.31899647],[10.501009187,57.318847189],[10.501084959,57.318852888],[10.50111966,57.318719197],[10.500843896,57.318698159],[10.500865521,57.318615178],[10.500859235,57.318614759],[10.500869461,57.318575699],[10.500954537,57.318582153],[10.501022347,57.31832223],[10.500736691,57.318300437],[10.500669133,57.318559689],[10.500674162,57.318560025],[10.500664272,57.318598498],[10.500383981,57.318577124],[10.500360092,57.318668738]]]},"properties":{"id":8842,"tags":{"ref":"AAL01-01","name":"Prime Data Centers - Saeby Campus AAL01","telecom":"data_center","website":"https://primedatacenters.com/locations/saeby/","building":"industrial","operator":"Prime Data Centers"},"type":"data_center","osm_id":618756435}} -{"id":-14367602,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[11.002243225,59.633471864],[11.003903261,59.633446551],[11.007046726,59.633399696],[11.006983443,59.632420941],[11.006982772,59.63227057],[11.006911023,59.632265289],[11.006852936,59.632253973],[11.006765848,59.632228576],[11.00671782,59.632205275],[11.006119687,59.632213573],[11.006119352,59.632202592],[11.006118933,59.63219245],[11.005930424,59.632194378],[11.005923635,59.632120869],[11.005980799,59.632120031],[11.005979542,59.632104524],[11.005977027,59.63206865],[11.00597225,59.631998242],[11.005412171,59.632035541],[11.005286945,59.632043839],[11.00527848,59.632044342],[11.003707962,59.632147272],[11.003107483,59.63219178],[11.002741864,59.632235533],[11.002729375,59.63223788],[11.002345987,59.632309294],[11.002308352,59.632316251],[11.002302066,59.632477351],[11.002277088,59.632913881],[11.002243225,59.633471864]]]},"properties":{"id":107,"tags":{"name":"STACK Infrastructure OSL04","type":"multipolygon","telecom":"data_center","website":"https://www.stackinfra.com","operator":"Stack Infrastructure","operator:wikidata":"Q115477173"},"type":"data_center","osm_id":-14367602}} +{"id":-14367602,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[11.0022433,59.6334719],[11.0022771,59.6329139],[11.0023021,59.6324774],[11.0023084,59.6323163],[11.002346,59.6323093],[11.0027294,59.6322379],[11.0027419,59.6322356],[11.0031075,59.6321918],[11.003708,59.6321473],[11.0052785,59.6320444],[11.005287,59.6320439],[11.0054122,59.6320356],[11.0059723,59.6319983],[11.0059771,59.6320687],[11.0059796,59.6321046],[11.0059808,59.6321201],[11.0059237,59.6321209],[11.0059305,59.6321944],[11.006119,59.6321925],[11.0061194,59.6322026],[11.0061197,59.6322136],[11.0067179,59.6322053],[11.0067659,59.6322286],[11.006853,59.632254],[11.0069111,59.6322653],[11.0069828,59.6322706],[11.0069835,59.632421],[11.0070468,59.6333997],[11.0039033,59.6334466],[11.0022433,59.6334719]]]},"properties":{"id":107,"tags":{"landuse":"industrial","name":"STACK Infrastructure OSL04","operator":"Stack Infrastructure","operator:wikidata":"Q115477173","telecom":"data_center","website":"https://www.stackinfra.com","type":"multipolygon"},"type":"data_center","osm_id":-14367602}} {"id":1028687251,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[11.004111886,59.632578437],[11.004142983,59.633037262],[11.004275166,59.633034999],[11.004274998,59.63303173],[11.00505133,59.633019157],[11.005056443,59.633099372],[11.005224416,59.63309669],[11.005190637,59.632562344],[11.005060885,59.632564439],[11.005060969,59.632567457],[11.004587978,59.632575419],[11.00458538,59.632536527],[11.004459819,59.632538623],[11.004462418,59.632576844],[11.004242476,59.6325807],[11.004242141,59.632576174],[11.004111886,59.632578437]]]},"properties":{"id":12226,"tags":{"building":"data_center","ref:bygningsnr":"300783262"},"type":"data_center","osm_id":1028687251}} {"id":1164840308,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[11.002600126,59.632599392],[11.002631223,59.633058217],[11.00276349,59.633055954],[11.002763238,59.633052685],[11.00353957,59.633040196],[11.003544683,59.633120327],[11.00371274,59.633117645],[11.003678961,59.632583382],[11.003549042,59.632585478],[11.003549293,59.632588495],[11.003076135,59.632596458],[11.00307362,59.632557566],[11.002948143,59.632559661],[11.002950657,59.632597883],[11.002730716,59.632601571],[11.002730465,59.632597129],[11.002600126,59.632599392]]]},"properties":{"id":15459,"tags":{"building":"data_center","ref:bygningsnr":"300783262"},"type":"data_center","osm_id":1164840308}} {"id":1164840309,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[11.005613672,59.632723444],[11.005644769,59.633182269],[11.005776951,59.633179922],[11.005776784,59.633176737],[11.006553199,59.633164164],[11.006558229,59.633244463],[11.006726286,59.633241781],[11.006692423,59.632707351],[11.006562671,59.632709446],[11.006562839,59.632712464],[11.006089764,59.632720426],[11.006087166,59.632681534],[11.005961772,59.632683797],[11.005964287,59.632721851],[11.005744262,59.632725623],[11.005743927,59.632721181],[11.005613672,59.632723444]]]},"properties":{"id":15460,"tags":{"building":"data_center","ref:bygningsnr":"300783262"},"type":"data_center","osm_id":1164840309}} -{"id":935092287,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.991467366,59.759024056],[10.992780978,59.759406942],[10.992773937,59.759423957],[10.992948197,59.759474248],[10.993073842,59.759363859],[10.993022293,59.759348939],[10.993291688,59.759112653],[10.993128827,59.759065547],[10.993181885,59.759019027],[10.993083062,59.758990445],[10.992976947,59.759083484],[10.992845519,59.759045346],[10.992813584,59.759073342],[10.9927876,59.759065798],[10.993409789,59.758524243],[10.99263002,59.758297094],[10.991938848,59.758898663],[10.991693594,59.758827165],[10.991467366,59.759024056]]]},"properties":{"id":11377,"tags":{"name":"Green Mountain Ytre Enebakk","telecom":"data_center","website":"https://greenmountain.no/","building":"warehouse","ref:bygningsnr":"300752944"},"type":"data_center","osm_id":935092287}} -{"id":1161764012,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.992826324,59.757448761],[10.994530281,59.757601396],[10.994634888,59.757303168],[10.992928332,59.757152545],[10.992826324,59.757448761]]]},"properties":{"id":15426,"tags":{"name":"Green Mountain Ytre Enebakk","telecom":"data_center","website":"https://greenmountain.no/","building":"industrial"},"type":"data_center","osm_id":1161764012}} +{"id":935092287,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.9914674,59.7590241],[10.9916936,59.7588272],[10.9919389,59.7588987],[10.9926301,59.7582971],[10.9934098,59.7585243],[10.9927876,59.7590658],[10.9928136,59.7590734],[10.9928456,59.7590454],[10.992977,59.7590835],[10.9930831,59.7589905],[10.9931819,59.7590191],[10.9931289,59.7590656],[10.9932917,59.7591127],[10.9930223,59.759349],[10.9930739,59.7593639],[10.9929482,59.7594743],[10.992774,59.759424],[10.992781,59.759407],[10.9914674,59.7590241]]]},"properties":{"id":11377,"tags":{"building":"warehouse","name":"Green Mountain Ytre Enebakk","ref:bygningsnr":"300752944","source":"Norway Orthophoto 2024;https://greenmountain.no/data-center/osl1-enebakk/","telecom":"data_center","website":"https://greenmountain.no/"},"type":"data_center","osm_id":935092287}} +{"id":1161764012,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.9928264,59.7574488],[10.9929284,59.7571526],[10.9946349,59.7573032],[10.9945303,59.7576014],[10.9928264,59.7574488]]]},"properties":{"id":15426,"tags":{"building":"industrial","name":"Green Mountain Ytre Enebakk","source":"Norway Orthophoto 2024;https://greenmountain.no/data-center/osl1-enebakk/","telecom":"data_center","website":"https://greenmountain.no/"},"type":"data_center","osm_id":1161764012}} {"id":30724580,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.808790995,59.924984985],[10.808893925,59.925031923],[10.80893793,59.92500594],[10.808982941,59.925025972],[10.809042955,59.92501298],[10.809083943,59.925031923],[10.809012948,59.925077856],[10.80926583,59.925189755],[10.80955115,59.925025972],[10.809676543,59.925081293],[10.81032899,59.924701928],[10.809890365,59.924508138],[10.80982155,59.924515347],[10.809691631,59.924457847],[10.808820667,59.924557592],[10.809099365,59.924699581],[10.808806418,59.924865291],[10.808918149,59.924914828],[10.808875988,59.924937962],[10.808790995,59.924984985]]]},"properties":{"id":261,"tags":{"name":"Stack Infrastructure OSLO1A","telecom":"data_center","building":"office","operator":"Stack Infrastructure","ref:bygningsnr":"80877293","building:levels":"4","operator:wikidata":"Q115477173"},"type":"data_center","osm_id":30724580}} {"id":80871917,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[10.83408892,59.938588981],[10.834305927,59.938720996],[10.834575992,59.938720996],[10.834574987,59.938736],[10.834581943,59.938736922],[10.834579932,59.938771958],[10.834736925,59.938771958],[10.834734997,59.93893792],[10.836717988,59.938944961],[10.836717988,59.938983937],[10.836817984,59.938983937],[10.83681899,59.938944961],[10.836844974,59.938944961],[10.836846985,59.938774976],[10.836841956,59.938774976],[10.836842962,59.938744968],[10.836849919,59.938744968],[10.836851931,59.938653941],[10.836683957,59.938652935],[10.836684125,59.938640278],[10.836686975,59.938414973],[10.83490297,59.938406926],[10.834899953,59.938646984],[10.834812949,59.938645978],[10.83481496,59.938551933],[10.834579932,59.938550927],[10.834580938,59.938564925],[10.834567946,59.938564925],[10.834567946,59.938551933],[10.834089926,59.938549922],[10.83408892,59.938588981]]]},"properties":{"id":1464,"tags":{"name":"Bulk Data Centers Oslo Internet Exchange","telecom":"data_center","website":"https://bulkinfrastructure.com/data-centers/locations/os-ix","building":"industrial","ref:bygningsnr":"80709943"},"type":"data_center","osm_id":80871917}} {"id":12158409495,"type":"Feature","geometry":{"type":"Point","coordinates":[9.689363435,62.59439936]},"properties":{"id":28723,"tags":{"name":"VisBook AS","telecom":"data_center","website":"https://visbook.com/"},"type":"data_center","osm_id":12158409495}} @@ -4349,3 +4349,4 @@ {"id":540895898,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[101.71747909,56.303855963],[101.71782015,56.304420065],[101.721113819,56.303806929],[101.720772759,56.303242994],[101.71747909,56.303855963]]]},"properties":{"id":8271,"tags":{"name":"Битривер Рус","name:en":"Bitriver Rus","name:ru":"Битривер Рус","telecom":"data_center","website":"https://bitriver.farm/","building":"data_center"},"type":"data_center","osm_id":540895898}} {"id":1082803064,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[123.948021193,47.340264299],[123.948354625,47.340268239],[123.948490663,47.340269831],[123.948488568,47.340349292],[123.948809762,47.340353064],[123.948811942,47.340267485],[123.948918643,47.340268658],[123.948922918,47.340137817],[123.948023959,47.340126082],[123.948021193,47.340264299]]]},"properties":{"id":13377,"tags":{"name":"联通云数据中心·齐齐哈尔","name:en":"Unicom Data Center of Qiqihar","name:zh":"联通云数据中心·齐齐哈尔","telecom":"data_center","building":"yes","name:zh-Hans":"联通云数据中心·齐齐哈尔","name:zh-Hant":"聯通雲數據中心·齊齊哈爾"},"type":"data_center","osm_id":1082803064}} {"id":442479059,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[142.76019226,46.910096898],[142.760192427,46.910325724],[142.760447656,46.910328155],[142.760460397,46.910102263],[142.76019226,46.910096898]]]},"properties":{"id":7471,"tags":{"telecom":"data_center","building":"yes"},"type":"data_center","osm_id":442479059}} +{"id":1551029155,"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[7.1069147,62.8627625],[7.1071207,62.8621617],[7.1075917,62.8616616],[7.1088647,62.8620476],[7.1106308,62.862645],[7.1098655,62.8632827],[7.1089604,62.8630042],[7.1086734,62.8632156],[7.1069147,62.8627625]]]},"properties":{"id":29471,"tags":{"layer":"-1","loc_name":"Troll Housing Klempertåsen","name":"Troll Housing","operator":"Troll Housing AS","telecom":"data_center","website":"https://www.trollhousing.no/"},"type":"data_center","osm_id":1551029155}} From 5b5d942d95dddd56b3bacb465c69098913688d1f Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:23:02 +0200 Subject: [PATCH 04/27] fix(data): make the Norway datacenter script portable and re-runnable Read and write files as UTF-8 with LF line endings so the script and --verify work on Windows, where the default codec is cp1252 (thanks to quanticsoul4772 for the report). Geofabrik keeps only its newest dated extract, so --extract now defaults to the current one and accepts a dated name for --verify while it is listed; the dataset README says so. Downloads time out instead of hanging. --- CHANGELOG.md | 1 + scripts/build-datacenters-norway.py | 52 +++++++++++++++-------- src/data/local_data/datacenters/README.md | 8 +++- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20a9da4e6..067ca16b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -184,6 +184,7 @@ `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, touching only features inside Geofabrik's Norway boundary, and the dataset README records the extract, checksum and filter. + - Add a keyless Air quality layer to the Weather group, with Environment and Climate Change Canada's AQHI as the first registered provider. Readings carry `{ provider, scale, value, band }` so another network can be added without diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py index 01a891a8b..687395f7b 100644 --- a/scripts/build-datacenters-norway.py +++ b/scripts/build-datacenters-norway.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 """Maintainer-only Norway refresh of the bundled datacenter extract (osmium-tool). -Downloads a pinned Geofabrik Norway extract, verifies its MD5, filters +Downloads a dated Geofabrik Norway extract, verifies its MD5, filters telecom=data_center / building=data_center with osmium, and merges the result into datacenters.geojsonl: features inside the Geofabrik Norway boundary are added, updated or removed; every other line is kept byte-for-byte. + +Geofabrik keeps only its newest dated extract, so --extract defaults to the +current one and a recorded extract can be re-verified only while it is listed. """ import argparse import hashlib @@ -16,17 +19,28 @@ import subprocess import urllib.request ROOT = Path(__file__).resolve().parent.parent -EXTRACT = 'norway-261005' GEOFABRIK = 'https://download.geofabrik.de/europe/' DATASET = ROOT / 'src/data/local_data/datacenters/datacenters.geojsonl' CONTACT_TAGS = {'email', 'phone', 'fax', 'mobile', 'whatsapp', 'contact:email', 'contact:phone', 'contact:fax', 'contact:mobile', 'contact:whatsapp'} SAME_GEOMETRY_METERS = 0.05 +TIMEOUT_SECONDS = 60 + + +def latest_extract(): + """Resolve norway-latest to the dated name Geofabrik redirects it to.""" + request = urllib.request.Request(GEOFABRIK + 'norway-latest.osm.pbf', method='HEAD') + with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response: + name = response.geturl().rsplit('/', 1)[-1] + if not name.startswith('norway-') or not name.endswith('.osm.pbf') or name == 'norway-latest.osm.pbf': + raise RuntimeError('Unexpected Geofabrik latest target: %s' % name) + return name[:-len('.osm.pbf')] def fetch(url, target): if not target.exists(): - with urllib.request.urlopen(url) as response, open(str(target) + '.part', 'wb') as out: + with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response, \ + open(str(target) + '.part', 'wb') as out: shutil.copyfileobj(response, out) Path(str(target) + '.part').rename(target) return target @@ -43,7 +57,7 @@ def md5(path): def read_poly(path): """Geofabrik .poly -> list of (ring, is_hole).""" rings, ring, hole = [], None, False - for raw in path.read_text().splitlines()[1:]: + for raw in path.read_text(encoding='utf-8').splitlines()[1:]: line = raw.strip() if ring is None: if line and line != 'END': @@ -80,15 +94,15 @@ def osm_key(feature): return ('n' if feature['geometry']['type'] == 'Point' else 'w') + str(osm_id) -def export(osmium, pbf, cache): - filtered = cache / (EXTRACT + '-datacenters.osm.pbf') - exported = cache / (EXTRACT + '-datacenters.geojsonseq') +def export(osmium, pbf, cache, extract): + filtered = cache / (extract + '-datacenters.osm.pbf') + exported = cache / (extract + '-datacenters.geojsonseq') subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf), 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id', '-o', str(exported), str(filtered)], check=True) fresh = {} - for line in exported.read_text().splitlines(): + for line in exported.read_text(encoding='utf-8').splitlines(): line = line.lstrip('\x1e').strip() if not line: continue @@ -169,26 +183,30 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--cache', type=Path, required=True) parser.add_argument('--osmium', default=shutil.which('osmium') or 'osmium') + parser.add_argument('--extract', default='latest', + help="dated Geofabrik name such as norway-261005, or 'latest'") parser.add_argument('--verify', action='store_true') args = parser.parse_args() args.cache.mkdir(parents=True, exist_ok=True) - pbf = fetch(GEOFABRIK + EXTRACT + '.osm.pbf', args.cache / (EXTRACT + '.osm.pbf')) - expected = fetch(GEOFABRIK + EXTRACT + '.osm.pbf.md5', - args.cache / (EXTRACT + '.osm.pbf.md5')).read_text().split()[0] + extract = latest_extract() if args.extract == 'latest' else args.extract + pbf = fetch(GEOFABRIK + extract + '.osm.pbf', args.cache / (extract + '.osm.pbf')) + expected = fetch(GEOFABRIK + extract + '.osm.pbf.md5', + args.cache / (extract + '.osm.pbf.md5')).read_text(encoding='utf-8').split()[0] actual = md5(pbf) if actual != expected: raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected)) rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly')) - fresh = export(args.osmium, pbf, args.cache) - current = [line for line in DATASET.read_text().splitlines() if line.strip()] + fresh = export(args.osmium, pbf, args.cache, extract) + current = [line for line in DATASET.read_text(encoding='utf-8').splitlines() if line.strip()] merged, report = merge(current, fresh, rings) payload = '\n'.join(merged) + '\n' if args.verify: - if payload != DATASET.read_text(): - raise SystemExit('datacenters.geojsonl is not the %s merge' % EXTRACT) + if payload != DATASET.read_text(encoding='utf-8'): + raise SystemExit('datacenters.geojsonl is not the %s merge' % extract) else: - DATASET.write_text(payload) - print(json.dumps({'extract': EXTRACT, 'md5': actual, 'norway': len(fresh), + with open(DATASET, 'w', encoding='utf-8', newline='\n') as handle: + handle.write(payload) + print(json.dumps({'extract': extract, 'md5': actual, 'norway': len(fresh), 'features': len(merged), **report})) diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index ef0720ae4..8b803dcc7 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -28,5 +28,9 @@ reflects OSM as of the extract below, everything else the original snapshot. - Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` - Contact tags are dropped as above; unchanged features keep their original line, so geometry is only rewritten when tags or vertices actually changed. -- Rebuild or check: `python3 scripts/build-datacenters-norway.py --cache ` - (add `--verify` to check without writing; requires `osmium-tool`) +- Refresh: `python3 scripts/build-datacenters-norway.py --cache ` uses + Geofabrik's current Norway extract and prints the dated name it used; record + that name, its OSM timestamp and MD5 here. Requires `osmium-tool`. +- Check: add `--extract --verify` to compare without writing. + Geofabrik keeps only its newest dated extract, so a recorded extract can be + re-verified only while it is still listed; the MD5 above identifies it. From a803be1af273051738e633e778ca04fce1ae0387 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:01:44 -0500 Subject: [PATCH 05/27] fix(cyclones): evict prior shared selection on storm switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression: explicit storm A→B→A keeps one active record. Reviewer finding: quanticsoul4772 (#847). --- changelog.d/847-cyclone-shared-selection.md | 4 ++++ docs/CURRENT-STATE.md | 5 +++++ src/layers/cyclones/index.js | 6 +++++- src/layers/cyclones/index.test.mjs | 20 +++++++++++++++----- 4 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 changelog.d/847-cyclone-shared-selection.md diff --git a/changelog.d/847-cyclone-shared-selection.md b/changelog.d/847-cyclone-shared-selection.md new file mode 100644 index 000000000..a2f58ca71 --- /dev/null +++ b/changelog.d/847-cyclone-shared-selection.md @@ -0,0 +1,4 @@ +- Explicit cyclone selections publish only the currently selected NHC/CPHC + storm into shared context. Switching storms evicts the previous identity + instead of leaving stale coordinates active (kvnloo, #847; reviewed by + quanticsoul4772). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 22ce1eb3b..8e2662bf4 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -218,6 +218,11 @@ not storm size or the full hazard area. Successful empty and unavailable states are distinct. One owned native Cesium data source preserves polygon holes and geographic seams and releases on disable; forecast animation is not implied. Cyclone markers, labels, tracks and cones are hidden beyond the horizon on every map source. +Explicit cyclone selections publish one shared `weather-cyclones` record +with official advisory provenance and source position. Switching storms removes +the prior record; automatic row selection never claims global context, and +refreshes do not re-announce selection. Clearing or losing an explicitly selected +storm removes shared context rather than assigning a fallback. Consult the linked official advisory for safety decisions. Voice and HUD snapshots reuse the existing feedState classifier. Analyst follow-ups retain their original data provenance; current-view results append provenance without replacing legacy fields. HUD context and deterministic telemetry include non-nominal feed state. diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 2738bebe0..ae4e5175a 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -79,8 +79,12 @@ export function createCyclonesLayer({ return null; } selectedContextEntity ||= { show: true }; + const contextId = `cyclone:${storm.id}`; + context.removeEntityContextsForLayer?.('weather-cyclones', { + retainIds: new Set([contextId]), + }); const record = context.registerEntityContext?.(selectedContextEntity, { - id: `cyclone:${storm.id}`, + id: contextId, layerId: 'weather-cyclones', layerName: 'Cyclone advisories', source: 'NOAA NHC / CPHC', diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index 7d796bef7..d1b7fefd4 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -419,11 +419,13 @@ test('explicit cyclone selection publishes shared context without auto-claiming selectedEvents += 1; return selectedContext; }, - removeEntityContextsForLayer(layerId) { + removeEntityContextsForLayer(layerId, { retainIds } = {}) { for (const [id, record] of records) { - if (record.layerId === layerId) records.delete(id); + if (record.layerId === layerId && !retainIds?.has(id)) + records.delete(id); } - if (selectedContext?.layerId === layerId) selectedContext = null; + if (selectedContext && !records.has(selectedContext.id)) + selectedContext = null; }, }; const h = harness({ getSnapshot: async () => next }, { context }); @@ -446,10 +448,18 @@ test('explicit cyclone selection publishes shared context without auto-claiming assert.equal(selectedContext?.longitude, -125.8); assert.equal(selectedContext?.properties.advisoryNumber, '10'); + h.layer.setParams({ stormId: 'ep152026' }); + assert.deepEqual([...records.keys()], ['cyclone:ep152026']); + assert.equal(selectedContext?.id, 'cyclone:ep152026'); + h.layer.setParams({ stormId: 'ep162026' }); + assert.deepEqual([...records.keys()], ['cyclone:ep162026']); + assert.equal(selectedContext?.id, 'cyclone:ep162026'); + assert.equal(selectedEvents, 3, 'each explicit storm switch announces once'); + await h.layer.update(); assert.equal( selectedEvents, - 1, + 3, 'refresh updates the selected record without re-announcing the click', ); @@ -463,7 +473,7 @@ test('explicit cyclone selection publishes shared context without auto-claiming assert.equal(records.size, 0); h.layer.setParams({ stormId: 'ep152026' }); - assert.equal(selectedEvents, 2); + assert.equal(selectedEvents, 4); h.layer.setParams({ clear: true }); assert.equal(selectedContext, null); assert.equal(records.size, 0); From 3a7c7b7c6a9c1c2a126bb6c415386f6016a2cdec Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:36:07 -0500 Subject: [PATCH 06/27] fix(cyclones): keep selected names source-owned for i18n --- src/layers/cyclones/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 2529b98e0..66bf13bbd 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -111,7 +111,9 @@ export function createCyclonesLayer({ layerId: 'weather-cyclones', layerName: 'Cyclone advisories', source: 'NOAA NHC / CPHC', - label: `Cyclone · ${storm.name}`, + // Layer identity is already carried separately. Keep source names literal + // rather than adding an untranslated user-facing prefix. + label: storm.name, latitude: storm.position.latitude, longitude: storm.position.longitude, properties: { From 0e2ae76f0ab0f75df72010c1c6ae41c3c052b505 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 17 Oct 2026 03:36:09 -0500 Subject: [PATCH 07/27] test(cyclones): assert source-owned context label --- src/layers/cyclones/index.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index 06342464b..f7cbcdc12 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -443,7 +443,7 @@ test('explicit cyclone selection publishes shared context without auto-claiming assert.equal(selectedEvents, 1); assert.equal(selectedContext?.layerId, 'weather-cyclones'); assert.equal(selectedContext?.source, 'NOAA NHC / CPHC'); - assert.equal(selectedContext?.label, 'Cyclone · Another system'); + assert.equal(selectedContext?.label, 'Another system'); assert.equal(selectedContext?.latitude, 15.5); assert.equal(selectedContext?.longitude, -125.8); assert.equal(selectedContext?.properties.advisoryNumber, '10'); From 4318f51de4114bfb68137ff48bedd070b976f771 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:03:23 +0200 Subject: [PATCH 08/27] docs: move the changelog entry to changelog.d --- CHANGELOG.md | 7 ------- changelog.d/953-datacenters-norway-refresh.md | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 changelog.d/953-datacenters-norway-refresh.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a285bc555..98d32365f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -284,13 +284,6 @@ ## [Unreleased] -- Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract - (2026-10-05). Troll Housing joins the layer (4,352 features) and four - Norwegian sites pick up current OSM tags; no site moved or was removed. - `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, - touching only features inside Geofabrik's Norway boundary, and the dataset - README records the extract, checksum and filter. - - Add a keyless Air quality layer to the Weather group, with Environment and Climate Change Canada's AQHI as the first registered provider. Readings carry `{ provider, scale, value, band }` so another network can be added without diff --git a/changelog.d/953-datacenters-norway-refresh.md b/changelog.d/953-datacenters-norway-refresh.md new file mode 100644 index 000000000..48a019605 --- /dev/null +++ b/changelog.d/953-datacenters-norway-refresh.md @@ -0,0 +1,6 @@ +- Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract + (2026-10-05). Troll Housing joins the layer (4,352 features) and four + Norwegian sites pick up current OSM tags; no site moved or was removed. + `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, + touching only features inside Geofabrik's Norway boundary, and the dataset + README records the extract, checksum and filter (trygvehhh, #953). From b358b5dba176e744fa53de73074a0b242b768a4f Mon Sep 17 00:00:00 2001 From: Said Turksever <28983926+asturksever@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:17:39 +0100 Subject: [PATCH 09/27] fix(cyclones): distinguish deliberate clears from feed eviction --- changelog.d/847-cyclone-shared-selection.md | 6 +-- docs/CURRENT-STATE.md | 2 + src/layers/cyclones/index.js | 18 ++++--- src/layers/cyclones/index.test.mjs | 59 +++++++++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/changelog.d/847-cyclone-shared-selection.md b/changelog.d/847-cyclone-shared-selection.md index a2f58ca71..24edf2ebb 100644 --- a/changelog.d/847-cyclone-shared-selection.md +++ b/changelog.d/847-cyclone-shared-selection.md @@ -1,4 +1,4 @@ - Explicit cyclone selections publish only the currently selected NHC/CPHC - storm into shared context. Switching storms evicts the previous identity - instead of leaving stale coordinates active (kvnloo, #847; reviewed by - quanticsoul4772). + storm into shared context. Switching storms removes the previous identity + without a false eviction event. Deliberate clears remain distinct from feed + disappearance (kvnloo, #847; reviewed by quanticsoul4772). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 42ebf7555..9d6680086 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -394,6 +394,8 @@ with official advisory provenance and source position. Switching storms removes the prior record; automatic row selection never claims global context, and refreshes do not re-announce selection. Clearing or losing an explicitly selected storm removes shared context rather than assigning a fallback. +Deliberate clear/disable reports `deliberate`; feed disappearance reports +`evicted`. Switching storms selects the replacement before pruning the old record. Consult the linked official advisory for safety decisions. Voice and HUD snapshots reuse the existing feedState classifier. Analyst follow-ups retain their original data provenance; current-view results append provenance without replacing legacy fields. HUD context and deterministic telemetry include non-nominal feed state. diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 66bf13bbd..1b45265c5 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -89,7 +89,9 @@ export function createCyclonesLayer({ const notify = () => listener?.(); const selected = () => snapshot?.storms.find((storm) => storm.id === selectedId) || null; - function clearSelectedContext() { + function clearSelectedContext({ evicted = false } = {}) { + if (!evicted) + context?.clearSelectedEntityContextForLayer?.('weather-cyclones'); context?.removeEntityContextsForLayer?.('weather-cyclones'); selectedContextStormId = null; selectedContextEntity = null; @@ -98,14 +100,11 @@ export function createCyclonesLayer({ function publishSelectedContext({ announce = false } = {}) { const storm = selected(); if (!storm || !context) { - if (!storm) clearSelectedContext(); + if (!storm) clearSelectedContext({ evicted: true }); return null; } selectedContextEntity ||= { show: true }; const contextId = `cyclone:${storm.id}`; - context.removeEntityContextsForLayer?.('weather-cyclones', { - retainIds: new Set([contextId]), - }); const record = context.registerEntityContext?.(selectedContextEntity, { id: contextId, layerId: 'weather-cyclones', @@ -131,6 +130,9 @@ export function createCyclonesLayer({ if (!record) return null; selectedContextStormId = storm.id; if (announce) context.selectEntityContext?.(selectedContextEntity); + context.removeEntityContextsForLayer?.('weather-cyclones', { + retainIds: new Set([contextId]), + }); return record; } @@ -302,7 +304,7 @@ export function createCyclonesLayer({ snapshot = next; selectedId = null; ++navigationGeneration; - clearSelectedContext(); + clearSelectedContext({ evicted: true }); error = next.reason || ADVISORIES_UNAVAILABLE; return true; } @@ -327,7 +329,7 @@ export function createCyclonesLayer({ else rendering.setSelection(selectedId); if (selectedContextStormId) { if (selectedId === selectedContextStormId) publishSelectedContext(); - else clearSelectedContext(); + else clearSelectedContext({ evicted: true }); } return true; } catch (cause) { @@ -338,7 +340,7 @@ export function createCyclonesLayer({ snapshot = null; selectedId = null; ++navigationGeneration; - clearSelectedContext(); + clearSelectedContext({ evicted: true }); return true; } finally { signal?.removeEventListener('abort', abort); diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index f7cbcdc12..f1c7656b1 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -7,6 +7,7 @@ import { isPointerFree, } from '../../data/inputOwnership.js'; import { isOwnedByOtherLayer } from '../../data/pickRegistry.js'; +import * as contextStore from '../../data/contextStore.js'; const time = '2026-09-16T03:00:00.000Z'; const storm = (id = 'ep152026') => ({ @@ -481,6 +482,64 @@ test('explicit cyclone selection publishes shared context without auto-claiming h.layer.destroy(); }); +test('shared cyclone context distinguishes switches, deliberate clears and feed eviction', async (t) => { + const previousWindow = Object.getOwnPropertyDescriptor(globalThis, 'window'); + const host = new EventTarget(); + Object.defineProperty(globalThis, 'window', { + configurable: true, + value: host, + }); + let h; + t.after(() => { + try { + h?.layer.destroy(); + } finally { + if (previousWindow) + Object.defineProperty(globalThis, 'window', previousWindow); + else delete globalThis.window; + } + }); + const events = []; + host.addEventListener('gev:entity-selected', ({ detail }) => + events.push(['selected', detail.id]), + ); + host.addEventListener('gev:entity-selection-cleared', ({ detail }) => + events.push(['cleared', detail.reason]), + ); + let next = snapshot([storm(), storm('ep162026')]); + h = harness({ getSnapshot: async () => next }, { context: contextStore }); + h.layer.enable(); + await h.layer.update(); + h.layer.setParams({ stormId: 'ep152026' }); + h.layer.setParams({ stormId: 'ep162026' }); + assert.deepEqual(events, [ + ['selected', 'cyclone:ep152026'], + ['selected', 'cyclone:ep162026'], + ]); + assert.deepEqual( + [...contextStore.getContextStore().entities.keys()], + ['cyclone:ep162026'], + ); + await h.layer.update(); + assert.equal(events.length, 2, 'refresh must not announce selection'); + + h.layer.setParams({ clear: true }); + assert.deepEqual(events.at(-1), ['cleared', 'deliberate']); + assert.equal(contextStore.getContextStore().entities.size, 0); + + h.layer.setParams({ stormId: 'ep162026' }); + next = snapshot([storm()]); + await h.layer.update(); + assert.deepEqual(events.at(-1), ['cleared', 'evicted']); + assert.equal(contextStore.getContextStore().entities.size, 0); + assert.equal(contextStore.getContextStore().selectedEntityId, null); + + h.layer.setParams({ stormId: 'ep152026' }); + h.layer.disable(); + assert.deepEqual(events.at(-1), ['cleared', 'deliberate']); + assert.equal(contextStore.getContextStore().entities.size, 0); +}); + test('refresh preserves selection intent, including explicit clears and missing storms', async () => { let next = snapshot([storm(), storm('ep162026')]); const h = harness({ getSnapshot: async () => next }); From 04d626a4b190149ffcb017290c5d9aa17d32748d Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 3 Nov 2026 15:02:42 +0400 Subject: [PATCH 10/27] fix(datacenters): preserve missing geometry and compare ring topology --- scripts/build-datacenters-norway.py | 64 ++++++++++++--- src/data/datacentersNorwayRefresh.test.mjs | 93 ++++++++++++++++++++++ src/data/local_data/datacenters/README.md | 4 +- 3 files changed, 148 insertions(+), 13 deletions(-) create mode 100644 src/data/datacentersNorwayRefresh.test.mjs diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py index 687395f7b..03286b864 100644 --- a/scripts/build-datacenters-norway.py +++ b/scripts/build-datacenters-norway.py @@ -4,7 +4,8 @@ Downloads a dated Geofabrik Norway extract, verifies its MD5, filters telecom=data_center / building=data_center with osmium, and merges the result into datacenters.geojsonl: features inside the Geofabrik Norway boundary are -added, updated or removed; every other line is kept byte-for-byte. +added, updated or removed once no longer tagged; every other line is kept +byte-for-byte. Geofabrik keeps only its newest dated extract, so --extract defaults to the current one and a recorded extract can be re-verified only while it is listed. @@ -16,6 +17,7 @@ import math from pathlib import Path import shutil import subprocess +import sys import urllib.request ROOT = Path(__file__).resolve().parent.parent @@ -95,10 +97,15 @@ def osm_key(feature): def export(osmium, pbf, cache, extract): + """Exported features by key, and the keys of every tagged object in the extract.""" filtered = cache / (extract + '-datacenters.osm.pbf') + listed = cache / (extract + '-datacenters.opl') exported = cache / (extract + '-datacenters.geojsonseq') subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf), 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) + subprocess.run([osmium, 'tags-filter', '-O', '-R', '-f', 'opl', '-o', str(listed), str(filtered), + 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) + tagged = {line.split(' ', 1)[0] for line in listed.read_text(encoding='utf-8').splitlines() if line} subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id', '-o', str(exported), str(filtered)], check=True) fresh = {} @@ -122,22 +129,48 @@ def export(osmium, pbf, cache, extract): elif key in fresh or geometry['type'] == 'LineString': continue # closed ways are taken from their area export fresh[key] = {'geometry': geometry, 'tags': tags} - return fresh + return fresh, tagged def same_geometry(old, new): - """Same type and vertex set within tolerance; ring start and winding may differ.""" + """Same type and rings within tolerance; ring start, winding and part or hole order may differ.""" if old['type'] != new['type']: return False - a, b = points(old['coordinates']), points(new['coordinates']) - if len(a) != len(b): - return False def meters(p, q): dx = (p[0] - q[0]) * 111320 * math.cos(math.radians(p[1])) return math.hypot(dx, (p[1] - q[1]) * 110540) - return all(min(meters(p, q) for q in b) <= SAME_GEOMETRY_METERS for p in a) and \ - all(min(meters(q, p) for p in a) <= SAME_GEOMETRY_METERS for q in b) + + def same_line(a, b): + return len(a) == len(b) and all(meters(p, q) <= SAME_GEOMETRY_METERS for p, q in zip(a, b)) + + def same_ring(a, b): + a, b = a[:-1], b[:-1] # closed rings repeat their first vertex + return len(a) == len(b) and \ + any(same_line(a[i:] + a[:i], c) for c in (b, b[::-1]) for i in range(len(a))) + + def same_parts(a, b, same): + rest = list(b) + for part in a: + match = next((i for i, other in enumerate(rest) if same(part, other)), None) + if match is None: + return False + rest.pop(match) + return not rest + + def same_polygon(a, b): + return same_ring(a[0], b[0]) and same_parts(a[1:], b[1:], same_ring) + + a, b = old['coordinates'], new['coordinates'] + if old['type'] == 'Point': + return same_line([a], [b]) + if old['type'] == 'LineString': + return same_line(a, b) + if old['type'] == 'Polygon': + return same_polygon(a, b) + if old['type'] == 'MultiPolygon': + return same_parts(a, b, same_polygon) + return a == b def feature_line(key, sequence, fresh): @@ -148,9 +181,9 @@ def feature_line(key, sequence, fresh): return json.dumps(feature, separators=(',', ':'), ensure_ascii=False) -def merge(lines, fresh, rings): +def merge(lines, fresh, rings, tagged): out, seen, sequence = [], set(), 0 - report = {'kept': 0, 'updated': [], 'removed': [], 'added': []} + report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': []} for line in lines: feature = json.loads(line) sequence = max(sequence, feature['properties']['id']) @@ -165,6 +198,10 @@ def merge(lines, fresh, rings): out.append(feature_line(key, feature['properties']['id'], fresh[key])) report['updated'].append(key) continue + if key in tagged: # still tagged, but osmium built no usable geometry: keep the old line + report['unexported'].append(key) + out.append(line) + continue lon, lat = points(feature['geometry']['coordinates'])[0][:2] if inside(rings, lon, lat): report['removed'].append(key) @@ -196,9 +233,12 @@ def main(): if actual != expected: raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected)) rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly')) - fresh = export(args.osmium, pbf, args.cache, extract) + fresh, tagged = export(args.osmium, pbf, args.cache, extract) current = [line for line in DATASET.read_text(encoding='utf-8').splitlines() if line.strip()] - merged, report = merge(current, fresh, rings) + merged, report = merge(current, fresh, rings, tagged) + if report['unexported']: + print('warning: kept the old line of tagged features osmium did not export: %s' + % ', '.join(report['unexported']), file=sys.stderr) payload = '\n'.join(merged) + '\n' if args.verify: if payload != DATASET.read_text(encoding='utf-8'): diff --git a/src/data/datacentersNorwayRefresh.test.mjs b/src/data/datacentersNorwayRefresh.test.mjs new file mode 100644 index 000000000..2b7b7d5bf --- /dev/null +++ b/src/data/datacentersNorwayRefresh.test.mjs @@ -0,0 +1,93 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const SCRIPT = fileURLToPath( + new URL('../../scripts/build-datacenters-norway.py', import.meta.url), +); + +function python(t, source) { + const result = spawnSync('python3', ['-c', source, SCRIPT], { + encoding: 'utf8', + }); + if (result.error?.code === 'ENOENT') return t.skip('python3 is not installed'); + assert.equal(result.status, 0, result.stderr); +} + +test('Norway refresh compares rings, not vertex sets', (t) => { + python( + t, + String.raw` +import runpy, sys +same = runpy.run_path(sys.argv[1])['same_geometry'] +poly = lambda *rings: {'type': 'Polygon', 'coordinates': list(rings)} +multi = lambda *polys: {'type': 'MultiPolygon', 'coordinates': [p['coordinates'] for p in polys]} +a, b, c, d, e = [0, 0], [4, 0], [4, 4], [0, 4], [2, 1] +assert not same(poly([a, b, c, d, e, a]), poly([a, b, e, c, d, a])), 'reordered edges' +assert same(poly([a, b, c, d, e, a]), poly([c, b, a, e, d, c])), 'ring start and winding' +assert same(poly([a, b, c, d, a]), poly([[0, 1e-7], b, c, d, [0, 1e-7]])), 'tolerance' +outer = [[0, 0], [9, 0], [9, 9], [0, 9], [0, 0]] +one, two = [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [6, 5], [6, 6], [5, 5]] +assert same(poly(outer, one, two), poly(outer, two, one)), 'hole order' +assert not same(poly(outer, one), poly(one, outer)), 'outer and inner roles swapped' +assert same(multi(poly(one), poly(two)), multi(poly(two), poly(one))), 'part order' +p, q, r, s, u, v = [0, 0], [1, 0], [0, 1], [5, 5], [6, 5], [5, 6] +assert not same(multi(poly([p, q, r, p]), poly([s, u, v, s])), + multi(poly([p, q, s, p]), poly([r, u, v, r]))), 'vertices moved between parts' +`, + ); +}); + +test('Norway refresh keeps a still-tagged feature osmium did not export', (t) => { + python( + t, + String.raw` +import json, runpy, sys +merge = runpy.run_path(sys.argv[1])['merge'] +square = [[[10, 60], [10.01, 60], [10.01, 60.01], [10, 60.01], [10, 60]]] +def line(osm_id, sequence): + return json.dumps({'id': osm_id, 'type': 'Feature', 'geometry': {'type': 'Polygon', 'coordinates': square}, + 'properties': {'id': sequence, 'tags': {'building': 'data_center'}, + 'type': 'data_center', 'osm_id': osm_id}}) +norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)] +lines = [line(7, 1), line(8, 2)] +out, report = merge(lines, {}, norway, {'w7'}) +assert out == [lines[0]], out +assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report +`, + ); +}); + +test('Norway refresh lists tagged objects apart from the exported geometry', (t) => { + python( + t, + String.raw` +import json, runpy, subprocess, sys, tempfile +from pathlib import Path +export = runpy.run_path(sys.argv[1])['export'] +tags = {'building': 'data_center'} +ring = [[10, 60], [10.01, 60], [10.01, 60.01], [10, 60]] +features = [ + {'id': 'n1', 'properties': {'telecom': 'data_center'}, 'geometry': {'type': 'Point', 'coordinates': [10, 60]}}, + {'id': 'w5', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring}}, + {'id': 'a10', 'properties': tags, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[ring]]}}, + {'id': 'w7', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring[:3]}}, +] +outputs = { + '.opl': 'n1 v1 Ttelecom=data_center x10 y60\nw5 v1 Tbuilding=data_center Nn2,n3,n4,n2\n' + 'w7 v1 Tbuilding=data_center Nn2,n3,n4\nr9 v1 Ttelecom=data_center Mw8@outer\n', + '.geojsonseq': ''.join(chr(30) + json.dumps({'type': 'Feature', **f}) + '\n' for f in features), +} +def run(args, check=False): + target = Path(args[args.index('-o') + 1]) + target.write_text(outputs.get(target.suffix, ''), encoding='utf-8') +subprocess.run = run +with tempfile.TemporaryDirectory() as cache: + fresh, tagged = export('osmium', Path(cache) / 'norway-test.osm.pbf', Path(cache), 'norway-test') +assert sorted(fresh) == ['n1', 'w5'], fresh +assert fresh['w5']['geometry'] == {'type': 'Polygon', 'coordinates': [ring]}, fresh['w5'] +assert tagged == {'n1', 'w5', 'w7', 'r9'}, tagged +`, + ); +}); diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index 8b803dcc7..5a80df267 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -27,7 +27,9 @@ reflects OSM as of the extract below, everything else the original snapshot. 2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0` - Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` - Contact tags are dropped as above; unchanged features keep their original - line, so geometry is only rewritten when tags or vertices actually changed. + line, so geometry is only rewritten when tags or rings actually changed. + A feature is removed only once it is no longer tagged; one still tagged but + without an exportable geometry keeps its line and is reported as unexported. - Refresh: `python3 scripts/build-datacenters-norway.py --cache ` uses Geofabrik's current Norway extract and prints the dated name it used; record that name, its OSM timestamp and MD5 here. Requires `osmium-tool`. From 92666fdbafe61e81f52dfa481095e70cabb60d35 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 3 Nov 2026 15:10:20 +0400 Subject: [PATCH 11/27] fix(cyclones): preserve movement in selected voice context --- src/layers/cyclones/index.js | 4 ++- src/layers/cyclones/index.test.mjs | 48 +++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 1b45265c5..ba7db3df6 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -123,7 +123,9 @@ export function createCyclonesLayer({ positionAt: storm.positionAt, windKt: storm.windKt, pressureHpa: storm.pressureHpa, - movement: storm.movement, + // Flat scalars: the voice context summary drops nested objects. + movementDirectionDegrees: storm.movement?.directionDegrees, + movementSpeedKt: storm.movement?.speedKt, advisoryUrl: storm.advisoryUrl, }, }); diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index f1c7656b1..406cdd257 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -8,6 +8,8 @@ import { } from '../../data/inputOwnership.js'; import { isOwnedByOtherLayer } from '../../data/pickRegistry.js'; import * as contextStore from '../../data/contextStore.js'; +import { createGevActionRunner } from '../../voice/gevActions.js'; +import * as Cesium from 'cesium'; const time = '2026-09-16T03:00:00.000Z'; const storm = (id = 'ep152026') => ({ @@ -506,12 +508,56 @@ test('shared cyclone context distinguishes switches, deliberate clears and feed host.addEventListener('gev:entity-selection-cleared', ({ detail }) => events.push(['cleared', detail.reason]), ); - let next = snapshot([storm(), storm('ep162026')]); + // The voice runner reads the same store and bounds its basemap wait. + Object.assign(host, { setTimeout, clearTimeout }); + const position = Cesium.Cartesian3.fromDegrees(-125.8, 15.5, 5_000_000); + const runner = createGevActionRunner({ + viewer: { + clock: { onTick: { addEventListener: () => () => {} } }, + scene: { canvas: { addEventListener() {}, removeEventListener() {} } }, + camera: { + moveEnd: { addEventListener() {} }, + positionWC: position, + positionCartographic: Cesium.Cartographic.fromCartesian(position), + heading: 0, + pitch: 0, + }, + }, + styleManager: {}, + dataManager: { + layers: new Map(), + isEnabled: (id) => id === 'weather-cyclones', + getAll: () => [], + }, + }); + const selectedProperties = async () => { + const result = await runner('get_entity_context', { scope: 'selected' }); + assert.equal(result.scope, 'selected'); + return result.selected.properties; + }; + let next = snapshot([ + { ...storm(), movement: { directionDegrees: 315, speedKt: 12 } }, + { ...storm('ep162026'), movement: { directionDegrees: 0, speedKt: 0 } }, + ]); h = harness({ getSnapshot: async () => next }, { context: contextStore }); h.layer.enable(); await h.layer.update(); h.layer.setParams({ stormId: 'ep152026' }); + let properties = await selectedProperties(); + assert.equal( + properties.movementDirectionDegrees, + '315', + 'selected storm movement should survive voice summarization', + ); + assert.equal(properties.movementSpeedKt, '12'); h.layer.setParams({ stormId: 'ep162026' }); + properties = await selectedProperties(); + assert.equal( + properties.movementDirectionDegrees, + '0', + 'a due-north heading is a value, not a missing one', + ); + assert.equal(properties.movementSpeedKt, '0', 'stationary is a value'); assert.deepEqual(events, [ ['selected', 'cyclone:ep152026'], ['selected', 'cyclone:ep162026'], From 2c936ae131ed8b24abfc45cc861c8de75df50061 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Tue, 3 Nov 2026 16:26:45 +0000 Subject: [PATCH 12/27] docs: correct hidden-contact rotation interval --- docs/CURRENT-STATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 5be325222..2a6dc5fdb 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4989,7 +4989,7 @@ silently demoting every later lookup for the session. ### Motion & Symbology Correctness (June 10, 2026) -- **Flights (commercial + military)** render one poll interval behind real time (30s/15s) and interpolate between two known feed-stamped fixes (OpenSky `time_position`; adsb.lol `receipt − seen_pos`). The display latency is an intentional product decision — do not "fix" it away. The fleet dead-reckons at ~12Hz (1m² write gating); a contact already hidden behind the horizon whose last position still is re-reckons on a staggered one-in-12-ticks rotation (about once a second) instead, so the worldwide feed's ~13k contacts no longer cost a full pass when well under 1% are on screen. Anything that position puts in view, and any contact with a fleet model, takes the full pass immediately, and hidden-inclusive proximity queries read positions at most about a second old; aircraft get a 3-poll grace period (faded icon) before removal. When a position epoch pauses, both layers coast for at least 60 seconds of contact grace with an absolute five-minute ceiling. Source backoff marks each contact and the cockpit `STALE`; the cockpit then holds the exact layer position instead of continuing inertial flight. Repeated-position kinematic changes create a forward-only synthetic fix rather than mutating history, and grounded history is lifted only when no owned 3D model already controls its datum. A nominally successful worldwide OpenSky response whose own snapshot epoch is more than two minutes old prefers the existing 250-nm viewport-scoped adsb.lol fallback and labels the source/coverage accordingly when that upstream is available. If the fallback is also unavailable, the layer's freshness/error fields use the source epoch—never the cache receipt time—so the UI reports an old snapshot rather than “just now.” +- **Flights (commercial + military)** render one poll interval behind real time (30s/15s) and interpolate between two known feed-stamped fixes (OpenSky `time_position`; adsb.lol `receipt − seen_pos`). The display latency is an intentional product decision — do not "fix" it away. The fleet dead-reckons at ~12Hz (1m² write gating); a contact already hidden behind the horizon whose last position still is re-reckons on a staggered one-in-13-ticks rotation (about once a second) instead, so the worldwide feed's ~13k contacts no longer cost a full pass when well under 1% are on screen. Anything that position puts in view, and any contact with a fleet model, takes the full pass immediately, and hidden-inclusive proximity queries read positions at most about a second old; aircraft get a 3-poll grace period (faded icon) before removal. When a position epoch pauses, both layers coast for at least 60 seconds of contact grace with an absolute five-minute ceiling. Source backoff marks each contact and the cockpit `STALE`; the cockpit then holds the exact layer position instead of continuing inertial flight. Repeated-position kinematic changes create a forward-only synthetic fix rather than mutating history, and grounded history is lifted only when no owned 3D model already controls its datum. A nominally successful worldwide OpenSky response whose own snapshot epoch is more than two minutes old prefers the existing 250-nm viewport-scoped adsb.lol fallback and labels the source/coverage accordingly when that upstream is available. If the fallback is also unavailable, the layer's freshness/error fields use the source epoch—never the cache receipt time—so the UI reports an old snapshot rather than “just now.” - **World-space headings at every angle** (`src/data/iconOrientation.js`): aircraft/vessel icon rotation uses the camera right/up basis per tick (alignedAxis always ZERO), which is exact at screen center and for orthographic/nadir views and remains stable through >180° tracked orbits. Perspective rays vary across the viewport, so off-center contacts at oblique pitch can diverge from an exact finite-difference window projection; a regression test pins that known regime, and field evidence decides whether to adopt exact projection with the basis method as fallback. Fleet rotations refresh on camera-pose change; tracked entities per frame. Billboards are horizon-culled via a shared EllipsoidalOccluder. - **Military/OpenSky reconciliation** (`src/data/militaryRegistry.js`): known-military ICAOs render amber in the flights layer (60s self-poll of the cached mil endpoint when the military layer is off) and are suppressed there while the military layer renders them. - **Satellites**: 838-sat core catalog (stations/visual/GPS/GLONASS/Galileo/GEO), tracking lands ~726km out via `viewFrom` (tracked entity owns a point graphic so the tracking camera engages), rings realign via primitive modelMatrix (no per-second rebuild flicker), optional `setParams({catalog:'dense'})` Starlink mode. From 36fceb56752297c6fb567e6766437bfbf9108e02 Mon Sep 17 00:00:00 2001 From: Dylan Maniatakes Date: Tue, 3 Nov 2026 11:35:13 -0500 Subject: [PATCH 13/27] Add adjustable event age filtering to S2 GhostMaps --- docs/CURRENT-STATE.md | 1 + scripts/qa-ghostmaps.mjs | 76 +++++- src/i18n/locales/ar.json | 7 +- src/i18n/locales/de.json | 7 +- src/i18n/locales/es.json | 7 +- src/i18n/locales/fr.json | 7 +- src/i18n/locales/it.json | 7 +- src/i18n/locales/ja.json | 7 +- src/i18n/locales/pt.json | 7 +- src/i18n/locales/ru.json | 7 +- src/i18n/locales/uk.json | 7 +- src/i18n/locales/zh-CN.json | 7 +- src/i18n/locales/zh-Hant.json | 7 +- src/layers/ghostMaps/index.js | 271 ++++++++++++++------ src/layers/ghostMaps/lifecycle.test.mjs | 317 +++++++++++++++++++++++- src/layers/ghostMaps/records.js | 112 ++++++++- src/layers/ghostMaps/records.test.mjs | 83 +++++++ 17 files changed, 820 insertions(+), 117 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 59557e46b..827035fe0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5663,6 +5663,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. +- **Event age filter.** The same filter applies to every selected S2 product, including Border Crisis, Iran Picture, Iran Kinetic, Yemen FLOT and Latest Drop. S2 products default to seven days, with row buttons for 1, 3, 7, 14 or 30 days and **All markers**. The choice persists independently of product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage uses seven days. Only the placemark's `Date of Event` (or `EventDate`/`Date`) determines its age, never report dates, dates in prose or KMZ edition/upload dates. Undated, blank or unparseable event dates and future dates stay visible. ArcGIS `M/D/YYYY` dates and optional AM/PM times, plus ISO dates/timestamps, are supported; values without a zone use UTC and date-only values use midnight UTC. The exact cutoff is included. Events age out every minute and during catalog updates, even without a new file or a successful catalog fetch. Widening the window or choosing All markers restores omitted events from the cached KMZ. Removed entities and selected cards disappear together; surviving pick ids stay stable. Product defaults and the update stream are unchanged. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/scripts/qa-ghostmaps.mjs b/scripts/qa-ghostmaps.mjs index 83dc8ed97..24edcc60b 100644 --- a/scripts/qa-ghostmaps.mjs +++ b/scripts/qa-ghostmaps.mjs @@ -28,6 +28,8 @@ * (vii) FAR SIDE — pins skip the depth test, so they are horizon-culled: * seen from over the Indian Ocean no pin clearly behind the planet is * shown; back over North America the near-side pins show again. + * (viii) AGE — seven days is the default; shorter/longer windows and All + * markers update the globe, and the age choice survives a reload. * * Screenshots go to qa-shots/ghostmaps-*.png (gitignored). * @@ -201,12 +203,15 @@ async function main() { ); record( 'LIVE: one chip per product, defaults pressed', - chips.length === catalog.products.length && - chips.every( - (chip) => - chip.active === - catalog.products.find((p) => p.id === chip.id)?.defaultOn, - ), + chips.filter((chip) => !chip.id.startsWith('age-')).length === + catalog.products.length && + chips + .filter((chip) => !chip.id.startsWith('age-')) + .every( + (chip) => + chip.active === + catalog.products.find((p) => p.id === chip.id)?.defaultOn, + ), chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '), ); const stream = await page @@ -224,6 +229,65 @@ async function main() { record('LIVE: the /api/ghostmaps/events stream is open', stream); await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') }); + const ageChips = chips.filter((chip) => chip.id.startsWith('age-')); + record( + 'AGE: seven days is the default with an All markers option', + live.ageDays === 7 && + ageChips.length === 6 && + ageChips + .filter((chip) => chip.active) + .map((chip) => chip.id) + .join() === 'age-7' && + ageChips.some((chip) => chip.id === 'age-all'), + ); + const selectAge = async (days) => { + await showRow(page); + const selector = `[data-layer-id="${LAYER_ID}"] [data-chip-id="age-${days ?? 'all'}"]`; + await page.waitForSelector(selector, { timeout: 15000 }); + // Dispatch through the rendered button, matching the product-chip gate. + await page.$eval(selector, (button) => button.click()); + await page.waitForFunction( + (id, age) => { + const s = window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats(); + return s.ageDays === age && !s.loading; + }, + { timeout: TIMEOUT_MS }, + LAYER_ID, + days, + ); + return stats(page); + }; + const all = await selectAge(null); + record( + 'AGE: All markers restores old events without duplicate data sources', + all.count > live.count && (await sources(page)).length === 2, + `${live.count} → ${all.count}`, + ); + await page.reload({ waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__godsEyeView?.dataManager, { + timeout: TIMEOUT_MS, + }); + await enableAndSettle(page); + const persistedAge = await stats(page); + record( + 'AGE: All markers survives a reload', + persistedAge.ageDays === null && persistedAge.count === all.count, + ); + const thirty = await selectAge(30); + const one = await selectAge(1); + const seven = await selectAge(7); + record( + 'AGE: shorter and longer windows change the count and seven days restores the default', + one.count <= live.count && + thirty.count >= live.count && + thirty.count <= all.count && + seven.count === live.count && + seven.ageDays === 7, + `1d=${one.count}, 7d=${seven.count}, 30d=${thirty.count}, all=${all.count}`, + ); + // ---- (ii) PICK ---------------------------------------------------------- const target = await page.evaluate((id) => { const gev = window.__godsEyeView; diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index ca04bf306..345fbc0ab 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3306,5 +3306,10 @@ "counted over the bounding box of {area}, not its outline": "عُدّ ضمن المستطيل المحيط ({area})، لا ضمن الحدود", "Search cancelled": "أُلغي البحث", "Sea Marks": "علامات بحرية", - "Ski Pistes": "مسارات التزلج" + "Ski Pistes": "مسارات التزلج", + "{days}d": "{days}ي", + "All markers": "كل العلامات", + "Show markers of every age": "عرض العلامات من جميع الأعمار", + "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index f990eae81..b59916432 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3006,5 +3006,10 @@ "counted over the bounding box of {area}, not its outline": "über das umgebende Rechteck gezählt ({area}), nicht über den Umriss", "Search cancelled": "Suche abgebrochen", "Sea Marks": "Seezeichen", - "Ski Pistes": "Skipisten" + "Ski Pistes": "Skipisten", + "{days}d": "{days}T", + "All markers": "Alle Markierungen", + "Show markers of every age": "Markierungen jeden Alters anzeigen", + "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index fcdd09185..47768e808 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contado en el rectángulo que lo rodea ({area}), no en su contorno", "Search cancelled": "Búsqueda cancelada", "Sea Marks": "Señales marítimas", - "Ski Pistes": "Pistas de esquí" + "Ski Pistes": "Pistas de esquí", + "{days}d": "{days}d", + "All markers": "Todos los marcadores", + "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", + "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index fd5728596..103bf00a9 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "compté sur le rectangle englobant ({area}), pas sur le contour", "Search cancelled": "Recherche annulée", "Sea Marks": "Balisage maritime", - "Ski Pistes": "Pistes de ski" + "Ski Pistes": "Pistes de ski", + "{days}d": "{days}j", + "All markers": "Tous les marqueurs", + "Show markers of every age": "Afficher les marqueurs de tout âge", + "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index cd0e85571..0cb73f516 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contato sul riquadro di delimitazione ({area}), non sul contorno", "Search cancelled": "Ricerca annullata", "Sea Marks": "Segnalamenti marittimi", - "Ski Pistes": "Piste da sci" + "Ski Pistes": "Piste da sci", + "{days}d": "{days}g", + "All markers": "Tutti gli indicatori", + "Show markers of every age": "Mostra gli indicatori di qualsiasi età", + "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 9a38c8549..ff79dafb8 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "外接矩形 ({area}) で数えた結果で、輪郭ではありません", "Search cancelled": "検索をキャンセルしました", "Sea Marks": "航路標識", - "Ski Pistes": "スキーゲレンデ" + "Ski Pistes": "スキーゲレンデ", + "{days}d": "{days}日", + "All markers": "すべてのマーカー", + "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", + "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数:{days}日。日付のないマーカーは保持", + "Event age: {days}d · undated kept": "経過日数:{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index 54225789d..9ed99b438 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contado na caixa delimitadora ({area}), não no contorno", "Search cancelled": "Busca cancelada", "Sea Marks": "Sinais marítimos", - "Ski Pistes": "Pistas de esqui" + "Ski Pistes": "Pistas de esqui", + "{days}d": "{days}d", + "All markers": "Todos os marcadores", + "Show markers of every age": "Mostrar marcadores de qualquer idade", + "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index eb4ca4dad..333a3518d 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3156,5 +3156,10 @@ "counted over the bounding box of {area}, not its outline": "подсчитано по ограничивающему прямоугольнику ({area}), а не по контуру", "Search cancelled": "Поиск отменён", "Sea Marks": "Морские знаки", - "Ski Pistes": "Горнолыжные трассы" + "Ski Pistes": "Горнолыжные трассы", + "{days}d": "{days}д", + "All markers": "Все метки", + "Show markers of every age": "Показать метки любого возраста", + "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index 903db6586..a989f456f 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3156,5 +3156,10 @@ "counted over the bounding box of {area}, not its outline": "підраховано за обмежувальним прямокутником ({area}), а не за контуром", "Search cancelled": "Пошук скасовано", "Sea Marks": "Морські знаки", - "Ski Pistes": "Гірськолижні траси" + "Ski Pistes": "Гірськолижні траси", + "{days}d": "{days}д", + "All markers": "Усі позначки", + "Show markers of every age": "Показати позначки будь-якого віку", + "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index 78a62c593..ddd567e92 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "按外接矩形({area})计数,而非其轮廓", "Search cancelled": "搜索已取消", "Sea Marks": "航标", - "Ski Pistes": "滑雪道" + "Ski Pistes": "滑雪道", + "{days}d": "{days}天", + "All markers": "所有标记", + "Show markers of every age": "显示所有时间的标记", + "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间:{days}天;保留无日期标记", + "Event age: {days}d · undated kept": "事件时间:{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index e2a5b9cc4..f1a52b067 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "按外接矩形({area})計數,而非其輪廓", "Search cancelled": "搜索已取消", "Sea Marks": "航標", - "Ski Pistes": "滑雪道" + "Ski Pistes": "滑雪道", + "{days}d": "{days}天", + "All markers": "所有標記", + "Show markers of every age": "顯示所有時間的標記", + "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間:{days}天;保留無日期標記", + "Event age: {days}d · undated kept": "事件時間:{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 01d564d05..7899c26b5 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -16,7 +16,15 @@ import { } from '../../i18n/index.js'; import { createHorizonCull } from '../streetLevel/view.js'; import { readGhostMapsKml } from './kml.js'; -import { fieldLabel, parseDescriptionFields, wrapCardText } from './records.js'; +import { + fieldLabel, + GHOSTMAPS_AGE_OPTIONS, + GHOSTMAPS_DEFAULT_AGE_DAYS, + ghostMapsEventDate, + isGhostMapsEventExpired, + parseDescriptionFields, + wrapCardText, +} from './records.js'; export * from './catalog.js'; export * from './records.js'; export { readGhostMapsKml } from './kml.js'; @@ -26,6 +34,7 @@ export const GHOSTMAPS_LAYER_ID = 's2-ghostmaps'; export const GHOSTMAPS_OVERLAY_SOURCE_ID = 's2-ghostmaps'; const PICK_PREFIX = 'ghostmaps:'; const STORAGE_KEY = 'gev.ghostmaps.products'; +const AGE_STORAGE_KEY = 'gev.ghostmaps.ageDays'; const ACCENT = '#c8ff00'; const MAX_ICON_BYTES = 512 * 1024; const CARD_HOST_OPTIONS = Object.freeze({ @@ -82,6 +91,19 @@ function writeStoredSelection(storage, selected) { } } +function readStoredAge(storage) { + try { + const raw = storage?.getItem(AGE_STORAGE_KEY); + if (raw !== null && raw !== undefined) { + const value = JSON.parse(raw); + if (GHOSTMAPS_AGE_OPTIONS.includes(value)) return value; + } + } catch { + /* unavailable or invalid storage uses the default */ + } + return GHOSTMAPS_DEFAULT_AGE_DAYS; +} + /** A round marker for placemarks whose icon is missing or remote. */ const markerCache = new Map(); function markerImage(color) { @@ -137,6 +159,7 @@ export function createGhostMapsLayer({ pointer = null, openExternal = null, storage = defaultStorage(), + now = () => Date.now(), } = {}) { if (typeof source?.getCatalog !== 'function') throw new TypeError('GhostMaps layer requires a catalog source'); @@ -154,7 +177,9 @@ export function createGhostMapsLayer({ let _selectedLink = null; let _lifetime = new AbortController(); let _selection = readStoredSelection(storage); - /** productId → {version, dataSource, urls, count, features} */ + let _ageDays = readStoredAge(storage); + let _expiryTimer = null; + /** productId → {version, dataSource, urls, features: Map, pins} */ const _loaded = new Map(); /** productId → in-flight version */ const _loading = new Map(); @@ -192,8 +217,31 @@ export function createGhostMapsLayer({ if (_selectedKey?.startsWith(`${id}:`)) clearSelection(); } + // Remove expired entities, rather than hiding them: horizon culling must + // never make an old pin visible again. Map keys keep surviving picks stable. + function expireEvents() { + const at = now(); + let changed = false; + for (const [productId, item] of _loaded) { + for (const [index, feature] of item.features) { + if (!isGhostMapsEventExpired(feature.dateMs, at, _ageDays)) continue; + for (const entity of feature.entities) + item.dataSource.entities.remove(entity); + item.features.delete(index); + if (_selectedKey === `${productId}:${index}`) clearSelection(); + changed = true; + } + item.pins = item.pins.filter((pin) => item.features.has(pin.index)); + } + if (changed) { + notifyRow(); + render(); + } + } + /** Build one product's data source off-scene, then swap it in. */ async function load(product, signal) { + const ageDays = _ageDays; const bytes = await source.getFile(product.id, product.version, { signal }); signal.throwIfAborted(); const entries = await readZipEntries(bytes); @@ -223,12 +271,16 @@ export function createGhostMapsLayer({ const dataSource = new Cesium.CustomDataSource( `${GHOSTMAPS_LAYER_ID}:${product.id}`, ); - const features = []; + const features = new Map(); const pins = []; + const at = now(); dataSource.entities.suspendEvents(); parsed.features.forEach((feature, index) => { + const dateMs = ghostMapsEventDate(feature.description); + if (isGhostMapsEventExpired(dateMs, at, ageDays)) return; const key = `${product.id}:${index}`; const { style } = feature; + const entities = []; feature.geometries.forEach((geometry, part) => { const id = `${PICK_PREFIX}${key}:${part}`; if (geometry.type === 'point') { @@ -255,7 +307,9 @@ export function createGhostMapsLayer({ disableDepthTestDistance: Number.POSITIVE_INFINITY, }, }); + entities.push(entity); pins.push({ + index, position, get show() { return entity.show; @@ -265,61 +319,76 @@ export function createGhostMapsLayer({ }, }); } else if (geometry.type === 'line') { - dataSource.entities.add({ - id, - polyline: { - positions: Cesium.Cartesian3.fromDegreesArray( - geometry.positions.flat(), - ), - width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), - material: cesiumColor(style.lineColor, color), - clampToGround: true, - }, - }); - } else { - const ring = (points) => - Cesium.Cartesian3.fromDegreesArray(points.flat()); - dataSource.entities.add({ - id, - polygon: { - hierarchy: new Cesium.PolygonHierarchy( - ring(geometry.outer), - geometry.holes.map( - (hole) => new Cesium.PolygonHierarchy(ring(hole)), - ), - ), - material: - style.fill === false - ? Cesium.Color.TRANSPARENT - : cesiumColor( - style.polyColor, - color, - style.polyColor ? undefined : 0.25, - ), - classificationType: Cesium.ClassificationType.BOTH, - }, - }); - if (style.outline !== false) + entities.push( dataSource.entities.add({ - id: `${id}:outline`, + id, polyline: { - positions: ring([...geometry.outer, geometry.outer[0]]), - width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + positions: Cesium.Cartesian3.fromDegreesArray( + geometry.positions.flat(), + ), + width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), material: cesiumColor(style.lineColor, color), clampToGround: true, }, - }); + }), + ); + } else { + const ring = (points) => + Cesium.Cartesian3.fromDegreesArray(points.flat()); + entities.push( + dataSource.entities.add({ + id, + polygon: { + hierarchy: new Cesium.PolygonHierarchy( + ring(geometry.outer), + geometry.holes.map( + (hole) => new Cesium.PolygonHierarchy(ring(hole)), + ), + ), + material: + style.fill === false + ? Cesium.Color.TRANSPARENT + : cesiumColor( + style.polyColor, + color, + style.polyColor ? undefined : 0.25, + ), + classificationType: Cesium.ClassificationType.BOTH, + }, + }), + ); + if (style.outline !== false) + entities.push( + dataSource.entities.add({ + id: `${id}:outline`, + polyline: { + positions: ring([...geometry.outer, geometry.outer[0]]), + width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + material: cesiumColor(style.lineColor, color), + clampToGround: true, + }, + }), + ); } }); - features.push({ + features.set(index, { name: feature.name, folder: feature.folder, description: feature.description, + dateMs, + entities, anchor: anchorOf(feature.geometries[0]), }); }); dataSource.entities.resumeEvents(); - return { version: product.version, dataSource, urls, features, pins }; + return { + version: product.version, + ageDays, + dataSource, + urls, + features, + pins, + }; } catch (error) { urls.forEach((url) => URL.revokeObjectURL(url)); throw error; @@ -330,6 +399,7 @@ export function createGhostMapsLayer({ function sync() { const run = _queue.then(async () => { if (!_enabled || !_viewer || !_catalog) return; + expireEvents(); const signal = _lifetime.signal; const products = new Map(_catalog.products.map((p) => [p.id, p])); for (const id of [..._loaded.keys()]) { @@ -338,13 +408,15 @@ export function createGhostMapsLayer({ } for (const product of _catalog.products) { if (!wanted(product) || signal.aborted) continue; - if (_loaded.get(product.id)?.version === product.version) continue; + const loaded = _loaded.get(product.id); + if (loaded?.version === product.version && loaded.ageDays === _ageDays) + continue; _loading.set(product.id, product.version); _failed.delete(product.id); notifyRow(); try { const next = await load(product, signal); - if (signal.aborted || !_enabled) { + if (signal.aborted || !_enabled || next.ageDays !== _ageDays) { next.urls.forEach((url) => URL.revokeObjectURL(url)); return; } @@ -391,12 +463,28 @@ export function createGhostMapsLayer({ void sync(); } + function setAgeDays(ageDays) { + if (!GHOSTMAPS_AGE_OPTIONS.includes(ageDays) || ageDays === _ageDays) + return; + _ageDays = ageDays; + try { + storage?.setItem(AGE_STORAGE_KEY, JSON.stringify(ageDays)); + } catch { + /* private window or blocked storage: keep the session choice */ + } + expireEvents(); + notifyRow(); + // A wider window must restore previously omitted placemarks from the + // cached KMZ, even though its upstream version has not changed. + return sync(); + } + // ---- selection card ---------------------------------------------------- function featureFor(key) { const [productId, index] = key.split(':'); return { product: _catalog?.products.find((p) => p.id === productId) || null, - feature: _loaded.get(productId)?.features[Number(index)] || null, + feature: _loaded.get(productId)?.features.get(Number(index)) || null, }; } @@ -490,7 +578,7 @@ export function createGhostMapsLayer({ if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX)) return null; const [productId, index] = pickId.slice(PICK_PREFIX.length).split(':'); - return _loaded.get(productId)?.features[Number(index)] + return _loaded.get(productId)?.features.has(Number(index)) ? `${productId}:${index}` : null; } @@ -542,7 +630,7 @@ export function createGhostMapsLayer({ function featureCount() { let count = 0; - for (const item of _loaded.values()) count += item.features.length; + for (const item of _loaded.values()) count += item.features.size; return count; } @@ -563,6 +651,8 @@ export function createGhostMapsLayer({ enable() { _enabled = true; _lifetime = new AbortController(); + if (_expiryTimer === null) + _expiryTimer = setInterval(expireEvents, 60_000); for (const item of _loaded.values()) item.dataSource.show = true; installClickHandler(); if (!_unsubscribe && typeof source.subscribe === 'function') @@ -580,6 +670,8 @@ export function createGhostMapsLayer({ disable() { _enabled = false; + clearInterval(_expiryTimer); + _expiryTimer = null; _lifetime.abort(); _unsubscribe?.(); _unsubscribe = null; @@ -593,6 +685,7 @@ export function createGhostMapsLayer({ async update(_viewerArg, { signal } = {}) { if (!_enabled || signal?.aborted) return false; + expireEvents(); try { const catalog = await source.getCatalog({ signal }); if (!_enabled) return false; @@ -637,37 +730,54 @@ export function createGhostMapsLayer({ }) : t('Connecting to the S2 GhostMaps repository…'); return { - chips: products.map((product) => { - const loaded = _loaded.get(product.id); - const failed = _failed.get(product.id); - return { - id: product.id, - label: t(PRODUCT_LABELS[product.id] || product.title), - active: wanted(product), - busy: _loading.has(product.id), - state: failed ? 'error' : undefined, - color: PRODUCT_COLORS[product.id], - title: [ - product.fileTitle, - loaded - ? tn( - loaded.features.length, - '{count} placemark', - '{count} placemarks', - { - count: formatNumber(loaded.features.length), - }, - ) - : null, - failed ? t(failed) : null, - ] - .filter(Boolean) - .join(' · '), - onClick: () => toggleProduct(product.id), - }; - }), + chips: [ + ...GHOSTMAPS_AGE_OPTIONS.map((days) => ({ + id: `age-${days ?? 'all'}`, + label: + days === null + ? t('All markers') + : t('{days}d', { days: formatNumber(days) }), + title: + days === null + ? t('Show markers of every age') + : t('Maximum event age: {days}d; undated markers are kept', { + days: formatNumber(days), + }), + active: days === _ageDays, + onClick: () => setAgeDays(days), + })), + ...products.map((product) => { + const loaded = _loaded.get(product.id); + const failed = _failed.get(product.id); + return { + id: product.id, + label: t(PRODUCT_LABELS[product.id] || product.title), + active: wanted(product), + busy: _loading.has(product.id), + state: failed ? 'error' : undefined, + color: PRODUCT_COLORS[product.id], + title: [ + product.fileTitle, + loaded + ? tn( + loaded.features.size, + '{count} placemark', + '{count} placemarks', + { + count: formatNumber(loaded.features.size), + }, + ) + : null, + failed ? t(failed) : null, + ] + .filter(Boolean) + .join(' · '), + onClick: () => toggleProduct(product.id), + }; + }), + ], legend: [], - info, + info: `${_ageDays === null ? t('All markers') : t('Event age: {days}d · undated kept', { days: formatNumber(_ageDays) })} · ${info}`, infoTitle: t( 'S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.', ), @@ -677,6 +787,7 @@ export function createGhostMapsLayer({ getStats() { return { count: featureCount(), + ageDays: _ageDays, products: [..._loaded.keys()], commit: _catalog?.commit || null, lastUpdate: _catalog?.committedAt || _catalog?.changedAt || null, @@ -693,7 +804,7 @@ export function createGhostMapsLayer({ if (!_enabled) return []; const out = []; for (const [productId, item] of _loaded) { - for (const feature of item.features) { + for (const feature of item.features.values()) { if (out.length >= maxCount) return out; out.push({ product: productId, diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 51d546017..0b3a580e6 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import * as Cesium from 'cesium'; import { createGhostMapsLayer } from './index.js'; +import { GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP } from './catalog.js'; function element(localName, children = [], value = '', attributes = {}) { const node = { @@ -22,7 +23,7 @@ function element(localName, children = [], value = '', attributes = {}) { return node; } -function documentFixture() { +function documentFixture(reports = [{ name: 'Test report' }]) { return element('document', [ element('kml', [ element('Document', [ @@ -36,11 +37,14 @@ function documentFixture() { '', { id: 'pin' }, ), - element('Placemark', [ - element('name', [], 'Test report'), - element('styleUrl', [], '#pin'), - element('Point', [element('coordinates', [], '12,34')]), - ]), + ...reports.map(({ name, description = '', geometry }) => + element('Placemark', [ + element('name', [], name), + element('description', [], description), + element('styleUrl', [], '#pin'), + geometry || element('Point', [element('coordinates', [], '12,34')]), + ]), + ), ]), ]), ]); @@ -83,11 +87,21 @@ function fixtureKmz() { return new Uint8Array(Buffer.concat([...local, directory, end])); } -function fixture(t, { beforeFileReturns, onCreateUrl } = {}) { +function fixture( + t, + { + beforeFileReturns, + onCreateUrl, + reports, + now, + products, + controls = {}, + } = {}, +) { const previous = Object.getOwnPropertyDescriptor(globalThis, 'DOMParser'); globalThis.DOMParser = class { parseFromString() { - return documentFixture(); + return documentFixture(reports); } }; t.after(() => { @@ -119,9 +133,13 @@ function fixture(t, { beforeFileReturns, onCreateUrl } = {}) { }; const layer = createGhostMapsLayer({ storage: null, + now, + ...controls, source: { getCatalog: async () => ({ - products: [{ id: 'cip', version: 'a'.repeat(40), defaultOn: true }], + products: products ?? [ + { id: 'cip', version: 'a'.repeat(40), defaultOn: true }, + ], }), getFile: async () => { beforeFileReturns?.(layer); @@ -178,3 +196,284 @@ test('successful load keeps icons until the product is disabled', async (t) => { assert.deepEqual(revoked, created); assert.equal(mounted.size, 0); }); + +const dated = (name, value, label = 'Date') => ({ + name, + description: `
${label}${value}
`, +}); + +const ageChip = (layer, days) => + layer.getRowControls().chips.find(({ id }) => id === `age-${days ?? 'all'}`); + +test('age controls default to seven days and widening restores filtered and expired reports', async (t) => { + let at = Date.UTC(2026, 10, 3, 12); + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + dated('Old', '10/1/2026'), + dated('Ten days', '10/24/2026'), + dated('Recent', '11/2/2026'), + { name: 'Undated' }, + ], + }); + await layer.update(viewer); + assert.equal(layer.getStats().ageDays, 7); + assert.equal(ageChip(layer, 7).active, true); + assert.equal(layer.getStats().count, 2); + await ageChip(layer, 14).onClick(); + assert.equal(layer.getStats().count, 3); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().count, 4); + at += 100 * 86400_000; + await layer.update(viewer); + assert.equal(layer.getStats().count, 4, 'all markers never ages out'); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); + assert.equal(layer.getAnalystRecords()[0].name, 'Undated'); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().count, 4); + assert.equal(mounted.size, 1, 'filter changes never duplicate data sources'); + const ageChips = layer + .getRowControls() + .chips.filter(({ id }) => id.startsWith('age-')); + assert.equal(ageChips.filter(({ active }) => active).length, 1); + assert.equal(ageChip(layer, null).active, true); +}); + +test('age preferences persist independently of product choices and reject invalid storage', async (t) => { + const values = new Map([['gev.ghostmaps.products', '["cip"]']]); + const storage = { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => values.set(key, value), + }; + await t.test('save All markers without changing products', async (t) => { + const { layer, viewer } = fixture(t, { controls: { storage } }); + await layer.update(viewer); + await ageChip(layer, null).onClick(); + assert.equal(values.get('gev.ghostmaps.ageDays'), 'null'); + assert.equal(values.get('gev.ghostmaps.products'), '["cip"]'); + }); + await t.test('restore All markers and save thirty days', async (t) => { + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, null); + await ageChip(layer, 30).onClick(); + assert.equal(values.get('gev.ghostmaps.ageDays'), '30'); + }); + await t.test('restore thirty days', (t) => { + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, 30); + }); + for (const value of ['garbage', '"7"', '-1', '0', '2', '{}']) { + await t.test(`reject ${value}`, (t) => { + values.set('gev.ghostmaps.ageDays', value); + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, 7); + }); + } +}); + +test('blocked storage still permits changing the age within the session', async (t) => { + const storage = { + getItem() { + throw new Error('blocked'); + }, + setItem() { + throw new Error('blocked'); + }, + }; + const { layer, viewer } = fixture(t, { controls: { storage } }); + await layer.update(viewer); + assert.equal(layer.getStats().ageDays, 7); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().ageDays, null); +}); + +test('a filter changed during a pending load never installs the superseded filter', async (t) => { + let changed = false; + const { layer, viewer, mounted, revoked, created } = fixture(t, { + now: () => Date.UTC(2026, 10, 3), + reports: [dated('Old', '10/1/2026')], + beforeFileReturns: (layer) => { + if (changed) return; + changed = true; + ageChip(layer, null).onClick(); + }, + }); + await layer.update(viewer); + await layer.update(viewer); + assert.equal(layer.getStats().count, 1); + assert.equal(layer.getStats().ageDays, null); + assert.equal(mounted.size, 1); + assert.equal( + revoked.length, + created.length - 1, + 'discarded load releases icons', + ); + + test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { + for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { + await t.test(product.id, async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + products: [{ ...product, version: 'a'.repeat(40) }], + reports: [ + dated( + 'Old event', + '10/1/2026', + product.id === 'cip' ? 'Date of Event' : 'Date', + ), + dated('Recent event', '11/2/2026'), + { name: 'Undated reference feature' }, + ], + }); + await layer.update(viewer); + if (!product.defaultOn) { + layer + .getRowControls() + .chips.find(({ id }) => id === product.id) + .onClick(); + await layer.update(viewer); + } + assert.deepEqual(layer.getStats().products, [product.id]); + assert.equal(layer.getStats().count, 2); + assert.deepEqual( + layer.getAnalystRecords().map(({ name }) => name), + ['Recent event', 'Undated reference feature'], + ); + await ageChip(layer, null).onClick(); + assert.equal( + layer.getStats().count, + 3, + "All markers restores this product's old events", + ); + await ageChip(layer, 7).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal(mounted.size, 1); + }); + } + }); +}); + +test('default products filter old event dates while keeping recent, future and undated markers', async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [ + dated('Old event', '10/20/2026'), + dated('Recent event', '11/2/2026'), + dated('At cutoff', '10/27/2026 12:00:00 PM'), + dated('Before cutoff', '10/27/2026 11:59:59 AM'), + dated('Future event', '11/4/2026'), + { name: 'Undated site' }, + dated('Blank date', '<Null>'), + dated('Invalid date', '2/30/2026'), + dated('Only a report date', '10/1/2026', 'Date Of Report'), + { + name: 'Old event reported recently', + description: + '
Date of Event10/1/2026
Date Of Report11/2/2026
', + }, + ], + }); + await layer.update(viewer); + assert.deepEqual(layer.getStats().products, ['cip']); + assert.deepEqual( + layer.getAnalystRecords().map((r) => r.name), + [ + 'Recent event', + 'At cutoff', + 'Future event', + 'Undated site', + 'Blank date', + 'Invalid date', + 'Only a report date', + ], + ); + assert.equal(layer.getStats().count, 7); + assert.equal([...mounted][0].entities.values.length, 7); + // Filtering cannot shift pick ids onto a different report. + assert.ok([...mounted][0].entities.getById('ghostmaps:cip:1:0')); + assert.equal( + [...mounted][0].entities.getById('ghostmaps:cip:0:0'), + undefined, + ); +}); + +test('unchanged cached products age out on the timer, clear selected cards and keep surviving picks', async (t) => { + t.mock.timers.enable({ apis: ['setInterval'] }); + let at = Date.UTC(2026, 10, 3, 12); + let click; + const cards = []; + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + dated('Expiring', '10/27/2026 12:00:00 PM'), + dated('Surviving', '11/2/2026'), + ], + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + const ds = [...mounted][0]; + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + at += 60_000; + t.mock.timers.tick(60_000); + assert.equal(layer.getStats().count, 1); + assert.equal(ds.entities.values.length, 1); + assert.equal(ds.entities.getById('ghostmaps:cip:0:0'), undefined); + assert.ok(ds.entities.getById('ghostmaps:cip:1:0')); + assert.deepEqual(cards.at(-1), []); + await layer.update(viewer); + assert.equal([...mounted][0], ds, 'same-version KMZ is not reloaded'); + viewer.scene.pick = () => 'ghostmaps:cip:1:0'; + click({ position: { x: 0, y: 0 } }); + assert.match(cards.at(-1)[0].title, /Surviving/); + layer.disable(); + at += 14 * 86400_000; + t.mock.timers.tick(60_000); + assert.equal(mounted.size, 0); + layer.enable(); + await layer.update(viewer); + assert.equal(layer.getStats().count, 0); +}); + +test('expiry removes every geometry and polygon outline belonging to an old report', async (t) => { + t.mock.timers.enable({ apis: ['setInterval'] }); + let at = Date.UTC(2026, 10, 3); + const polygon = element('Polygon', [ + element('outerBoundaryIs', [ + element('LinearRing', [ + element('coordinates', [], '12,34 13,34 13,35 12,34'), + ]), + ]), + ]); + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + { + ...dated('Multi-part', '10/27/2026'), + geometry: element('MultiGeometry', [ + element('Point', [element('coordinates', [], '12,34')]), + element('LineString', [element('coordinates', [], '12,34 13,35')]), + polygon, + ]), + }, + ], + }); + await layer.update(viewer); + const ds = [...mounted][0]; + assert.equal(ds.entities.values.length, 4); + at += 60_000; + t.mock.timers.tick(60_000); + assert.equal(ds.entities.values.length, 0); + assert.equal(layer.getStats().count, 0); +}); diff --git a/src/layers/ghostMaps/records.js b/src/layers/ghostMaps/records.js index 7c4eb58bf..3c307fa63 100644 --- a/src/layers/ghostMaps/records.js +++ b/src/layers/ghostMaps/records.js @@ -1,4 +1,4 @@ -// Pure helpers for GhostMaps placemarks: description tables to plain fields, +// Pure helpers for GhostMaps placemarks: report fields and event dates, // KML colors, and card text wrapping. Strings in, data out — description // markup is only ever read as text, never inserted into the page. @@ -51,6 +51,104 @@ const MAX_FIELDS = 24; const MAX_VALUE = 600; const MAX_LINKS = 6; +// Use the same leaf rows for report cards and event dates, without the card's +// field-count/value-length limits affecting the age filter. +function* descriptionRows(source) { + for (const row of source.matchAll( + /]*>((?:(?!/gi, + )) { + const cells = [ + ...row[1].matchAll(/]*>([\s\S]*?)<\/t[dh]\s*>/gi), + ].map((cell) => cell[1]); + if (cells.length === 2) + yield { + label: htmlToText(cells[0]), + value: htmlToText(cells[1]), + markup: cells[1], + }; + } +} + +export const GHOSTMAPS_DEFAULT_AGE_DAYS = 7; +export const GHOSTMAPS_AGE_OPTIONS = Object.freeze([1, 3, 7, 14, 30, null]); + +/** ArcGIS US dates (optionally with a time) and ISO dates, interpreted in UTC + * when no zone is supplied. Reject calendar rollovers instead of guessing. */ +export function parseGhostMapsEventDate(value) { + const input = String(value ?? '').trim(); + const us = + /^(\d{1,2})\/(\d{1,2})\/(\d{4})(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM))?$/i.exec( + input, + ); + const iso = + /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:\d{2})?)?$/.exec( + input, + ); + if (!us && !iso) return null; + const year = Number(us ? us[3] : iso[1]); + const month = Number(us ? us[1] : iso[2]); + const day = Number(us ? us[2] : iso[3]); + let hour = Number(us?.[4] ?? iso?.[4] ?? 0); + const minute = Number(us?.[5] ?? iso?.[5] ?? 0); + const second = Number(us?.[6] ?? iso?.[6] ?? 0); + if (us?.[7]) { + if (hour < 1 || hour > 12) return null; + hour = (hour % 12) + (us[7].toUpperCase() === 'PM' ? 12 : 0); + } + const at = Date.UTC(year, month - 1, day, hour, minute, second); + const date = new Date(at); + if ( + year < 1900 || + month < 1 || + month > 12 || + day < 1 || + date.getUTCDate() !== day || + hour > 23 || + minute > 59 || + second > 59 + ) + return null; + if (iso?.[4]) { + const zoned = /(?:Z|[+-]\d{2}:\d{2})$/.test(input) ? input : `${input}Z`; + const parsed = Date.parse(zoned); + return Number.isFinite(parsed) ? parsed : null; + } + return at; +} + +/** The event's date, never its report/upload date or dates mentioned in prose. */ +export function ghostMapsEventDate(description) { + const rows = [ + ...descriptionRows( + String(description ?? '').replace( + /<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi, + ' ', + ), + ), + ]; + for (const name of ['dateofevent', 'eventdate', 'date']) { + for (const { label, value } of rows) { + if (label.toLowerCase().replace(/[\s_]+/g, '') !== name) continue; + const at = parseGhostMapsEventDate(value); + if (at !== null) return at; + } + } + return null; +} + +/** Unknown and future dates stay visible; only a confirmed old event expires. */ +export function isGhostMapsEventExpired( + dateMs, + now = Date.now(), + ageDays = GHOSTMAPS_DEFAULT_AGE_DAYS, +) { + return ( + ageDays !== null && + Number.isFinite(dateMs) && + dateMs < now - ageDays * 86400_000 + ); +} + /** * Fields and links from a placemark description. ArcGIS "popup" exports are a * two-column table (label, value); anything else becomes one unlabelled field. @@ -73,18 +171,10 @@ export function parseDescriptionFields(html) { }; // Innermost rows only: ArcGIS nests the field table inside an outer row, // and only leaf rows with exactly two cells are label/value pairs. - for (const row of source.matchAll( - /]*>((?:(?!/gi, - )) { - const cells = [ - ...row[1].matchAll(/]*>([\s\S]*?)<\/t[dh]\s*>/gi), - ].map((cell) => cell[1]); - if (cells.length !== 2) continue; - const label = htmlToText(cells[0]); - const value = htmlToText(cells[1]); + for (const { label, value, markup } of descriptionRows(source)) { if (!label || SKIPPED_LABELS.has(label.toLowerCase())) continue; if (EMPTY_VALUES.has(value.toLowerCase())) continue; - addLinks(cells[1]); + addLinks(markup); // A bare research link is carried by `links`; the card does not need // the URL twice. if (/^https?:\/\/\S+$/i.test(value)) continue; diff --git a/src/layers/ghostMaps/records.test.mjs b/src/layers/ghostMaps/records.test.mjs index 86fecb52b..a28d1f36d 100644 --- a/src/layers/ghostMaps/records.test.mjs +++ b/src/layers/ghostMaps/records.test.mjs @@ -7,8 +7,83 @@ import { kmlColor, parseDescriptionFields, wrapCardText, + ghostMapsEventDate, + isGhostMapsEventExpired, + parseGhostMapsEventDate, } from './records.js'; +test('event dates parse ArcGIS calendar dates and timestamps consistently in UTC', () => { + for (const [value, iso] of [ + ['10/28/2026', '2026-10-28T00:00:00Z'], + ['10/28/2026 6:12:00 PM', '2026-10-28T18:12:00Z'], + ['1/17/2026 9:46:41 AM', '2026-01-17T09:46:41Z'], + ['10/28/2026 12:00 AM', '2026-10-28T00:00:00Z'], + ['10/28/2026 12:00 PM', '2026-10-28T12:00:00Z'], + ['2026-10-28', '2026-10-28T00:00:00Z'], + ['2026-10-28T12:34:56', '2026-10-28T12:34:56Z'], + ['2026-10-28T12:34:56.123-04:00', '2026-10-28T16:34:56.123Z'], + ['2/29/2024', '2024-02-29T00:00:00Z'], + ]) + assert.equal(parseGhostMapsEventDate(value), Date.parse(iso), value); + for (const value of [ + '', + '', + 'N/A', + 'unknown', + '2/29/2026', + '9/31/2026', + '2026-02-30', + '13/1/2026', + '1/0/2026', + '10/28/2026 13:00 PM', + '2026-10-28T24:00:00Z', + '2026-10-28T12:60:00Z', + ]) { + assert.equal(parseGhostMapsEventDate(value), null, value); + } +}); + +test('event date fields take precedence over report dates and prose, even after long report tables', () => { + const row = (label, value) => `${label}${value}`; + const html = `
${ + row('Date Of Report', '11/2/2026') + + row('Description', 'Incident mentioned on 11/1/2026') + + Array.from({ length: 30 }, (_, i) => row(`Field${i}`, 'value')).join('') + + row('Date', '10/30/2026') + + row('Date of Event', '10/28/2026') + }
`; + assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 28)); + assert.equal( + ghostMapsEventDate( + `${row('Date', '<Null>')}${row('Date Of Report', '11/2/2026')}
`, + ), + null, + ); + assert.equal( + ghostMapsEventDate(`${row('Date_Of_Event', '10/28/2026')}
`), + Date.UTC(2026, 9, 28), + ); + assert.equal(ghostMapsEventDate('Occurred 10/28/2026'), null); + assert.equal( + ghostMapsEventDate( + ``, + ), + null, + ); +}); + +test('seven-day expiry keeps the exact boundary, future dates and unknown dates', () => { + const at = Date.UTC(2026, 10, 3, 12); + assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12), at), false); + assert.equal( + isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12) - 1, at), + true, + ); + assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 10, 4), at), false); + for (const value of [null, undefined, NaN]) + assert.equal(isGhostMapsEventExpired(value, at), false); +}); + // The shape ArcGIS Pro writes into every S2 placemark description. const ARCGIS = ` @@ -27,6 +102,14 @@ const ARCGIS = ` function changeImage(a, b) { document.getElementById('x').innerHTML = b; } `; +test('custom ages change the cutoff and all markers disables expiry', () => { + const at = Date.UTC(2026, 10, 3, 12); + const event = at - 10 * 86400_000; + assert.equal(isGhostMapsEventExpired(event, at, 7), true); + assert.equal(isGhostMapsEventExpired(event, at, 14), false); + assert.equal(isGhostMapsEventExpired(event, at, null), false); +}); + test('ArcGIS popup tables become plain label/value fields and http links', () => { const { fields, links } = parseDescriptionFields(ARCGIS); assert.deepEqual(fields, [ From 134597c3cf53aab6252fce260e1fa8cdc0f8c6ad Mon Sep 17 00:00:00 2001 From: Dylan Maniatakes Date: Tue, 3 Nov 2026 13:23:40 -0500 Subject: [PATCH 14/27] docs: add S2 age filter changelog for PR 1528 --- changelog.d/1528-s2-ghostmaps-age-filter.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1528-s2-ghostmaps-age-filter.md diff --git a/changelog.d/1528-s2-ghostmaps-age-filter.md b/changelog.d/1528-s2-ghostmaps-age-filter.md new file mode 100644 index 000000000..2ba019286 --- /dev/null +++ b/changelog.d/1528-s2-ghostmaps-age-filter.md @@ -0,0 +1,4 @@ +- Add an adjustable event-age filter across S2 GhostMaps products, defaulting + to seven days with 1-, 3-, 14- and 30-day choices and an All markers option. + Preserve undated features, save the selected window and expire old events + while the map stays open. Thanks @dylanmaniatakes (#1528). From 1f0a8b5f1640921d0742cd94cdd8520310a90e2f Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:58:09 +0100 Subject: [PATCH 15/27] fix(datacenters): report newly tagged features osmium cannot export A still-tagged feature without exportable geometry already keeps its old line. A newly tagged one had nowhere to go and was dropped silently; it is now listed as skipped with a warning, so a refresh never loses a site or hides a broken one. --- scripts/build-datacenters-norway.py | 10 ++++++++-- src/data/datacentersNorwayRefresh.test.mjs | 3 ++- src/data/local_data/datacenters/README.md | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py index 03286b864..d1203f38d 100644 --- a/scripts/build-datacenters-norway.py +++ b/scripts/build-datacenters-norway.py @@ -182,12 +182,13 @@ def feature_line(key, sequence, fresh): def merge(lines, fresh, rings, tagged): - out, seen, sequence = [], set(), 0 - report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': []} + out, seen, known, sequence = [], set(), set(), 0 + report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': [], 'skipped': []} for line in lines: feature = json.loads(line) sequence = max(sequence, feature['properties']['id']) key = osm_key(feature) + known.add(key) if key in fresh: seen.add(key) old_tags = feature['properties']['tags'] @@ -211,6 +212,8 @@ def merge(lines, fresh, rings, tagged): sequence += 1 out.append(feature_line(key, sequence, fresh[key])) report['added'].append(key) + # newly tagged, but osmium built no usable geometry: nothing to add yet + report['skipped'] = sorted(set(tagged) - set(fresh) - known, key=lambda k: (k[0], int(k[1:]))) if len({osm_key(json.loads(line)) for line in out}) != len(out): raise ValueError('Duplicate OSM identity') return out, report @@ -239,6 +242,9 @@ def main(): if report['unexported']: print('warning: kept the old line of tagged features osmium did not export: %s' % ', '.join(report['unexported']), file=sys.stderr) + if report['skipped']: + print('warning: did not add newly tagged features osmium did not export: %s' + % ', '.join(report['skipped']), file=sys.stderr) payload = '\n'.join(merged) + '\n' if args.verify: if payload != DATASET.read_text(encoding='utf-8'): diff --git a/src/data/datacentersNorwayRefresh.test.mjs b/src/data/datacentersNorwayRefresh.test.mjs index 2b7b7d5bf..88527295b 100644 --- a/src/data/datacentersNorwayRefresh.test.mjs +++ b/src/data/datacentersNorwayRefresh.test.mjs @@ -52,9 +52,10 @@ def line(osm_id, sequence): 'type': 'data_center', 'osm_id': osm_id}}) norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)] lines = [line(7, 1), line(8, 2)] -out, report = merge(lines, {}, norway, {'w7'}) +out, report = merge(lines, {}, norway, {'w7', 'w12'}) assert out == [lines[0]], out assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report +assert report['skipped'] == ['w12'] and report['added'] == [], report `, ); }); diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index 5a80df267..86f573c61 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -29,7 +29,8 @@ reflects OSM as of the extract below, everything else the original snapshot. - Contact tags are dropped as above; unchanged features keep their original line, so geometry is only rewritten when tags or rings actually changed. A feature is removed only once it is no longer tagged; one still tagged but - without an exportable geometry keeps its line and is reported as unexported. + without an exportable geometry keeps its line and is reported as unexported, + and a newly tagged one osmium cannot export is reported as skipped. - Refresh: `python3 scripts/build-datacenters-norway.py --cache ` uses Geofabrik's current Norway extract and prints the dated name it used; record that name, its OSM timestamp and MD5 here. Requires `osmium-tool`. From e4fda9d4e6d7f0d5c8cc8bfba82dd33d959dbc64 Mon Sep 17 00:00:00 2001 From: Mert Kaya <47145541+mertkayacs@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:39:34 +0300 Subject: [PATCH 16/27] fix(geocode): preserve non-Latin names in keyless search --- changelog.d/1585-keyless-unicode-search.md | 3 + docs/CURRENT-STATE.md | 6 +- scripts/package-boundaries.json | 1 + src/data/fixtures/README.md | 5 + src/data/fixtures/photon-moscow-search.json | 110 ++++++++++++++++++++ src/keylessGeocoder.js | 16 ++- src/keylessGeocoder.test.mjs | 37 ++++++- 7 files changed, 166 insertions(+), 12 deletions(-) create mode 100644 changelog.d/1585-keyless-unicode-search.md create mode 100644 src/data/fixtures/photon-moscow-search.json diff --git a/changelog.d/1585-keyless-unicode-search.md b/changelog.d/1585-keyless-unicode-search.md new file mode 100644 index 000000000..276ade93f --- /dev/null +++ b/changelog.d/1585-keyless-unicode-search.md @@ -0,0 +1,3 @@ +- Keep keyless place search on a nearby exact match for names written in + non-Latin scripts instead of discarding the match and retrying without its + proximity ranking (Mert Kaya, @mertkayacs, #PRNUM). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index f1244947d..45fe2999e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5720,8 +5720,10 @@ HTTP refusals and outages remain retryable. Searches share a 12-second total deadline, with Photon requests capped at six seconds each. Caller/application cancellation stops retries and late cache writes. Replacing a location search cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five -candidates, and an unbiased retry for name mismatches. A "place, region" query -prefers a candidate named exactly for the place whose city, county, state, +candidates, and an unbiased retry for name mismatches. Name comparison folds +Latin accents while preserving letters and marks in other scripts, so an exact +non-Latin name keeps its proximity ranking. A "place, region" query prefers a +candidate named exactly for the place whose city, county, state, country or country code (or USA/UK) contains every word of the region, and a settlement or administrative area over a POI when both fit. Nearby results that fail that check are retried unbiased. After the region checks of both passes, a diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index d008ceaaa..72652d8da 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -552,6 +552,7 @@ "src/search/placeSearch.js", "src/search/google.js", "src/keylessGeocoder.js", + "src/data/placeNames.js", "src/search/geospatial.js", "src/search/http.js", "src/search/defaults.js", diff --git a/src/data/fixtures/README.md b/src/data/fixtures/README.md index e9dd55f50..257eaf8a5 100644 --- a/src/data/fixtures/README.md +++ b/src/data/fixtures/README.md @@ -1,5 +1,10 @@ # Test fixtures +- `photon-moscow-search.json` - Photon responses for `Москва`, captured + 2026-11-05 with and without a proximity bias near the Tajikistan namesake. + The fixture is trimmed to the fields the keyless geocoder consumes and is + used only by offline search tests. Data from OpenStreetMap contributors, + ODbL 1.0 (https://www.openstreetmap.org/copyright). - `tomtom-flow-austin-12-935-1686.pbf` — one real TomTom traffic-flow vector tile (Mapbox Vector Tile protobuf, layer `"Traffic flow"`), downtown Austin z12 x935 y1686, captured 2026-07-16 from diff --git a/src/data/fixtures/photon-moscow-search.json b/src/data/fixtures/photon-moscow-search.json new file mode 100644 index 000000000..7b1a4d25a --- /dev/null +++ b/src/data/fixtures/photon-moscow-search.json @@ -0,0 +1,110 @@ +{ + "biased": { + "features": [ + { + "geometry": { + "type": "Point", + "coordinates": [71.8339931, 38.948412] + }, + "properties": { + "name": "Москва", + "country": "Тоҷикистон" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6063916, 55.625578] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6174782, 55.7505412] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.0536908, 55.7098009] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [32.163334, 56.917778] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + } + ] + }, + "unbiased": { + "features": [ + { + "geometry": { + "type": "Point", + "coordinates": [37.6063916, 55.625578] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.6174782, 55.7505412] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [37.0536908, 55.7098009] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [32.163334, 56.917778] + }, + "properties": { + "name": "Москва", + "country": "Россия" + } + }, + { + "geometry": { + "type": "Point", + "coordinates": [71.8339931, 38.948412] + }, + "properties": { + "name": "Москва", + "country": "Тоҷикистон" + } + } + ] + } +} diff --git a/src/keylessGeocoder.js b/src/keylessGeocoder.js index 4e01f1c74..eb881f970 100644 --- a/src/keylessGeocoder.js +++ b/src/keylessGeocoder.js @@ -1,3 +1,5 @@ +import { foldPlaceName } from './data/placeNames.js'; + /** Photon/OpenStreetMap place-search adapter. */ const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/'; @@ -233,19 +235,15 @@ export function photonSearchUrl( } /** - * Place name reduced to comparable form: unaccented, lowercased, punctuation - * collapsed to single spaces. "Huế" and "Hue" have to compare equal, because a - * user typing either means the same city. + * Place name reduced to comparable form: lowercased, Latin accents removed, + * other scripts preserved, and punctuation collapsed to single spaces. "Huế" + * and "Hue" have to compare equal, because a user typing either means the same + * city. * @param {string} text * @returns {string} */ export function normalizeToponym(text) { - return String(text ?? '') - .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - .toLowerCase() - .replace(/[^a-z0-9]+/g, ' ') - .trim(); + return foldPlaceName(text); } /** diff --git a/src/keylessGeocoder.test.mjs b/src/keylessGeocoder.test.mjs index 95098a10c..e4c762f62 100644 --- a/src/keylessGeocoder.test.mjs +++ b/src/keylessGeocoder.test.mjs @@ -35,6 +35,12 @@ import { import { geocodeNavigationMode } from './locations.js'; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const MOSCOW_RESPONSES = JSON.parse( + fs.readFileSync( + path.join(ROOT, 'src/data/fixtures/photon-moscow-search.json'), + 'utf8', + ), +); /** Photon's real answer for "Hanoi", trimmed to the fields we consume. */ const HANOI = { @@ -223,7 +229,6 @@ const HUE = { extent: [107.3763, 16.7739, 108.0518, 16.1276], }, }; - /** Answers biased requests with `nearby`, unbiased ones with `anywhere`. */ function photonPair(nearby, anywhere) { const urls = []; @@ -249,6 +254,36 @@ test('a biased near-miss is refused, and the unbiased answer is taken instead', assert.equal(fetchImpl.urls.length, 2, 'the refused bias costs exactly one extra request'); }); +test('a non-Latin exact match keeps the nearby namesake', async () => { + const fetchImpl = photonPair( + MOSCOW_RESPONSES.biased.features, + MOSCOW_RESPONSES.unbiased.features, + ); + + const result = await geocodeKeyless('Москва', { + bias: '38.90,71.80|39.00,71.90', + fetchImpl, + }); + + assert.equal(result.label, 'Москва, Тоҷикистон'); + assert.equal(fetchImpl.urls.length, 1); +}); + +test('toponym folding preserves non-Latin scripts', () => { + const cases = [ + ['Huế', 'hue'], + ['Москва', 'москва'], + ['Αθήνα', 'αθήνα'], + ['القاهرة', 'القاهرة'], + ['東京', '東京'], + ['São 東京', 'sao 東京'], + ]; + + for (const [input, expected] of cases) { + assert.equal(normalizeToponym(input), expected, input); + } +}); + test('a biased match that leads with the name is taken without a second request', async () => { // The other half of the rule, and the reason bias exists at all: "Sixth // Street" over Austin must stay Austin's. From bc65c274da4418edc80df8d3b4fa3b1ed4bb42aa Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 5 Nov 2026 17:13:00 +0000 Subject: [PATCH 17/27] fix(geocode): keep a leading "the" in keyless name matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit foldPlaceName drops a leading "the", so "The Hague" over upstate New York accepted Hague, NY as an exact nearby match. Split the script-aware fold into foldPlaceText and use it for keyless toponyms; foldPlaceName keeps its behaviour for the packs and street matchers. Also: changelog fragment credits #1585, folding test pins the Latin letters NFD does not decompose (ß, ø, ł), and CURRENT-STATE notes that non-Latin names match only when spelled with their marks. --- changelog.d/1585-keyless-unicode-search.md | 2 +- docs/CURRENT-STATE.md | 5 ++-- src/data/placeNames.js | 28 +++++++++++++++------- src/keylessGeocoder.js | 4 ++-- src/keylessGeocoder.test.mjs | 23 ++++++++++++++++++ 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/changelog.d/1585-keyless-unicode-search.md b/changelog.d/1585-keyless-unicode-search.md index 276ade93f..1f8b3af41 100644 --- a/changelog.d/1585-keyless-unicode-search.md +++ b/changelog.d/1585-keyless-unicode-search.md @@ -1,3 +1,3 @@ - Keep keyless place search on a nearby exact match for names written in non-Latin scripts instead of discarding the match and retrying without its - proximity ranking (Mert Kaya, @mertkayacs, #PRNUM). + proximity ranking (mertkayacs, #1585). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 45fe2999e..d60b351cc 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5721,8 +5721,9 @@ deadline, with Photon requests capped at six seconds each. Caller/application cancellation stops retries and late cache writes. Replacing a location search cancels the previous lookup; disposing its controls cancels the active lookup. Photon uses a soft proximity bias, up to five candidates, and an unbiased retry for name mismatches. Name comparison folds -Latin accents while preserving letters and marks in other scripts, so an exact -non-Latin name keeps its proximity ranking. A "place, region" query prefers a +Latin accents while preserving letters and marks in other scripts, so a +non-Latin name spelled exactly, marks included, keeps its proximity ranking. A +leading "the" counts: "The Hague" does not take a nearby "Hague". A "place, region" query prefers a candidate named exactly for the place whose city, county, state, country or country code (or USA/UK) contains every word of the region, and a settlement or administrative area over a POI when both fit. Nearby results that diff --git a/src/data/placeNames.js b/src/data/placeNames.js index 300d09209..90b458dd3 100644 --- a/src/data/placeNames.js +++ b/src/data/placeNames.js @@ -11,17 +11,16 @@ const FOLD = { }; /** - * Comparable form of a place name, shared by the bundled packs, the - * OpenFreeMap street and grounds matchers and the Nominatim server: case - * folded, accents removed from Latin letters only, punctuation and symbols - * collapsed to single spaces, and a leading "the" dropped. Other scripts keep - * their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while "São Paulo", - * "sao paulo" and "SÃO PAULO!" meet. + * Script-aware fold of a place name: case folded, accents removed from Latin + * letters only, punctuation and symbols collapsed to single spaces. Other + * scripts keep their marks, so "कुल मार्ग" and "काली मार्ग" stay distinct while + * "São Paulo", "sao paulo" and "SÃO PAULO!" meet. Every word is kept: "The + * Hague" is not "Hague". * * @param {unknown} value * @returns {string} */ -export function foldPlaceName(value) { +export function foldPlaceText(value) { return String(value ?? '') .toLowerCase() .replace(/[ßøæłđıœþ]/g, (c) => FOLD[c]) @@ -29,8 +28,19 @@ export function foldPlaceName(value) { .replace(/(\p{Script=Latin})\p{M}+/gu, '$1') .normalize('NFKC') .replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ') - .trim() - .replace(/^the /, ''); + .trim(); +} + +/** + * Comparable form of a place name, shared by the bundled packs, the + * OpenFreeMap street and grounds matchers and the Nominatim server: + * `foldPlaceText` with a leading "the" dropped. + * + * @param {unknown} value + * @returns {string} + */ +export function foldPlaceName(value) { + return foldPlaceText(value).replace(/^the /, ''); } /** diff --git a/src/keylessGeocoder.js b/src/keylessGeocoder.js index eb881f970..3a5cd0035 100644 --- a/src/keylessGeocoder.js +++ b/src/keylessGeocoder.js @@ -1,4 +1,4 @@ -import { foldPlaceName } from './data/placeNames.js'; +import { foldPlaceText } from './data/placeNames.js'; /** Photon/OpenStreetMap place-search adapter. */ const PHOTON_ENDPOINT = 'https://photon.komoot.io/api/'; @@ -243,7 +243,7 @@ export function photonSearchUrl( * @returns {string} */ export function normalizeToponym(text) { - return foldPlaceName(text); + return foldPlaceText(text); } /** diff --git a/src/keylessGeocoder.test.mjs b/src/keylessGeocoder.test.mjs index e4c762f62..7f50ebc2e 100644 --- a/src/keylessGeocoder.test.mjs +++ b/src/keylessGeocoder.test.mjs @@ -229,6 +229,7 @@ const HUE = { extent: [107.3763, 16.7739, 108.0518, 16.1276], }, }; + /** Answers biased requests with `nearby`, unbiased ones with `anywhere`. */ function photonPair(nearby, anywhere) { const urls = []; @@ -277,6 +278,10 @@ test('toponym folding preserves non-Latin scripts', () => { ['القاهرة', 'القاهرة'], ['東京', '東京'], ['São 東京', 'sao 東京'], + ['Straße', 'strasse'], + ['Tromsø', 'tromso'], + ['Łódź', 'lodz'], + ['The Hague', 'the hague'], ]; for (const [input, expected] of cases) { @@ -284,6 +289,24 @@ test('toponym folding preserves non-Latin scripts', () => { } }); +test('a leading "the" is part of the name a nearby match must equal', async () => { + // "The Hague" over upstate New York means the Dutch city, not Hague, NY. + const hagueNy = { + geometry: { type: 'Point', coordinates: [-73.5004, 43.6967] }, + properties: { osm_key: 'place', osm_value: 'town', type: 'city', name: 'Hague', country: 'United States' }, + }; + const theHague = { + geometry: { type: 'Point', coordinates: [4.3113, 52.0799] }, + properties: { osm_key: 'place', osm_value: 'city', type: 'city', name: 'The Hague', country: 'Nederland' }, + }; + const fetchImpl = photonPair([hagueNy], [theHague]); + + const result = await geocodeKeyless('The Hague', { bias: '43.65,-73.55|43.75,-73.45', fetchImpl }); + + assert.equal(Math.round(result.lat), 52, `got ${result.label}`); + assert.equal(fetchImpl.urls.length, 2); +}); + test('a biased match that leads with the name is taken without a second request', async () => { // The other half of the rule, and the reason bias exists at all: "Sixth // Street" over Austin must stay Austin's. From 067383fda80b7bbc66bf792275433df771910426 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Thu, 5 Nov 2026 21:42:22 +0400 Subject: [PATCH 18/27] fix(datacenters): drop the redundant seen set and test skipped against exports The merge test now passes an exported newly tagged feature, so removing '- set(fresh)' from the skipped report fails it. --- scripts/build-datacenters-norway.py | 5 ++--- src/data/datacentersNorwayRefresh.test.mjs | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py index d1203f38d..3c5eb92dd 100644 --- a/scripts/build-datacenters-norway.py +++ b/scripts/build-datacenters-norway.py @@ -182,7 +182,7 @@ def feature_line(key, sequence, fresh): def merge(lines, fresh, rings, tagged): - out, seen, known, sequence = [], set(), set(), 0 + out, known, sequence = [], set(), 0 report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': [], 'skipped': []} for line in lines: feature = json.loads(line) @@ -190,7 +190,6 @@ def merge(lines, fresh, rings, tagged): key = osm_key(feature) known.add(key) if key in fresh: - seen.add(key) old_tags = feature['properties']['tags'] if old_tags == fresh[key]['tags'] and same_geometry(feature['geometry'], fresh[key]['geometry']): out.append(line) @@ -208,7 +207,7 @@ def merge(lines, fresh, rings, tagged): report['removed'].append(key) continue out.append(line) - for key in sorted(set(fresh) - seen, key=lambda k: (k[0], int(k[1:]))): + for key in sorted(set(fresh) - known, key=lambda k: (k[0], int(k[1:]))): sequence += 1 out.append(feature_line(key, sequence, fresh[key])) report['added'].append(key) diff --git a/src/data/datacentersNorwayRefresh.test.mjs b/src/data/datacentersNorwayRefresh.test.mjs index 88527295b..99278769e 100644 --- a/src/data/datacentersNorwayRefresh.test.mjs +++ b/src/data/datacentersNorwayRefresh.test.mjs @@ -52,10 +52,11 @@ def line(osm_id, sequence): 'type': 'data_center', 'osm_id': osm_id}}) norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)] lines = [line(7, 1), line(8, 2)] -out, report = merge(lines, {}, norway, {'w7', 'w12'}) -assert out == [lines[0]], out +fresh = {'w13': {'geometry': {'type': 'Polygon', 'coordinates': square}, 'tags': {'building': 'data_center'}}} +out, report = merge(lines, fresh, norway, {'w7', 'w12', 'w13'}) +assert out[0] == lines[0] and len(out) == 2, out assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report -assert report['skipped'] == ['w12'] and report['added'] == [], report +assert report['skipped'] == ['w12'] and report['added'] == ['w13'], report `, ); }); From 5d8f2b878f5ef204b5efb4519a7fbbbc3699b59f Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 15:47:36 +0400 Subject: [PATCH 19/27] chore(datacenters): drop the one-off Norway refresh script The data refresh is committed and its provenance (extract, timestamp, MD5, filter) stays in the dataset README. The script needed osmium-tool and could only re-verify while Geofabrik still listed that extract, so it goes with its test. --- changelog.d/953-datacenters-norway-refresh.md | 6 +- scripts/build-datacenters-norway.py | 259 ------------------ src/data/datacentersNorwayRefresh.test.mjs | 95 ------- src/data/local_data/datacenters/README.md | 12 +- 4 files changed, 3 insertions(+), 369 deletions(-) delete mode 100644 scripts/build-datacenters-norway.py delete mode 100644 src/data/datacentersNorwayRefresh.test.mjs diff --git a/changelog.d/953-datacenters-norway-refresh.md b/changelog.d/953-datacenters-norway-refresh.md index 48a019605..5db897234 100644 --- a/changelog.d/953-datacenters-norway-refresh.md +++ b/changelog.d/953-datacenters-norway-refresh.md @@ -1,6 +1,4 @@ - Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract (2026-10-05). Troll Housing joins the layer (4,352 features) and four - Norwegian sites pick up current OSM tags; no site moved or was removed. - `scripts/build-datacenters-norway.py` reproduces or `--verify`s the merge, - touching only features inside Geofabrik's Norway boundary, and the dataset - README records the extract, checksum and filter (trygvehhh, #953). + Norwegian sites pick up current OSM tags; no site moved or was removed + (trygvehhh, #953). diff --git a/scripts/build-datacenters-norway.py b/scripts/build-datacenters-norway.py deleted file mode 100644 index 3c5eb92dd..000000000 --- a/scripts/build-datacenters-norway.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python3 -"""Maintainer-only Norway refresh of the bundled datacenter extract (osmium-tool). - -Downloads a dated Geofabrik Norway extract, verifies its MD5, filters -telecom=data_center / building=data_center with osmium, and merges the result -into datacenters.geojsonl: features inside the Geofabrik Norway boundary are -added, updated or removed once no longer tagged; every other line is kept -byte-for-byte. - -Geofabrik keeps only its newest dated extract, so --extract defaults to the -current one and a recorded extract can be re-verified only while it is listed. -""" -import argparse -import hashlib -import json -import math -from pathlib import Path -import shutil -import subprocess -import sys -import urllib.request - -ROOT = Path(__file__).resolve().parent.parent -GEOFABRIK = 'https://download.geofabrik.de/europe/' -DATASET = ROOT / 'src/data/local_data/datacenters/datacenters.geojsonl' -CONTACT_TAGS = {'email', 'phone', 'fax', 'mobile', 'whatsapp', 'contact:email', 'contact:phone', - 'contact:fax', 'contact:mobile', 'contact:whatsapp'} -SAME_GEOMETRY_METERS = 0.05 -TIMEOUT_SECONDS = 60 - - -def latest_extract(): - """Resolve norway-latest to the dated name Geofabrik redirects it to.""" - request = urllib.request.Request(GEOFABRIK + 'norway-latest.osm.pbf', method='HEAD') - with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS) as response: - name = response.geturl().rsplit('/', 1)[-1] - if not name.startswith('norway-') or not name.endswith('.osm.pbf') or name == 'norway-latest.osm.pbf': - raise RuntimeError('Unexpected Geofabrik latest target: %s' % name) - return name[:-len('.osm.pbf')] - - -def fetch(url, target): - if not target.exists(): - with urllib.request.urlopen(url, timeout=TIMEOUT_SECONDS) as response, \ - open(str(target) + '.part', 'wb') as out: - shutil.copyfileobj(response, out) - Path(str(target) + '.part').rename(target) - return target - - -def md5(path): - digest = hashlib.md5() - with open(path, 'rb') as handle: - for block in iter(lambda: handle.read(1 << 20), b''): - digest.update(block) - return digest.hexdigest() - - -def read_poly(path): - """Geofabrik .poly -> list of (ring, is_hole).""" - rings, ring, hole = [], None, False - for raw in path.read_text(encoding='utf-8').splitlines()[1:]: - line = raw.strip() - if ring is None: - if line and line != 'END': - ring, hole = [], line.startswith('!') - elif line == 'END': - rings.append((ring, hole)) - ring = None - elif line: - lon, lat = (float(v) for v in line.split()) - ring.append((lon, lat)) - return rings - - -def inside(rings, lon, lat): - def in_ring(ring): - hit = False - for (x1, y1), (x2, y2) in zip(ring, ring[1:] + ring[:1]): - if (y1 > lat) != (y2 > lat) and lon < x1 + (lat - y1) * (x2 - x1) / (y2 - y1): - hit = not hit - return hit - return any(in_ring(r) for r, h in rings if not h) and not any(in_ring(r) for r, h in rings if h) - - -def points(coordinates): - if isinstance(coordinates[0], (int, float)): - return [coordinates] - return [p for part in coordinates for p in points(part)] - - -def osm_key(feature): - osm_id = feature['properties']['osm_id'] - if osm_id < 0: - return 'r%d' % -osm_id - return ('n' if feature['geometry']['type'] == 'Point' else 'w') + str(osm_id) - - -def export(osmium, pbf, cache, extract): - """Exported features by key, and the keys of every tagged object in the extract.""" - filtered = cache / (extract + '-datacenters.osm.pbf') - listed = cache / (extract + '-datacenters.opl') - exported = cache / (extract + '-datacenters.geojsonseq') - subprocess.run([osmium, 'tags-filter', '-O', '-o', str(filtered), str(pbf), - 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) - subprocess.run([osmium, 'tags-filter', '-O', '-R', '-f', 'opl', '-o', str(listed), str(filtered), - 'nwr/telecom=data_center', 'nwr/building=data_center'], check=True) - tagged = {line.split(' ', 1)[0] for line in listed.read_text(encoding='utf-8').splitlines() if line} - subprocess.run([osmium, 'export', '-O', '-f', 'geojsonseq', '--add-unique-id=type_id', - '-o', str(exported), str(filtered)], check=True) - fresh = {} - for line in exported.read_text(encoding='utf-8').splitlines(): - line = line.lstrip('\x1e').strip() - if not line: - continue - feature = json.loads(line) - tags = {k: v for k, v in feature['properties'].items() if k not in CONTACT_TAGS} - if tags.get('telecom') != 'data_center' and tags.get('building') != 'data_center': - continue # untagged member ways exported only for geometry - key = feature['id'] - geometry = feature['geometry'] - if key[0] == 'a': # osmium area id: way * 2 or relation * 2 + 1 - number = int(key[1:]) - key = ('w%d' % (number // 2)) if number % 2 == 0 else ('r%d' % (number // 2)) - if key[0] == 'r': - tags = {**tags, 'type': 'multipolygon'} - if geometry['type'] == 'MultiPolygon' and len(geometry['coordinates']) == 1: - geometry = {'type': 'Polygon', 'coordinates': geometry['coordinates'][0]} - elif key in fresh or geometry['type'] == 'LineString': - continue # closed ways are taken from their area export - fresh[key] = {'geometry': geometry, 'tags': tags} - return fresh, tagged - - -def same_geometry(old, new): - """Same type and rings within tolerance; ring start, winding and part or hole order may differ.""" - if old['type'] != new['type']: - return False - - def meters(p, q): - dx = (p[0] - q[0]) * 111320 * math.cos(math.radians(p[1])) - return math.hypot(dx, (p[1] - q[1]) * 110540) - - def same_line(a, b): - return len(a) == len(b) and all(meters(p, q) <= SAME_GEOMETRY_METERS for p, q in zip(a, b)) - - def same_ring(a, b): - a, b = a[:-1], b[:-1] # closed rings repeat their first vertex - return len(a) == len(b) and \ - any(same_line(a[i:] + a[:i], c) for c in (b, b[::-1]) for i in range(len(a))) - - def same_parts(a, b, same): - rest = list(b) - for part in a: - match = next((i for i, other in enumerate(rest) if same(part, other)), None) - if match is None: - return False - rest.pop(match) - return not rest - - def same_polygon(a, b): - return same_ring(a[0], b[0]) and same_parts(a[1:], b[1:], same_ring) - - a, b = old['coordinates'], new['coordinates'] - if old['type'] == 'Point': - return same_line([a], [b]) - if old['type'] == 'LineString': - return same_line(a, b) - if old['type'] == 'Polygon': - return same_polygon(a, b) - if old['type'] == 'MultiPolygon': - return same_parts(a, b, same_polygon) - return a == b - - -def feature_line(key, sequence, fresh): - osm_id = int(key[1:]) * (-1 if key[0] == 'r' else 1) - feature = {'id': osm_id, 'type': 'Feature', 'geometry': fresh['geometry'], - 'properties': {'id': sequence, 'tags': fresh['tags'], 'type': 'data_center', - 'osm_id': osm_id}} - return json.dumps(feature, separators=(',', ':'), ensure_ascii=False) - - -def merge(lines, fresh, rings, tagged): - out, known, sequence = [], set(), 0 - report = {'kept': 0, 'updated': [], 'removed': [], 'added': [], 'unexported': [], 'skipped': []} - for line in lines: - feature = json.loads(line) - sequence = max(sequence, feature['properties']['id']) - key = osm_key(feature) - known.add(key) - if key in fresh: - old_tags = feature['properties']['tags'] - if old_tags == fresh[key]['tags'] and same_geometry(feature['geometry'], fresh[key]['geometry']): - out.append(line) - report['kept'] += 1 - else: - out.append(feature_line(key, feature['properties']['id'], fresh[key])) - report['updated'].append(key) - continue - if key in tagged: # still tagged, but osmium built no usable geometry: keep the old line - report['unexported'].append(key) - out.append(line) - continue - lon, lat = points(feature['geometry']['coordinates'])[0][:2] - if inside(rings, lon, lat): - report['removed'].append(key) - continue - out.append(line) - for key in sorted(set(fresh) - known, key=lambda k: (k[0], int(k[1:]))): - sequence += 1 - out.append(feature_line(key, sequence, fresh[key])) - report['added'].append(key) - # newly tagged, but osmium built no usable geometry: nothing to add yet - report['skipped'] = sorted(set(tagged) - set(fresh) - known, key=lambda k: (k[0], int(k[1:]))) - if len({osm_key(json.loads(line)) for line in out}) != len(out): - raise ValueError('Duplicate OSM identity') - return out, report - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--cache', type=Path, required=True) - parser.add_argument('--osmium', default=shutil.which('osmium') or 'osmium') - parser.add_argument('--extract', default='latest', - help="dated Geofabrik name such as norway-261005, or 'latest'") - parser.add_argument('--verify', action='store_true') - args = parser.parse_args() - args.cache.mkdir(parents=True, exist_ok=True) - extract = latest_extract() if args.extract == 'latest' else args.extract - pbf = fetch(GEOFABRIK + extract + '.osm.pbf', args.cache / (extract + '.osm.pbf')) - expected = fetch(GEOFABRIK + extract + '.osm.pbf.md5', - args.cache / (extract + '.osm.pbf.md5')).read_text(encoding='utf-8').split()[0] - actual = md5(pbf) - if actual != expected: - raise RuntimeError('MD5 mismatch for %s: %s != %s' % (pbf.name, actual, expected)) - rings = read_poly(fetch(GEOFABRIK + 'norway.poly', args.cache / 'norway.poly')) - fresh, tagged = export(args.osmium, pbf, args.cache, extract) - current = [line for line in DATASET.read_text(encoding='utf-8').splitlines() if line.strip()] - merged, report = merge(current, fresh, rings, tagged) - if report['unexported']: - print('warning: kept the old line of tagged features osmium did not export: %s' - % ', '.join(report['unexported']), file=sys.stderr) - if report['skipped']: - print('warning: did not add newly tagged features osmium did not export: %s' - % ', '.join(report['skipped']), file=sys.stderr) - payload = '\n'.join(merged) + '\n' - if args.verify: - if payload != DATASET.read_text(encoding='utf-8'): - raise SystemExit('datacenters.geojsonl is not the %s merge' % extract) - else: - with open(DATASET, 'w', encoding='utf-8', newline='\n') as handle: - handle.write(payload) - print(json.dumps({'extract': extract, 'md5': actual, 'norway': len(fresh), - 'features': len(merged), **report})) - - -if __name__ == '__main__': - main() diff --git a/src/data/datacentersNorwayRefresh.test.mjs b/src/data/datacentersNorwayRefresh.test.mjs deleted file mode 100644 index 99278769e..000000000 --- a/src/data/datacentersNorwayRefresh.test.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import test from 'node:test'; -import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; -import { fileURLToPath } from 'node:url'; - -const SCRIPT = fileURLToPath( - new URL('../../scripts/build-datacenters-norway.py', import.meta.url), -); - -function python(t, source) { - const result = spawnSync('python3', ['-c', source, SCRIPT], { - encoding: 'utf8', - }); - if (result.error?.code === 'ENOENT') return t.skip('python3 is not installed'); - assert.equal(result.status, 0, result.stderr); -} - -test('Norway refresh compares rings, not vertex sets', (t) => { - python( - t, - String.raw` -import runpy, sys -same = runpy.run_path(sys.argv[1])['same_geometry'] -poly = lambda *rings: {'type': 'Polygon', 'coordinates': list(rings)} -multi = lambda *polys: {'type': 'MultiPolygon', 'coordinates': [p['coordinates'] for p in polys]} -a, b, c, d, e = [0, 0], [4, 0], [4, 4], [0, 4], [2, 1] -assert not same(poly([a, b, c, d, e, a]), poly([a, b, e, c, d, a])), 'reordered edges' -assert same(poly([a, b, c, d, e, a]), poly([c, b, a, e, d, c])), 'ring start and winding' -assert same(poly([a, b, c, d, a]), poly([[0, 1e-7], b, c, d, [0, 1e-7]])), 'tolerance' -outer = [[0, 0], [9, 0], [9, 9], [0, 9], [0, 0]] -one, two = [[1, 1], [2, 1], [2, 2], [1, 1]], [[5, 5], [6, 5], [6, 6], [5, 5]] -assert same(poly(outer, one, two), poly(outer, two, one)), 'hole order' -assert not same(poly(outer, one), poly(one, outer)), 'outer and inner roles swapped' -assert same(multi(poly(one), poly(two)), multi(poly(two), poly(one))), 'part order' -p, q, r, s, u, v = [0, 0], [1, 0], [0, 1], [5, 5], [6, 5], [5, 6] -assert not same(multi(poly([p, q, r, p]), poly([s, u, v, s])), - multi(poly([p, q, s, p]), poly([r, u, v, r]))), 'vertices moved between parts' -`, - ); -}); - -test('Norway refresh keeps a still-tagged feature osmium did not export', (t) => { - python( - t, - String.raw` -import json, runpy, sys -merge = runpy.run_path(sys.argv[1])['merge'] -square = [[[10, 60], [10.01, 60], [10.01, 60.01], [10, 60.01], [10, 60]]] -def line(osm_id, sequence): - return json.dumps({'id': osm_id, 'type': 'Feature', 'geometry': {'type': 'Polygon', 'coordinates': square}, - 'properties': {'id': sequence, 'tags': {'building': 'data_center'}, - 'type': 'data_center', 'osm_id': osm_id}}) -norway = [([(0, 0), (20, 0), (20, 80), (0, 80)], False)] -lines = [line(7, 1), line(8, 2)] -fresh = {'w13': {'geometry': {'type': 'Polygon', 'coordinates': square}, 'tags': {'building': 'data_center'}}} -out, report = merge(lines, fresh, norway, {'w7', 'w12', 'w13'}) -assert out[0] == lines[0] and len(out) == 2, out -assert report['unexported'] == ['w7'] and report['removed'] == ['w8'], report -assert report['skipped'] == ['w12'] and report['added'] == ['w13'], report -`, - ); -}); - -test('Norway refresh lists tagged objects apart from the exported geometry', (t) => { - python( - t, - String.raw` -import json, runpy, subprocess, sys, tempfile -from pathlib import Path -export = runpy.run_path(sys.argv[1])['export'] -tags = {'building': 'data_center'} -ring = [[10, 60], [10.01, 60], [10.01, 60.01], [10, 60]] -features = [ - {'id': 'n1', 'properties': {'telecom': 'data_center'}, 'geometry': {'type': 'Point', 'coordinates': [10, 60]}}, - {'id': 'w5', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring}}, - {'id': 'a10', 'properties': tags, 'geometry': {'type': 'MultiPolygon', 'coordinates': [[ring]]}}, - {'id': 'w7', 'properties': tags, 'geometry': {'type': 'LineString', 'coordinates': ring[:3]}}, -] -outputs = { - '.opl': 'n1 v1 Ttelecom=data_center x10 y60\nw5 v1 Tbuilding=data_center Nn2,n3,n4,n2\n' - 'w7 v1 Tbuilding=data_center Nn2,n3,n4\nr9 v1 Ttelecom=data_center Mw8@outer\n', - '.geojsonseq': ''.join(chr(30) + json.dumps({'type': 'Feature', **f}) + '\n' for f in features), -} -def run(args, check=False): - target = Path(args[args.index('-o') + 1]) - target.write_text(outputs.get(target.suffix, ''), encoding='utf-8') -subprocess.run = run -with tempfile.TemporaryDirectory() as cache: - fresh, tagged = export('osmium', Path(cache) / 'norway-test.osm.pbf', Path(cache), 'norway-test') -assert sorted(fresh) == ['n1', 'w5'], fresh -assert fresh['w5']['geometry'] == {'type': 'Polygon', 'coordinates': [ring]}, fresh['w5'] -assert tagged == {'n1', 'w5', 'w7', 'r9'}, tagged -`, - ); -}); diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index 86f573c61..c360ee527 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -26,14 +26,4 @@ reflects OSM as of the extract below, everything else the original snapshot. - Extract: Geofabrik `norway-261005.osm.pbf`, OSM data 2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0` - Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` -- Contact tags are dropped as above; unchanged features keep their original - line, so geometry is only rewritten when tags or rings actually changed. - A feature is removed only once it is no longer tagged; one still tagged but - without an exportable geometry keeps its line and is reported as unexported, - and a newly tagged one osmium cannot export is reported as skipped. -- Refresh: `python3 scripts/build-datacenters-norway.py --cache ` uses - Geofabrik's current Norway extract and prints the dated name it used; record - that name, its OSM timestamp and MD5 here. Requires `osmium-tool`. -- Check: add `--extract --verify` to compare without writing. - Geofabrik keeps only its newest dated extract, so a recorded extract can be - re-verified only while it is still listed; the MD5 above identifies it. +- Contact tags are dropped as above. From a69d0492d0cda16a1eef15d022629f4949f293b5 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 15:54:53 +0400 Subject: [PATCH 20/27] docs(datacenters): tighten the Norway refresh notes --- changelog.d/953-datacenters-norway-refresh.md | 6 ++---- docs/INFRASTRUCTURE-LAYERS.md | 2 +- src/data/local_data/datacenters/README.md | 6 ++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/changelog.d/953-datacenters-norway-refresh.md b/changelog.d/953-datacenters-norway-refresh.md index 5db897234..d60d4ac00 100644 --- a/changelog.d/953-datacenters-norway-refresh.md +++ b/changelog.d/953-datacenters-norway-refresh.md @@ -1,4 +1,2 @@ -- Refresh Norway's datacenters from a Geofabrik OpenStreetMap extract - (2026-10-05). Troll Housing joins the layer (4,352 features) and four - Norwegian sites pick up current OSM tags; no site moved or was removed - (trygvehhh, #953). +- Add Troll Housing to the Datacenters layer (4,352 features) by refreshing + Norway from OpenStreetMap data of 2026-10-05 (trygvehhh, #953). diff --git a/docs/INFRASTRUCTURE-LAYERS.md b/docs/INFRASTRUCTURE-LAYERS.md index aa8cd0558..798ff09af 100644 --- a/docs/INFRASTRUCTURE-LAYERS.md +++ b/docs/INFRASTRUCTURE-LAYERS.md @@ -59,4 +59,4 @@ globals. Dataset files and source/license notices remain under `src/data/local_data/`. Their input counts are 4,352 datacenter features and 704 dam features. Cesium may expand multipart geometries into multiple entities; entity and stem counts are -different measurements. This change does not refresh or relicense the datasets. +different measurements. diff --git a/src/data/local_data/datacenters/README.md b/src/data/local_data/datacenters/README.md index c360ee527..a89894fdb 100644 --- a/src/data/local_data/datacenters/README.md +++ b/src/data/local_data/datacenters/README.md @@ -19,11 +19,9 @@ snapshot. Future refreshes should record both before replacing the file. ## Norway refresh -Features inside the Geofabrik Norway boundary (`norway.poly`, which includes -Svalbard) were refreshed separately, so this file mixes two vintages: Norway -reflects OSM as of the extract below, everything else the original snapshot. +Features inside Geofabrik's Norway boundary (`norway.poly`, with Svalbard) come +from a newer extract than the rest of the file. Contact tags are dropped as above. - Extract: Geofabrik `norway-261005.osm.pbf`, OSM data 2026-10-05T20:21:35Z, MD5 `1d2cfdeb98980fce11b8891d0f9146c0` - Filter: `osmium tags-filter nwr/telecom=data_center nwr/building=data_center` -- Contact tags are dropped as above. From c058ecf21b98f3097adea6cd512b4931cb864723 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 15:57:27 +0400 Subject: [PATCH 21/27] docs(infrastructure): keep the dataset scope note --- docs/INFRASTRUCTURE-LAYERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/INFRASTRUCTURE-LAYERS.md b/docs/INFRASTRUCTURE-LAYERS.md index 798ff09af..aa8cd0558 100644 --- a/docs/INFRASTRUCTURE-LAYERS.md +++ b/docs/INFRASTRUCTURE-LAYERS.md @@ -59,4 +59,4 @@ globals. Dataset files and source/license notices remain under `src/data/local_data/`. Their input counts are 4,352 datacenter features and 704 dam features. Cesium may expand multipart geometries into multiple entities; entity and stem counts are -different measurements. +different measurements. This change does not refresh or relicense the datasets. From 141181f83a0cc2fa6379a4c77af04469aee2df92 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 6 Nov 2026 12:02:30 +0000 Subject: [PATCH 22/27] docs(flights): changelog names the one-in-13-ticks rotation, about once a second --- changelog.d/870-hidden-contact-dr.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/870-hidden-contact-dr.md b/changelog.d/870-hidden-contact-dr.md index f88084d7a..aab55229e 100644 --- a/changelog.d/870-hidden-contact-dr.md +++ b/changelog.d/870-hidden-contact-dr.md @@ -1,4 +1,4 @@ - Reduce dead-reckoning work for flights already hidden behind the horizon. - Hidden contacts update on a staggered one-in-12-ticks rotation while visible - contacts and contacts with fleet models retain the full update path - (Wilmer Alzate, #870). + Hidden contacts update on a staggered one-in-13-ticks rotation (about once a + second) while visible contacts and contacts with fleet models retain the full + update path (Wilmer Alzate, #870). From 4e518821b514e688d6e2663eded607395ea4e45f Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 19:43:04 +0400 Subject: [PATCH 23/27] fix(cyclones): name the classification and mark a stale cache in shared context - Publish classificationName next to the raw NHC code; unknown codes stay unexpanded. - Publish staleCacheFetchedAt only while the proxy serves its cache. - Touch the shared store on clear only after this layer published, so a feed failure no longer evicts another layer's dangling selection. - Drop the unreachable no-storm branch and the unused return value. - Tests: split the heading fixture into due north and stationary, cover the unavailable and failure paths with the real store, and check that clear/disable leave another layer's selection alone; fold the fake-store test into the real-store one. --- docs/CURRENT-STATE.md | 12 +- src/layers/cyclones/index.js | 33 ++- src/layers/cyclones/index.test.mjs | 349 ++++++++++++++++++----------- 3 files changed, 247 insertions(+), 147 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fa535c295..a1b0874c1 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -743,12 +743,12 @@ NHC movement speed is converted from miles per hour to rounded knots by the proxy; advisory wind intensity is already in knots and is passed through. Cyclone markers, labels, tracks and cones are hidden beyond the horizon on every map source. Explicit cyclone selections publish one shared `weather-cyclones` record -with official advisory provenance and source position. Switching storms removes -the prior record; automatic row selection never claims global context, and -refreshes do not re-announce selection. Clearing or losing an explicitly selected -storm removes shared context rather than assigning a fallback. -Deliberate clear/disable reports `deliberate`; feed disappearance reports -`evicted`. Switching storms selects the replacement before pruning the old record. +with official advisory provenance, source position, the English classification +name and, while the proxy serves its cache, the cache fetch time. Switching +storms selects the replacement before pruning the old record; automatic row +selection never claims global context, and refreshes do not re-announce +selection. Deliberate clear/disable reports `deliberate`; losing the selected +storm from the feed reports `evicted` and assigns no fallback. Consult the linked official advisory for safety decisions. Voice and HUD snapshots reuse the existing feedState classifier. Analyst follow-ups retain their original data provenance; current-view results append provenance without replacing legacy fields. HUD context and deterministic telemetry include non-nominal feed state. diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index ef8bbad27..8741ad8a3 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -96,19 +96,19 @@ export function createCyclonesLayer({ const selected = () => snapshot?.storms.find((storm) => storm.id === selectedId) || null; function clearSelectedContext({ evicted = false } = {}) { - if (!evicted) - context?.clearSelectedEntityContextForLayer?.('weather-cyclones'); - context?.removeEntityContextsForLayer?.('weather-cyclones'); + if (selectedContextStormId) { + if (!evicted) + context?.clearSelectedEntityContextForLayer?.('weather-cyclones'); + context?.removeEntityContextsForLayer?.('weather-cyclones'); + } selectedContextStormId = null; selectedContextEntity = null; } + // Callers publish only while selectedId names a storm in the snapshot. function publishSelectedContext({ announce = false } = {}) { const storm = selected(); - if (!storm || !context) { - if (!storm) clearSelectedContext({ evicted: true }); - return null; - } + if (!context) return; selectedContextEntity ||= { show: true }; const contextId = `cyclone:${storm.id}`; const record = context.registerEntityContext?.(selectedContextEntity, { @@ -116,13 +116,20 @@ export function createCyclonesLayer({ layerId: 'weather-cyclones', layerName: 'Cyclone advisories', source: 'NOAA NHC / CPHC', - // Layer identity is already carried separately. Keep source names literal - // rather than adding an untranslated user-facing prefix. + // Keep official storm names verbatim; layerName identifies the layer. label: storm.name, latitude: storm.position.latitude, longitude: storm.position.longitude, + // At most 12 properties reach the voice summary; keep the list short. properties: { classification: storm.classification, + // English source meaning; unknown codes stay unexpanded. + classificationName: Object.hasOwn( + CLASSIFICATION_NAMES, + storm.classification, + ) + ? CLASSIFICATION_NAMES[storm.classification] + : undefined, basin: storm.basin, advisoryNumber: storm.advisoryNumber, issuedAt: storm.issuedAt, @@ -133,15 +140,19 @@ export function createCyclonesLayer({ movementDirectionDegrees: storm.movement?.directionDegrees, movementSpeedKt: storm.movement?.speedKt, advisoryUrl: storm.advisoryUrl, + // Present only while the server serves its cache with NHC unreachable. + staleCacheFetchedAt: + snapshot.stale && Number.isFinite(snapshot.fetchedAt) + ? new Date(snapshot.fetchedAt).toISOString() + : undefined, }, }); - if (!record) return null; + if (!record) return; selectedContextStormId = storm.id; if (announce) context.selectEntityContext?.(selectedContextEntity); context.removeEntityContextsForLayer?.('weather-cyclones', { retainIds: new Set([contextId]), }); - return record; } function select(id) { diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index f70f4f6a8..a3e40fc95 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -405,96 +405,19 @@ function harness( }, }; } -test('explicit cyclone selection publishes shared context without auto-claiming it', async () => { - let next = snapshot([storm(), storm('ep162026')]); - const records = new Map(); - let selectedContext = null; - let selectedEvents = 0; - const context = { - registerEntityContext(entity, metadata) { - entity.__gevContextId = metadata.id; - const record = { ...metadata, entity }; - records.set(metadata.id, record); - return record; - }, - selectEntityContext(entity) { - selectedContext = records.get(entity.__gevContextId) || null; - selectedEvents += 1; - return selectedContext; - }, - removeEntityContextsForLayer(layerId, { retainIds } = {}) { - for (const [id, record] of records) { - if (record.layerId === layerId && !retainIds?.has(id)) - records.delete(id); - } - if (selectedContext && !records.has(selectedContext.id)) - selectedContext = null; - }, - }; - const h = harness({ getSnapshot: async () => next }, { context }); - h.layer.enable(); - await h.layer.update(); - - assert.equal( - selectedEvents, - 0, - 'the automatic first-storm row selection must not claim shared context', - ); - assert.equal(records.size, 0); - - h.layer.setParams({ stormId: 'ep162026' }); - assert.equal(selectedEvents, 1); - assert.equal(selectedContext?.layerId, 'weather-cyclones'); - assert.equal(selectedContext?.source, 'NOAA NHC / CPHC'); - assert.equal(selectedContext?.label, 'Another system'); - assert.equal(selectedContext?.latitude, 15.5); - assert.equal(selectedContext?.longitude, -125.8); - assert.equal(selectedContext?.properties.advisoryNumber, '10'); - - h.layer.setParams({ stormId: 'ep152026' }); - assert.deepEqual([...records.keys()], ['cyclone:ep152026']); - assert.equal(selectedContext?.id, 'cyclone:ep152026'); - h.layer.setParams({ stormId: 'ep162026' }); - assert.deepEqual([...records.keys()], ['cyclone:ep162026']); - assert.equal(selectedContext?.id, 'cyclone:ep162026'); - assert.equal(selectedEvents, 3, 'each explicit storm switch announces once'); - - await h.layer.update(); - assert.equal( - selectedEvents, - 3, - 'refresh updates the selected record without re-announcing the click', - ); - - next = snapshot([storm()]); - await h.layer.update(); - assert.equal( - selectedContext, - null, - 'a vanished user-selected storm must clear instead of transferring context to the fallback row', - ); - assert.equal(records.size, 0); - - h.layer.setParams({ stormId: 'ep152026' }); - assert.equal(selectedEvents, 4); - h.layer.setParams({ clear: true }); - assert.equal(selectedContext, null); - assert.equal(records.size, 0); - - h.layer.destroy(); -}); - -test('shared cyclone context distinguishes switches, deliberate clears and feed eviction', async (t) => { +// Runs a test body against the real shared-context store on a stand-in window, +// with the voice runner reading the same store. +async function withSharedContext(t, body) { const previousWindow = Object.getOwnPropertyDescriptor(globalThis, 'window'); const host = new EventTarget(); Object.defineProperty(globalThis, 'window', { configurable: true, value: host, }); - let h; + const layers = []; t.after(() => { try { - h?.layer.destroy(); + for (const layer of layers) layer.destroy(); } finally { if (previousWindow) Object.defineProperty(globalThis, 'window', previousWindow); @@ -506,9 +429,9 @@ test('shared cyclone context distinguishes switches, deliberate clears and feed events.push(['selected', detail.id]), ); host.addEventListener('gev:entity-selection-cleared', ({ detail }) => - events.push(['cleared', detail.reason]), + events.push(['cleared', detail.layerId, detail.reason]), ); - // The voice runner reads the same store and bounds its basemap wait. + // The voice runner bounds its basemap wait with the window timers. Object.assign(host, { setTimeout, clearTimeout }); const position = Cesium.Cartesian3.fromDegrees(-125.8, 15.5, 5_000_000); const runner = createGevActionRunner({ @@ -535,55 +458,221 @@ test('shared cyclone context distinguishes switches, deliberate clears and feed assert.equal(result.scope, 'selected'); return result.selected.properties; }; - let next = snapshot([ - { ...storm(), movement: { directionDegrees: 315, speedKt: 12 } }, - { ...storm('ep162026'), movement: { directionDegrees: 0, speedKt: 0 } }, - ]); - h = harness({ getSnapshot: async () => next }, { context: contextStore }); - h.layer.enable(); - await h.layer.update(); - h.layer.setParams({ stormId: 'ep152026' }); - let properties = await selectedProperties(); - assert.equal( - properties.movementDirectionDegrees, - '315', - 'selected storm movement should survive voice summarization', - ); - assert.equal(properties.movementSpeedKt, '12'); - h.layer.setParams({ stormId: 'ep162026' }); - properties = await selectedProperties(); - assert.equal( - properties.movementDirectionDegrees, - '0', - 'a due-north heading is a value, not a missing one', - ); - assert.equal(properties.movementSpeedKt, '0', 'stationary is a value'); - assert.deepEqual(events, [ - ['selected', 'cyclone:ep152026'], - ['selected', 'cyclone:ep162026'], - ]); - assert.deepEqual( - [...contextStore.getContextStore().entities.keys()], - ['cyclone:ep162026'], - ); - await h.layer.update(); - assert.equal(events.length, 2, 'refresh must not announce selection'); + const store = contextStore.getContextStore(); + const open = (feed) => { + const h = harness(feed, { context: contextStore }); + layers.push(h.layer); + return h; + }; + await body({ events, selectedProperties, store, open }); +} - h.layer.setParams({ clear: true }); - assert.deepEqual(events.at(-1), ['cleared', 'deliberate']); - assert.equal(contextStore.getContextStore().entities.size, 0); +test('shared cyclone context distinguishes switches, deliberate clears and feed eviction', async (t) => { + await withSharedContext( + t, + async ({ events, selectedProperties, store, open }) => { + let next = snapshot([ + { ...storm(), movement: { directionDegrees: 315, speedKt: 12 } }, + { + ...storm('ep162026'), + movement: { directionDegrees: 0, speedKt: 10 }, + }, + ]); + const h = open({ getSnapshot: async () => next }); + h.layer.enable(); + await h.layer.update(); + assert.deepEqual( + events, + [], + 'the automatic first-storm row selection must not claim shared context', + ); + assert.equal(store.entities.size, 0); - h.layer.setParams({ stormId: 'ep162026' }); - next = snapshot([storm()]); - await h.layer.update(); - assert.deepEqual(events.at(-1), ['cleared', 'evicted']); - assert.equal(contextStore.getContextStore().entities.size, 0); - assert.equal(contextStore.getContextStore().selectedEntityId, null); + h.layer.setParams({ stormId: 'ep152026' }); + const record = store.entities.get('cyclone:ep152026'); + assert.equal(record.layerId, 'weather-cyclones'); + assert.equal(record.source, 'NOAA NHC / CPHC'); + assert.equal(record.label, 'Fifteen-E'); + assert.equal(record.latitude, 15.5); + assert.equal(record.longitude, -125.8); + let properties = await selectedProperties(); + assert.equal(properties.classification, 'PTC'); + assert.equal(properties.classificationName, 'Post-tropical cyclone'); + assert.equal(properties.advisoryNumber, '10'); + assert.equal( + properties.movementDirectionDegrees, + '315', + 'selected storm movement should survive voice summarization', + ); + assert.equal(properties.movementSpeedKt, '12'); + assert.equal( + properties.advisoryUrl, + 'https://www.nhc.noaa.gov/text/MIATCMEP5.shtml', + ); + assert.equal(properties.staleCacheFetchedAt, undefined); - h.layer.setParams({ stormId: 'ep152026' }); - h.layer.disable(); - assert.deepEqual(events.at(-1), ['cleared', 'deliberate']); - assert.equal(contextStore.getContextStore().entities.size, 0); + h.layer.setParams({ stormId: 'ep162026' }); + properties = await selectedProperties(); + assert.equal( + properties.movementDirectionDegrees, + '0', + 'a due-north heading is a value, not a missing one', + ); + assert.equal(properties.movementSpeedKt, '10'); + assert.deepEqual(events, [ + ['selected', 'cyclone:ep152026'], + ['selected', 'cyclone:ep162026'], + ]); + assert.deepEqual([...store.entities.keys()], ['cyclone:ep162026']); + + next = snapshot([ + { ...storm(), movement: { directionDegrees: 315, speedKt: 12 } }, + { ...storm('ep162026'), movement: { directionDegrees: 0, speedKt: 0 } }, + ]); + await h.layer.update(); + assert.equal(events.length, 2, 'refresh must not announce selection'); + properties = await selectedProperties(); + assert.equal( + properties.movementSpeedKt, + '0', + 'stationary is a value, refreshed into the selected record', + ); + + h.layer.setParams({ clear: true }); + assert.deepEqual(events.at(-1), [ + 'cleared', + 'weather-cyclones', + 'deliberate', + ]); + assert.equal(store.entities.size, 0); + + h.layer.setParams({ stormId: 'ep162026' }); + next = snapshot([storm()]); + await h.layer.update(); + assert.deepEqual( + events.at(-1), + ['cleared', 'weather-cyclones', 'evicted'], + 'a vanished user-selected storm must clear instead of transferring context to the fallback row', + ); + assert.equal(store.entities.size, 0); + assert.equal(store.selectedEntityId, null); + + h.layer.setParams({ stormId: 'ep152026' }); + h.layer.disable(); + assert.deepEqual(events.at(-1), [ + 'cleared', + 'weather-cyclones', + 'deliberate', + ]); + assert.equal(store.entities.size, 0); + }, + ); +}); + +test('shared cyclone context is evicted when the source fails and marks a stale cache', async (t) => { + await withSharedContext( + t, + async ({ events, selectedProperties, store, open }) => { + let next = snapshot([{ ...storm(), classification: 'XX' }]); + let fail = false; + const h = open({ + getSnapshot: async () => { + if (fail) throw new Error('Network unavailable'); + return next; + }, + }); + h.layer.enable(); + await h.layer.update(); + h.layer.setParams({ stormId: 'ep152026' }); + let properties = await selectedProperties(); + assert.equal(properties.classification, 'XX'); + assert.equal( + properties.classificationName, + undefined, + 'an unknown code is not given a guessed meaning', + ); + + next = { ...snapshot(), stale: true }; + await h.layer.update(); + properties = await selectedProperties(); + assert.equal(properties.staleCacheFetchedAt, time); + assert.equal(events.length, 1, 'refresh must not announce selection'); + + next = { + ...snapshot([]), + unavailable: true, + reason: 'Cyclone data unavailable', + }; + await h.layer.update(); + assert.deepEqual(events.at(-1), [ + 'cleared', + 'weather-cyclones', + 'evicted', + ]); + assert.equal(store.entities.size, 0); + + next = snapshot(); + await h.layer.update(); + h.layer.setParams({ stormId: 'ep152026' }); + assert.equal(store.selectedEntityId, 'cyclone:ep152026'); + fail = true; + await h.layer.update(); + assert.deepEqual(events.at(-1), [ + 'cleared', + 'weather-cyclones', + 'evicted', + ]); + assert.equal(store.entities.size, 0); + assert.equal(store.selectedEntityId, null); + }, + ); +}); + +test('cyclone clears leave another layer selection and its records alone', async (t) => { + await withSharedContext(t, async ({ events, store, open }) => { + const h = open({ getSnapshot: async () => snapshot() }); + h.layer.enable(); + await h.layer.update(); + h.layer.setParams({ stormId: 'ep152026' }); + const foreign = {}; + contextStore.registerEntityContext(foreign, { + id: 'flight:abc123', + layerId: 'flights', + label: 'Foreign', + }); + contextStore.selectEntityContext(foreign); + const before = events.length; + + h.layer.setParams({ clear: true }); + assert.equal(store.selectedEntityId, 'flight:abc123'); + assert.ok(store.entities.has('flight:abc123')); + assert.equal(store.entities.has('cyclone:ep152026'), false); + assert.equal( + events.length, + before, + 'no clear is reported for a foreign selection', + ); + + h.layer.disable(); + assert.equal(store.selectedEntityId, 'flight:abc123'); + assert.equal(events.length, before); + + // A foreign record removed without clearing its selection: a cyclone + // feed failure must not report that selection as evicted by cyclones. + store.entities.delete('flight:abc123'); + let fail = false; + const other = open({ + getSnapshot: async () => { + if (fail) throw new Error('Network unavailable'); + return snapshot(); + }, + }); + other.layer.enable(); + await other.layer.update(); + fail = true; + await other.layer.update(); + assert.equal(events.length, before); + }); }); test('refresh preserves selection intent, including explicit clears and missing storms', async () => { From 5e64e599e1c2079d08a7ddf153a4ad66d7c07c37 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 20:07:56 +0400 Subject: [PATCH 24/27] docs(cyclones): tighten shared-context comments, changelog and notes --- changelog.d/847-cyclone-shared-selection.md | 7 +++---- docs/CURRENT-STATE.md | 13 ++++++------- src/app/layers/cyclones.js | 2 +- src/layers/cyclones/index.js | 4 +--- src/layers/cyclones/index.test.mjs | 6 ++---- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/changelog.d/847-cyclone-shared-selection.md b/changelog.d/847-cyclone-shared-selection.md index 24edf2ebb..150295823 100644 --- a/changelog.d/847-cyclone-shared-selection.md +++ b/changelog.d/847-cyclone-shared-selection.md @@ -1,4 +1,3 @@ -- Explicit cyclone selections publish only the currently selected NHC/CPHC - storm into shared context. Switching storms removes the previous identity - without a false eviction event. Deliberate clears remain distinct from feed - disappearance (kvnloo, #847; reviewed by quanticsoul4772). +- Let the voice assistant read the storm you select in Cyclone advisories: + its official advisory, position, intensity and movement. Clearing the + selection and the storm leaving the feed are reported apart (kvnloo, #847). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 6df75c27e..f3b3d3396 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -742,13 +742,12 @@ geographic seams and releases on disable; forecast animation is not implied. NHC movement speed is converted from miles per hour to rounded knots by the proxy; advisory wind intensity is already in knots and is passed through. Cyclone markers, labels, tracks and cones are hidden beyond the horizon on every map source. -Explicit cyclone selections publish one shared `weather-cyclones` record -with official advisory provenance, source position, the English classification -name and, while the proxy serves its cache, the cache fetch time. Switching -storms selects the replacement before pruning the old record; automatic row -selection never claims global context, and refreshes do not re-announce -selection. Deliberate clear/disable reports `deliberate`; losing the selected -storm from the feed reports `evicted` and assigns no fallback. +Explicitly selecting a storm publishes one shared `weather-cyclones` record: +advisory provenance, source position, the English classification name and, +while the proxy serves its cache, the cache fetch time. The automatic row +selection does not claim shared context; refreshes do not re-announce it. +Clear or disable reports `deliberate`; a storm leaving the feed reports +`evicted`, with no fallback. Consult the linked official advisory for safety decisions. Voice and HUD snapshots reuse the existing feedState classifier. Analyst follow-ups retain their original data provenance; current-view results append provenance without replacing legacy fields. HUD context and deterministic telemetry include non-nominal feed state. diff --git a/src/app/layers/cyclones.js b/src/app/layers/cyclones.js index d671a11eb..bed75c76e 100644 --- a/src/app/layers/cyclones.js +++ b/src/app/layers/cyclones.js @@ -1,7 +1,7 @@ import { createCyclonesLayer } from '../../layers/cyclones/index.js'; import * as context from '../../data/contextStore.js'; -/** Wire cyclone selection into the application shared-context owner. */ +/** Wire cyclone selection to the application shared context. */ export function createApplicationCyclones(options) { const browserContext = typeof window !== 'undefined' ? { context } : {}; return createCyclonesLayer({ diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 8741ad8a3..95655d5d0 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -116,14 +116,12 @@ export function createCyclonesLayer({ layerId: 'weather-cyclones', layerName: 'Cyclone advisories', source: 'NOAA NHC / CPHC', - // Keep official storm names verbatim; layerName identifies the layer. label: storm.name, latitude: storm.position.latitude, longitude: storm.position.longitude, // At most 12 properties reach the voice summary; keep the list short. properties: { classification: storm.classification, - // English source meaning; unknown codes stay unexpanded. classificationName: Object.hasOwn( CLASSIFICATION_NAMES, storm.classification, @@ -140,7 +138,7 @@ export function createCyclonesLayer({ movementDirectionDegrees: storm.movement?.directionDegrees, movementSpeedKt: storm.movement?.speedKt, advisoryUrl: storm.advisoryUrl, - // Present only while the server serves its cache with NHC unreachable. + // Only while the proxy serves its cache because NHC is unreachable. staleCacheFetchedAt: snapshot.stale && Number.isFinite(snapshot.fetchedAt) ? new Date(snapshot.fetchedAt).toISOString() diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index a3e40fc95..41ecaf3d1 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -405,8 +405,7 @@ function harness( }, }; } -// Runs a test body against the real shared-context store on a stand-in window, -// with the voice runner reading the same store. +// Real shared-context store on a stand-in window; the voice runner reads it. async function withSharedContext(t, body) { const previousWindow = Object.getOwnPropertyDescriptor(globalThis, 'window'); const host = new EventTarget(); @@ -657,8 +656,7 @@ test('cyclone clears leave another layer selection and its records alone', async assert.equal(store.selectedEntityId, 'flight:abc123'); assert.equal(events.length, before); - // A foreign record removed without clearing its selection: a cyclone - // feed failure must not report that selection as evicted by cyclones. + // A cyclone feed failure does not evict a dangling foreign selection. store.entities.delete('flight:abc123'); let fail = false; const other = open({ From 5bab5ec26d9f4b13510e50660f06a1895340a4e7 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 6 Nov 2026 20:15:27 +0400 Subject: [PATCH 25/27] refactor(cyclones): drop defensive shared-context state and checks --- src/app/layers/cyclones.js | 3 +-- src/layers/cyclones/index.js | 15 ++++++--------- src/layers/cyclones/index.test.mjs | 24 +++--------------------- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/src/app/layers/cyclones.js b/src/app/layers/cyclones.js index bed75c76e..e0cd508b0 100644 --- a/src/app/layers/cyclones.js +++ b/src/app/layers/cyclones.js @@ -3,9 +3,8 @@ import * as context from '../../data/contextStore.js'; /** Wire cyclone selection to the application shared context. */ export function createApplicationCyclones(options) { - const browserContext = typeof window !== 'undefined' ? { context } : {}; return createCyclonesLayer({ - ...browserContext, + context: typeof window !== 'undefined' ? context : null, ...options, }); } diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 95655d5d0..90936edf3 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -84,7 +84,6 @@ export function createCyclonesLayer({ selectionIntent = 'auto', navigationGeneration = 0, selectedContextStormId = null, - selectedContextEntity = null, clickHandler = null, removeClickCapture = null; let enabled = false, @@ -98,20 +97,19 @@ export function createCyclonesLayer({ function clearSelectedContext({ evicted = false } = {}) { if (selectedContextStormId) { if (!evicted) - context?.clearSelectedEntityContextForLayer?.('weather-cyclones'); - context?.removeEntityContextsForLayer?.('weather-cyclones'); + context.clearSelectedEntityContextForLayer('weather-cyclones'); + context.removeEntityContextsForLayer('weather-cyclones'); } selectedContextStormId = null; - selectedContextEntity = null; } // Callers publish only while selectedId names a storm in the snapshot. function publishSelectedContext({ announce = false } = {}) { const storm = selected(); if (!context) return; - selectedContextEntity ||= { show: true }; + const entity = {}; const contextId = `cyclone:${storm.id}`; - const record = context.registerEntityContext?.(selectedContextEntity, { + context.registerEntityContext(entity, { id: contextId, layerId: 'weather-cyclones', layerName: 'Cyclone advisories', @@ -145,10 +143,9 @@ export function createCyclonesLayer({ : undefined, }, }); - if (!record) return; selectedContextStormId = storm.id; - if (announce) context.selectEntityContext?.(selectedContextEntity); - context.removeEntityContextsForLayer?.('weather-cyclones', { + if (announce) context.selectEntityContext(entity); + context.removeEntityContextsForLayer('weather-cyclones', { retainIds: new Set([contextId]), }); } diff --git a/src/layers/cyclones/index.test.mjs b/src/layers/cyclones/index.test.mjs index 41ecaf3d1..e826f392e 100644 --- a/src/layers/cyclones/index.test.mjs +++ b/src/layers/cyclones/index.test.mjs @@ -488,12 +488,7 @@ test('shared cyclone context distinguishes switches, deliberate clears and feed assert.equal(store.entities.size, 0); h.layer.setParams({ stormId: 'ep152026' }); - const record = store.entities.get('cyclone:ep152026'); - assert.equal(record.layerId, 'weather-cyclones'); - assert.equal(record.source, 'NOAA NHC / CPHC'); - assert.equal(record.label, 'Fifteen-E'); - assert.equal(record.latitude, 15.5); - assert.equal(record.longitude, -125.8); + assert.equal(store.entities.get('cyclone:ep152026').label, 'Fifteen-E'); let properties = await selectedProperties(); assert.equal(properties.classification, 'PTC'); assert.equal(properties.classificationName, 'Post-tropical cyclone'); @@ -652,24 +647,11 @@ test('cyclone clears leave another layer selection and its records alone', async 'no clear is reported for a foreign selection', ); + // A cyclone clear does not report a dangling foreign selection as evicted. + store.entities.delete('flight:abc123'); h.layer.disable(); assert.equal(store.selectedEntityId, 'flight:abc123'); assert.equal(events.length, before); - - // A cyclone feed failure does not evict a dangling foreign selection. - store.entities.delete('flight:abc123'); - let fail = false; - const other = open({ - getSnapshot: async () => { - if (fail) throw new Error('Network unavailable'); - return snapshot(); - }, - }); - other.layer.enable(); - await other.layer.update(); - fail = true; - await other.layer.update(); - assert.equal(events.length, before); }); }); From c4298c44501a0e5c21216648f91e6bf9a2eeafd7 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Fri, 6 Nov 2026 17:25:01 +0100 Subject: [PATCH 26/27] fix(ghostmaps): keep age-window changes consistent and make the age choice a switch Review fixes on top of #1528: - Narrowing the age window only removes markers; a product loaded with a narrower window is reloaded when the window widens again. Fixes markers missing after 7d -> 1d -> 7d and avoids reparsing every KMZ on narrowing. - A wider window over the same edition keeps the open report card. - Date-only event dates cover their whole UTC day, so 1d still shows yesterday's reports. - The age options are one labelled "Event age" radio switch (row `segmented` control, as FIRMS confidence) instead of toggle chips mixed with the product chips. - Un-nest the per-product age test; QA compares data-source counts with the live defaults instead of a hard-coded 2. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 61 ++++----- scripts/qa-ghostmaps.mjs | 33 +++-- src/i18n/locales/ar.json | 1 + src/i18n/locales/de.json | 1 + src/i18n/locales/es.json | 1 + src/i18n/locales/fr.json | 1 + src/i18n/locales/it.json | 1 + src/i18n/locales/ja.json | 1 + src/i18n/locales/pt.json | 1 + src/i18n/locales/ru.json | 1 + src/i18n/locales/uk.json | 1 + src/i18n/locales/zh-CN.json | 1 + src/i18n/locales/zh-Hant.json | 1 + src/layers/ghostMaps/index.js | 86 +++++++----- src/layers/ghostMaps/lifecycle.test.mjs | 175 +++++++++++++++++------- src/layers/ghostMaps/records.js | 6 +- src/layers/ghostMaps/records.test.mjs | 10 +- 17 files changed, 247 insertions(+), 135 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1a7205096..fe837e421 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -250,7 +250,6 @@ style listeners. Synthetic fixtures cover privacy, acquisition, rendering, selection and card placement. Deterministic operation checks run in `npm test`; wall-clock budgets run in `npm run test:perf`. All 11 locales are supported. - ## Prohibited & Restricted Airspace — FAA special-use airspace The Infrastructure group includes **Prohibited & Restricted Airspace** (`drone-airspace`, @@ -4342,35 +4341,35 @@ nearby place—always use station selection. Unqualified “turn on/start the ra requests use Play; a qualified Play-shaped tool call is normalized to Select so its criteria cannot be silently ignored. -| Layer | Source | File | Proxy | Update Interval | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | -| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/flights` (OAuth + fallback) | 30s | -| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/military` | 15s | -| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/vessels` | 60s (+800ms visibility pass) | -| Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | -| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | -| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | -| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | -| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | -| Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Hong Kong Transport Department + FL511 (FL, opt-in) + Freeway Bureau, MOTC (Taiwan) + Houston TranStar (opt-in) + Autostrada del Brennero (IT, A22 stills) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | -| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | -| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | -| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | -| Datacenters ▣ | OSM extract (bundled) | `src/data/infrastructure.js` | — | static | -| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/infrastructure.js` | — | static | -| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | -| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | -| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | -| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | -| GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | -| S2 GhostMaps 👻 (`s2-ghostmaps`) | S2 Underground's public GhostMaps repository on GitHub (KMZ intel products, keyless, CC BY-NC-SA 4.0) | `src/layers/ghostMaps/` via `src/app/layers/ghostMaps.js` | `/api/ghostmaps/catalog`, `/api/ghostmaps/events` (server-sent events), `/api/ghostmaps/file/{product}` | branch head checked every 15 s; new commits pushed to the page; 5 min catalog backstop | -| Sea Marks ⚓ (`sea-marks`) | OpenSeaMap seamark raster tiles (keyless; CC BY-SA 2.0 tiles, OSM data ODbL) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | -| Ski Pistes ⛷ (`ski-pistes`) | OpenSnowMap piste raster tiles (keyless; CC BY-SA tiles, OSM data ODbL; no bulk download) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | -| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | -| Prohibited & Restricted Airspace 🚫 | FAA Special Use Airspace prohibited (P) and restricted (R) zones (keyless FeatureServer, simplified geometry) | `src/layers/droneAirspace/` via `src/app/layers/droneAirspace.js` | `/api/drone-airspace` | 1 h (server cache 1 h; stale-on-error) | -| Wildlife Sightings 🦉 | iNaturalist wildlife observed within 48 h (whales 7 d), per view; Acartia Salish Sea marine mammal coverage (keyless) | `src/layers/wildlifeSightings/` via `src/app/layers/wildlifeSightings.js` | `/api/inaturalist/observations`, `/api/inaturalist/grid`, `/api/acartia/sightings` | camera moveEnd (600 ms debounce; enable settles 300 ms) + 5 min (observations 5 / 10 min; tiles 15 min; stale ≤ 1 h / 24 h) | +| Layer | Source | File | Proxy | Update Interval | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/flights` (OAuth + fallback) | 30s | +| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/military` | 15s | +| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/vessels` | 60s (+800ms visibility pass) | +| Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | +| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | +| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | +| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | +| Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Hong Kong Transport Department + FL511 (FL, opt-in) + Freeway Bureau, MOTC (Taiwan) + Houston TranStar (opt-in) + Autostrada del Brennero (IT, A22 stills) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | +| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | +| Datacenters ▣ | OSM extract (bundled) | `src/data/infrastructure.js` | — | static | +| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/infrastructure.js` | — | static | +| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | +| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | +| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | +| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | +| GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | +| S2 GhostMaps 👻 (`s2-ghostmaps`) | S2 Underground's public GhostMaps repository on GitHub (KMZ intel products, keyless, CC BY-NC-SA 4.0) | `src/layers/ghostMaps/` via `src/app/layers/ghostMaps.js` | `/api/ghostmaps/catalog`, `/api/ghostmaps/events` (server-sent events), `/api/ghostmaps/file/{product}` | branch head checked every 15 s; new commits pushed to the page; 5 min catalog backstop | +| Sea Marks ⚓ (`sea-marks`) | OpenSeaMap seamark raster tiles (keyless; CC BY-SA 2.0 tiles, OSM data ODbL) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | +| Ski Pistes ⛷ (`ski-pistes`) | OpenSnowMap piste raster tiles (keyless; CC BY-SA tiles, OSM data ODbL; no bulk download) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | +| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | +| Prohibited & Restricted Airspace 🚫 | FAA Special Use Airspace prohibited (P) and restricted (R) zones (keyless FeatureServer, simplified geometry) | `src/layers/droneAirspace/` via `src/app/layers/droneAirspace.js` | `/api/drone-airspace` | 1 h (server cache 1 h; stale-on-error) | +| Wildlife Sightings 🦉 | iNaturalist wildlife observed within 48 h (whales 7 d), per view; Acartia Salish Sea marine mammal coverage (keyless) | `src/layers/wildlifeSightings/` via `src/app/layers/wildlifeSightings.js` | `/api/inaturalist/observations`, `/api/inaturalist/grid`, `/api/acartia/sightings` | camera moveEnd (600 ms debounce; enable settles 300 ms) + 5 min (observations 5 / 10 min; tiles 15 min; stale ≤ 1 h / 24 h) | GNSS Integrity samples a 250 nm circle around the Earth point under the screen centre. Camera reads debounce for 600 ms with a monotonic 2.5 s minimum gap; @@ -6008,7 +6007,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. -- **Event age filter.** The same filter applies to every selected S2 product, including Border Crisis, Iran Picture, Iran Kinetic, Yemen FLOT and Latest Drop. S2 products default to seven days, with row buttons for 1, 3, 7, 14 or 30 days and **All markers**. The choice persists independently of product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage uses seven days. Only the placemark's `Date of Event` (or `EventDate`/`Date`) determines its age, never report dates, dates in prose or KMZ edition/upload dates. Undated, blank or unparseable event dates and future dates stay visible. ArcGIS `M/D/YYYY` dates and optional AM/PM times, plus ISO dates/timestamps, are supported; values without a zone use UTC and date-only values use midnight UTC. The exact cutoff is included. Events age out every minute and during catalog updates, even without a new file or a successful catalog fetch. Widening the window or choosing All markers restores omitted events from the cached KMZ. Removed entities and selected cards disappear together; surviving pick ids stay stable. Product defaults and the update stream are unchanged. +- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All markers**. The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/scripts/qa-ghostmaps.mjs b/scripts/qa-ghostmaps.mjs index 24edcc60b..3c3f0b880 100644 --- a/scripts/qa-ghostmaps.mjs +++ b/scripts/qa-ghostmaps.mjs @@ -203,15 +203,12 @@ async function main() { ); record( 'LIVE: one chip per product, defaults pressed', - chips.filter((chip) => !chip.id.startsWith('age-')).length === - catalog.products.length && - chips - .filter((chip) => !chip.id.startsWith('age-')) - .every( - (chip) => - chip.active === - catalog.products.find((p) => p.id === chip.id)?.defaultOn, - ), + chips.length === catalog.products.length && + chips.every( + (chip) => + chip.active === + catalog.products.find((p) => p.id === chip.id)?.defaultOn, + ), chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '), ); const stream = await page @@ -229,7 +226,18 @@ async function main() { record('LIVE: the /api/ghostmaps/events stream is open', stream); await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') }); - const ageChips = chips.filter((chip) => chip.id.startsWith('age-')); + const ageChips = await page.evaluate( + (id) => + [ + ...document.querySelectorAll( + `[data-layer-id="${id}"] [data-segmented-id="age"] [role="radio"]`, + ), + ].map((option) => ({ + id: option.dataset.chipId, + active: option.getAttribute('aria-checked') === 'true', + })), + LAYER_ID, + ); record( 'AGE: seven days is the default with an All markers option', live.ageDays === 7 && @@ -262,7 +270,8 @@ async function main() { const all = await selectAge(null); record( 'AGE: All markers restores old events without duplicate data sources', - all.count > live.count && (await sources(page)).length === 2, + all.count > live.count && + (await sources(page)).length === live.products.length, `${live.count} → ${all.count}`, ); await page.reload({ waitUntil: 'domcontentloaded' }); @@ -279,7 +288,7 @@ async function main() { const one = await selectAge(1); const seven = await selectAge(7); record( - 'AGE: shorter and longer windows change the count and seven days restores the default', + 'AGE: counts follow the window order and seven days restores the default', one.count <= live.count && thirty.count >= live.count && thirty.count <= all.count && diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index 456917068..99826e32c 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3471,5 +3471,6 @@ "All markers": "كل العلامات", "Show markers of every age": "عرض العلامات من جميع الأعمار", "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "Event age": "عمر الحدث", "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index d38cc685f..826091ee8 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3171,5 +3171,6 @@ "All markers": "Alle Markierungen", "Show markers of every age": "Markierungen jeden Alters anzeigen", "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "Event age": "Ereignisalter", "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index 45fd1961c..2f1cc02f1 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3246,5 +3246,6 @@ "All markers": "Todos los marcadores", "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "Event age": "Antigüedad", "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index 528d01d9c..ff0b02c01 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3246,5 +3246,6 @@ "All markers": "Tous les marqueurs", "Show markers of every age": "Afficher les marqueurs de tout âge", "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "Event age": "Ancienneté", "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index b8204a980..4a26811fa 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3246,5 +3246,6 @@ "All markers": "Tutti gli indicatori", "Show markers of every age": "Mostra gli indicatori di qualsiasi età", "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "Event age": "Età degli eventi", "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 6f88bd36d..9f111d91d 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -3096,5 +3096,6 @@ "All markers": "すべてのマーカー", "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数:{days}日。日付のないマーカーは保持", + "Event age": "経過日数", "Event age: {days}d · undated kept": "経過日数:{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index 0f328d23b..ed83191d9 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3246,5 +3246,6 @@ "All markers": "Todos os marcadores", "Show markers of every age": "Mostrar marcadores de qualquer idade", "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "Event age": "Idade do evento", "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index f3d050aba..82aa91f06 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3321,5 +3321,6 @@ "All markers": "Все метки", "Show markers of every age": "Показать метки любого возраста", "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "Event age": "Возраст события", "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index e340f2e12..4d8ff9711 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3321,5 +3321,6 @@ "All markers": "Усі позначки", "Show markers of every age": "Показати позначки будь-якого віку", "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "Event age": "Вік події", "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index 9e088ac3e..a7cb60910 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -3096,5 +3096,6 @@ "All markers": "所有标记", "Show markers of every age": "显示所有时间的标记", "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间:{days}天;保留无日期标记", + "Event age": "事件时间", "Event age: {days}d · undated kept": "事件时间:{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index 578fbcf40..ebbf0680c 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -3096,5 +3096,6 @@ "All markers": "所有標記", "Show markers of every age": "顯示所有時間的標記", "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間:{days}天;保留無日期標記", + "Event age": "事件時間", "Event age: {days}d · undated kept": "事件時間:{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 7899c26b5..7d8716483 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -208,13 +208,13 @@ export function createGhostMapsLayer({ return (_catalog?.products || []).filter(wanted).map(({ id }) => id); } - function unload(id) { + function unload(id, keepSelection = false) { const item = _loaded.get(id); if (!item) return; _loaded.delete(id); _viewer?.dataSources?.remove(item.dataSource, true); item.urls.forEach((url) => URL.revokeObjectURL(url)); - if (_selectedKey?.startsWith(`${id}:`)) clearSelection(); + if (!keepSelection && _selectedKey?.startsWith(`${id}:`)) clearSelection(); } // Remove expired entities, rather than hiding them: horizon culling must @@ -423,8 +423,9 @@ export function createGhostMapsLayer({ // Cull before the first frame so far-side pins never flash. _horizon.update(next.pins); await _viewer.dataSources.add(next.dataSource); - // The new edition is on the globe before the old one leaves. - unload(product.id); + // The new edition is on the globe before the old one leaves. A wider + // window over the same edition keeps indices, so the card stays. + unload(product.id, loaded?.version === next.version); _loaded.set(product.id, next); } catch (error) { if (signal.aborted) return; @@ -473,9 +474,16 @@ export function createGhostMapsLayer({ /* private window or blocked storage: keep the session choice */ } expireEvents(); + // Narrowing only removes markers. A product loaded with a narrower window + // keeps its old ageDays, so sync() restores omitted placemarks from the + // cached KMZ even though its upstream version has not changed. + for (const item of _loaded.values()) + if ( + item.ageDays === null || + (ageDays !== null && item.ageDays >= ageDays) + ) + item.ageDays = ageDays; notifyRow(); - // A wider window must restore previously omitted placemarks from the - // cached KMZ, even though its upstream version has not changed. return sync(); } @@ -730,8 +738,10 @@ export function createGhostMapsLayer({ }) : t('Connecting to the S2 GhostMaps repository…'); return { - chips: [ - ...GHOSTMAPS_AGE_OPTIONS.map((days) => ({ + segmented: { + id: 'age', + label: t('Event age'), + options: GHOSTMAPS_AGE_OPTIONS.map((days) => ({ id: `age-${days ?? 'all'}`, label: days === null @@ -746,36 +756,36 @@ export function createGhostMapsLayer({ active: days === _ageDays, onClick: () => setAgeDays(days), })), - ...products.map((product) => { - const loaded = _loaded.get(product.id); - const failed = _failed.get(product.id); - return { - id: product.id, - label: t(PRODUCT_LABELS[product.id] || product.title), - active: wanted(product), - busy: _loading.has(product.id), - state: failed ? 'error' : undefined, - color: PRODUCT_COLORS[product.id], - title: [ - product.fileTitle, - loaded - ? tn( - loaded.features.size, - '{count} placemark', - '{count} placemarks', - { - count: formatNumber(loaded.features.size), - }, - ) - : null, - failed ? t(failed) : null, - ] - .filter(Boolean) - .join(' · '), - onClick: () => toggleProduct(product.id), - }; - }), - ], + }, + chips: products.map((product) => { + const loaded = _loaded.get(product.id); + const failed = _failed.get(product.id); + return { + id: product.id, + label: t(PRODUCT_LABELS[product.id] || product.title), + active: wanted(product), + busy: _loading.has(product.id), + state: failed ? 'error' : undefined, + color: PRODUCT_COLORS[product.id], + title: [ + product.fileTitle, + loaded + ? tn( + loaded.features.size, + '{count} placemark', + '{count} placemarks', + { + count: formatNumber(loaded.features.size), + }, + ) + : null, + failed ? t(failed) : null, + ] + .filter(Boolean) + .join(' · '), + onClick: () => toggleProduct(product.id), + }; + }), legend: [], info: `${_ageDays === null ? t('All markers') : t('Event age: {days}d · undated kept', { days: formatNumber(_ageDays) })} · ${info}`, infoTitle: t( diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 0b3a580e6..7d1a067e6 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -203,7 +203,9 @@ const dated = (name, value, label = 'Date') => ({ }); const ageChip = (layer, days) => - layer.getRowControls().chips.find(({ id }) => id === `age-${days ?? 'all'}`); + layer + .getRowControls() + .segmented.options.find(({ id }) => id === `age-${days ?? 'all'}`); test('age controls default to seven days and widening restores filtered and expired reports', async (t) => { let at = Date.UTC(2026, 10, 3, 12); @@ -233,10 +235,16 @@ test('age controls default to seven days and widening restores filtered and expi await ageChip(layer, null).onClick(); assert.equal(layer.getStats().count, 4); assert.equal(mounted.size, 1, 'filter changes never duplicate data sources'); - const ageChips = layer - .getRowControls() - .chips.filter(({ id }) => id.startsWith('age-')); - assert.equal(ageChips.filter(({ active }) => active).length, 1); + const controls = layer.getRowControls(); + assert.equal( + controls.segmented.options.filter(({ active }) => active).length, + 1, + ); + assert.equal( + controls.chips.some(({ id }) => id.startsWith('age-')), + false, + 'age options are one switch, not product chips', + ); assert.equal(ageChip(layer, null).active, true); }); @@ -309,49 +317,49 @@ test('a filter changed during a pending load never installs the superseded filte created.length - 1, 'discarded load releases icons', ); +}); - test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { - for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { - await t.test(product.id, async (t) => { - const { layer, viewer, mounted } = fixture(t, { - now: () => Date.UTC(2026, 10, 3, 12), - products: [{ ...product, version: 'a'.repeat(40) }], - reports: [ - dated( - 'Old event', - '10/1/2026', - product.id === 'cip' ? 'Date of Event' : 'Date', - ), - dated('Recent event', '11/2/2026'), - { name: 'Undated reference feature' }, - ], - }); - await layer.update(viewer); - if (!product.defaultOn) { - layer - .getRowControls() - .chips.find(({ id }) => id === product.id) - .onClick(); - await layer.update(viewer); - } - assert.deepEqual(layer.getStats().products, [product.id]); - assert.equal(layer.getStats().count, 2); - assert.deepEqual( - layer.getAnalystRecords().map(({ name }) => name), - ['Recent event', 'Undated reference feature'], - ); - await ageChip(layer, null).onClick(); - assert.equal( - layer.getStats().count, - 3, - "All markers restores this product's old events", - ); - await ageChip(layer, 7).onClick(); - assert.equal(layer.getStats().count, 2); - assert.equal(mounted.size, 1); +test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { + for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { + await t.test(product.id, async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + products: [{ ...product, version: 'a'.repeat(40) }], + reports: [ + dated( + 'Old event', + '10/1/2026', + product.id === 'cip' ? 'Date of Event' : 'Date', + ), + dated('Recent event', '11/2/2026'), + { name: 'Undated reference feature' }, + ], }); - } - }); + await layer.update(viewer); + if (!product.defaultOn) { + layer + .getRowControls() + .chips.find(({ id }) => id === product.id) + .onClick(); + await layer.update(viewer); + } + assert.deepEqual(layer.getStats().products, [product.id]); + assert.equal(layer.getStats().count, 2); + assert.deepEqual( + layer.getAnalystRecords().map(({ name }) => name), + ['Recent event', 'Undated reference feature'], + ); + await ageChip(layer, null).onClick(); + assert.equal( + layer.getStats().count, + 3, + "All markers restores this product's old events", + ); + await ageChip(layer, 7).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal(mounted.size, 1); + }); + } }); test('default products filter old event dates while keeping recent, future and undated markers', async (t) => { @@ -448,7 +456,8 @@ test('unchanged cached products age out on the timer, clear selected cards and k test('expiry removes every geometry and polygon outline belonging to an old report', async (t) => { t.mock.timers.enable({ apis: ['setInterval'] }); - let at = Date.UTC(2026, 10, 3); + // 10/27 is a whole UTC day: at 7d it expires when 11/3 ends. + let at = Date.UTC(2026, 10, 4) - 30_000; const polygon = element('Polygon', [ element('outerBoundaryIs', [ element('LinearRing', [ @@ -477,3 +486,75 @@ test('expiry removes every geometry and polygon outline belonging to an old repo assert.equal(ds.entities.values.length, 0); assert.equal(layer.getStats().count, 0); }); + +test('narrowing then returning to the previous window restores its reports without a duplicate source', async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Today', '11/3/2026'), dated('Three days', '10/31/2026')], + }); + await layer.update(viewer); + assert.equal(layer.getStats().count, 2); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); + await ageChip(layer, 7).onClick(); + await layer.update(viewer); + assert.equal(layer.getStats().count, 2, '7d must show both reports again'); + assert.equal(mounted.size, 1); +}); + +test('two quick clicks, 1d then 7d, restore the report', async (t) => { + const at = Date.UTC(2026, 10, 3, 12); + const { layer, viewer } = fixture(t, { + now: () => at, + reports: [dated('Today', '11/3/2026'), dated('Three days', '10/31/2026')], + }); + await layer.update(viewer); + const a = ageChip(layer, 1).onClick(); + const b = ageChip(layer, 7).onClick(); + await a; + await b; + await layer.update(viewer); + assert.equal(layer.getStats().count, 2); +}); + +test('narrowing the window keeps the open card of a surviving marker', async (t) => { + let click; + const cards = []; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')], + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + await ageChip(layer, 30).onClick(); + assert.equal(layer.getStats().count, 2); + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + await ageChip(layer, 3).onClick(); + assert.equal(layer.getStats().count, 1); + assert.equal( + cards.at(-1).length, + 1, + 'card of the surviving marker stays open', + ); +}); + +test('a report dated yesterday is shown by 1d one hour into today (UTC)', async (t) => { + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 6, 1), + reports: [dated('Yesterday', '11/5/2026')], + }); + await layer.update(viewer); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); +}); diff --git a/src/layers/ghostMaps/records.js b/src/layers/ghostMaps/records.js index 3c307fa63..62e48d3b3 100644 --- a/src/layers/ghostMaps/records.js +++ b/src/layers/ghostMaps/records.js @@ -73,7 +73,9 @@ export const GHOSTMAPS_DEFAULT_AGE_DAYS = 7; export const GHOSTMAPS_AGE_OPTIONS = Object.freeze([1, 3, 7, 14, 30, null]); /** ArcGIS US dates (optionally with a time) and ISO dates, interpreted in UTC - * when no zone is supplied. Reject calendar rollovers instead of guessing. */ + * when no zone is supplied. A date without a time means the whole UTC day, so + * it resolves to the day's last millisecond: "1d" still shows yesterday's + * reports. Reject calendar rollovers instead of guessing. */ export function parseGhostMapsEventDate(value) { const input = String(value ?? '').trim(); const us = @@ -113,7 +115,7 @@ export function parseGhostMapsEventDate(value) { const parsed = Date.parse(zoned); return Number.isFinite(parsed) ? parsed : null; } - return at; + return us?.[4] ? at : at + 86400_000 - 1; } /** The event's date, never its report/upload date or dates mentioned in prose. */ diff --git a/src/layers/ghostMaps/records.test.mjs b/src/layers/ghostMaps/records.test.mjs index a28d1f36d..f674fb1cb 100644 --- a/src/layers/ghostMaps/records.test.mjs +++ b/src/layers/ghostMaps/records.test.mjs @@ -14,15 +14,15 @@ import { test('event dates parse ArcGIS calendar dates and timestamps consistently in UTC', () => { for (const [value, iso] of [ - ['10/28/2026', '2026-10-28T00:00:00Z'], + ['10/28/2026', '2026-10-28T23:59:59.999Z'], ['10/28/2026 6:12:00 PM', '2026-10-28T18:12:00Z'], ['1/17/2026 9:46:41 AM', '2026-01-17T09:46:41Z'], ['10/28/2026 12:00 AM', '2026-10-28T00:00:00Z'], ['10/28/2026 12:00 PM', '2026-10-28T12:00:00Z'], - ['2026-10-28', '2026-10-28T00:00:00Z'], + ['2026-10-28', '2026-10-28T23:59:59.999Z'], ['2026-10-28T12:34:56', '2026-10-28T12:34:56Z'], ['2026-10-28T12:34:56.123-04:00', '2026-10-28T16:34:56.123Z'], - ['2/29/2024', '2024-02-29T00:00:00Z'], + ['2/29/2024', '2024-02-29T23:59:59.999Z'], ]) assert.equal(parseGhostMapsEventDate(value), Date.parse(iso), value); for (const value of [ @@ -52,7 +52,7 @@ test('event date fields take precedence over report dates and prose, even after row('Date', '10/30/2026') + row('Date of Event', '10/28/2026') }`; - assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 28)); + assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 29) - 1); assert.equal( ghostMapsEventDate( `${row('Date', '<Null>')}${row('Date Of Report', '11/2/2026')}
`, @@ -61,7 +61,7 @@ test('event date fields take precedence over report dates and prose, even after ); assert.equal( ghostMapsEventDate(`${row('Date_Of_Event', '10/28/2026')}
`), - Date.UTC(2026, 9, 28), + Date.UTC(2026, 9, 29) - 1, ); assert.equal(ghostMapsEventDate('Occurred 10/28/2026'), null); assert.equal( From 91ab9b5811c93cb66fbd292fff66f051e1854ed7 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Fri, 6 Nov 2026 17:37:29 +0100 Subject: [PATCH 27/27] fix(ghostmaps): follow-up review fixes for the age switch - Keep the open card across a widened reload only when its index still holds the same report; the server may answer a pinned version with newer bytes. - Narrowing to a window the loaded data already covers clears a failed wider reload, so the row no longer stays in error. - Label the every-age option "All" (contextual) so the six-option switch fits the row in every locale; tooltip and info line keep "All markers". Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 2 +- src/i18n/locales/ar.json | 1 + src/i18n/locales/de.json | 1 + src/i18n/locales/es.json | 1 + src/i18n/locales/fr.json | 1 + src/i18n/locales/it.json | 1 + src/i18n/locales/ja.json | 1 + src/i18n/locales/pt.json | 1 + src/i18n/locales/ru.json | 1 + src/i18n/locales/uk.json | 1 + src/i18n/locales/zh-CN.json | 1 + src/i18n/locales/zh-Hant.json | 1 + src/layers/ghostMaps/index.js | 24 +++++++++--- src/layers/ghostMaps/lifecycle.test.mjs | 51 +++++++++++++++++++++++++ 14 files changed, 81 insertions(+), 7 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fe837e421..0e1cd7889 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -6007,7 +6007,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. -- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All markers**. The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. +- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All** (every marker). The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index 99826e32c..f1d9903ed 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3471,6 +3471,7 @@ "All markers": "كل العلامات", "Show markers of every age": "عرض العلامات من جميع الأعمار", "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "event age filter|All": "الكل", "Event age": "عمر الحدث", "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index 826091ee8..50715c99f 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3171,6 +3171,7 @@ "All markers": "Alle Markierungen", "Show markers of every age": "Markierungen jeden Alters anzeigen", "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "event age filter|All": "Alle", "Event age": "Ereignisalter", "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index 2f1cc02f1..6adb40da9 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3246,6 +3246,7 @@ "All markers": "Todos los marcadores", "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "event age filter|All": "Todos", "Event age": "Antigüedad", "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index ff0b02c01..788210f89 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3246,6 +3246,7 @@ "All markers": "Tous les marqueurs", "Show markers of every age": "Afficher les marqueurs de tout âge", "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "event age filter|All": "Tous", "Event age": "Ancienneté", "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index 4a26811fa..567c3c6ed 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3246,6 +3246,7 @@ "All markers": "Tutti gli indicatori", "Show markers of every age": "Mostra gli indicatori di qualsiasi età", "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "event age filter|All": "Tutti", "Event age": "Età degli eventi", "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 9f111d91d..5b96cb3a8 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -3096,6 +3096,7 @@ "All markers": "すべてのマーカー", "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数:{days}日。日付のないマーカーは保持", + "event age filter|All": "すべて", "Event age": "経過日数", "Event age: {days}d · undated kept": "経過日数:{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index ed83191d9..20bbea5aa 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3246,6 +3246,7 @@ "All markers": "Todos os marcadores", "Show markers of every age": "Mostrar marcadores de qualquer idade", "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "event age filter|All": "Todos", "Event age": "Idade do evento", "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index 82aa91f06..ca4813a5a 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3321,6 +3321,7 @@ "All markers": "Все метки", "Show markers of every age": "Показать метки любого возраста", "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "event age filter|All": "Все", "Event age": "Возраст события", "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index 4d8ff9711..9b7ae1f61 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3321,6 +3321,7 @@ "All markers": "Усі позначки", "Show markers of every age": "Показати позначки будь-якого віку", "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "event age filter|All": "Усі", "Event age": "Вік події", "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index a7cb60910..9b17dc66f 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -3096,6 +3096,7 @@ "All markers": "所有标记", "Show markers of every age": "显示所有时间的标记", "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间:{days}天;保留无日期标记", + "event age filter|All": "全部", "Event age": "事件时间", "Event age: {days}d · undated kept": "事件时间:{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index ebbf0680c..e3d9b716c 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -3096,6 +3096,7 @@ "All markers": "所有標記", "Show markers of every age": "顯示所有時間的標記", "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間:{days}天;保留無日期標記", + "event age filter|All": "全部", "Event age": "事件時間", "Event age: {days}d · undated kept": "事件時間:{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 7d8716483..79971c715 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -423,9 +423,18 @@ export function createGhostMapsLayer({ // Cull before the first frame so far-side pins never flash. _horizon.update(next.pins); await _viewer.dataSources.add(next.dataSource); - // The new edition is on the globe before the old one leaves. A wider - // window over the same edition keeps indices, so the card stays. - unload(product.id, loaded?.version === next.version); + // The new edition is on the globe before the old one leaves. The card + // stays only if its index still holds the same report: the server + // may answer a pinned version with newer bytes. + const index = Number(_selectedKey?.split(':')[1]); + const before = loaded?.features.get(index); + const after = next.features.get(index); + unload( + product.id, + Boolean(before && after) && + before.name === after.name && + before.description === after.description, + ); _loaded.set(product.id, next); } catch (error) { if (signal.aborted) return; @@ -477,12 +486,15 @@ export function createGhostMapsLayer({ // Narrowing only removes markers. A product loaded with a narrower window // keeps its old ageDays, so sync() restores omitted placemarks from the // cached KMZ even though its upstream version has not changed. - for (const item of _loaded.values()) + for (const [id, item] of _loaded) if ( item.ageDays === null || (ageDays !== null && item.ageDays >= ageDays) - ) + ) { item.ageDays = ageDays; + // A failed wider reload no longer matters; a stale edition retries. + _failed.delete(id); + } notifyRow(); return sync(); } @@ -745,7 +757,7 @@ export function createGhostMapsLayer({ id: `age-${days ?? 'all'}`, label: days === null - ? t('All markers') + ? t('All', { context: 'event age filter' }) : t('{days}d', { days: formatNumber(days) }), title: days === null diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 7d1a067e6..ae9f63911 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -558,3 +558,54 @@ test('a report dated yesterday is shown by 1d one hour into today (UTC)', async await ageChip(layer, 1).onClick(); assert.equal(layer.getStats().count, 1); }); + +test('a widened reload that returns a different report at the selected index closes its card', async (t) => { + let click; + const cards = []; + const reports = [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')]; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports, + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + // The server answers the pinned version with a newer edition's bytes. + reports[0] = dated('Replacement', '11/2/2026'); + await ageChip(layer, 30).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal( + cards.at(-1).length, + 0, + 'a different report must not keep the old card', + ); +}); + +test('narrowing to a window the loaded data covers clears a failed wider reload', async (t) => { + let fail = false; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')], + beforeFileReturns: () => { + if (fail) throw new Error('network down'); + }, + }); + await layer.update(viewer); + fail = true; + await ageChip(layer, 30).onClick(); + assert.ok(layer.getStats().error, 'the wider reload failed'); + await ageChip(layer, 3).onClick(); + assert.equal(layer.getStats().error, null); + assert.equal(layer.getStats().count, 1); +});