diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 61b971b71..ca774b47c 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -38,6 +38,7 @@ How to read this: | **Environment and Climate Change Canada** (MSC GeoMet OGC API) | Air quality (AQHI) — Canadian station observations | [ECCC Data Servers End-use Licence](https://eccc-msc.github.io/open-data/licence/readme_en/) (open, attribution required, no endorsement implied) | "Contains information licensed under the Environment and Climate Change Canada Data Servers End-use Licence" | | **NASA GIBS** (WMTS tiles + Worldview Snapshots) and **NASA CMR** (granule catalog) | Recent Imagery layer: HLS S30/L30 (Sentinel-2, Landsat 8/9) and VIIRS true-colour tiles draped on the map, per-day thumbnails and PNG exports; the HLS granule catalog (dates, cloud, footprints) for a selected box | U.S. public domain (NASA data); keyless, browser-direct (CORS `*`); no bulk use — one 30-day catalog query per box, tiles bounded to the box, six tile requests in flight | "We acknowledge the use of imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of NASA's Earth Science Data and Information System (ESDIS)." — [gibs.earthdata.nasa.gov](https://gibs.earthdata.nasa.gov), [HLS](https://lpdaac.usgs.gov/products/hlss30v002/) | | **NIFC WFIGS** (Wildland Fire Interagency Geospatial Services, ArcGIS feature service) | Fire Perimeters layer: current interagency wildfire incident perimeters, containment, and incident facts | U.S. public domain (interagency wildland-fire data published through the [NIFC Open Data portal](https://data-nifc.opendata.arcgis.com/)); fetched keyless through `/api/fire-perimeters` (5-minute server cache) with `exceededTransferLimit` paging | "Wildfire perimeters: National Interagency Fire Center (WFIGS)" | +| **S2 Underground GhostMaps** (`github.com/s2underground/GhostMaps`) | S2 GhostMaps layer: the current edition of S2's KMZ intel products (Common Intelligence Picture, 30-day kinetic pull, Border Crisis Map, Iran and Yemen products, newest incident drop). Keyless. `/api/ghostmaps` reads the branch head from git's ref advertisement every 15 s (`GHOSTMAPS_POLL_SECONDS`, minimum 5 s), lists the tree through the GitHub REST API only when the head changes (optional `GITHUB_TOKEN`), downloads files by commit-pinned raw URLs into `.gev-cache/ghostmaps/` and pushes new catalogs to browsers over server-sent events | [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/) (repository README: "S2 Underground © 2025 by S2A1"); non-commercial, attribution required; files are fetched at runtime, never stored in this repository | "S2 GhostMaps intel products: S2 Underground © by S2A1, CC BY-NC-SA 4.0" — shown in-app | | **InciWeb** (inciweb.wildfire.gov) | Per-incident "InciWeb ↗" card links to official incident pages (narratives, evacuation/closure notices) | U.S. government public incident information. Links are matched best-effort by incident name and state, with incident page origin and update times checked on selection. Same-origin `/api/fire-perimeters/inciweb` routes cache the catalog for 1 hour and publication metadata for 30 minutes; linked pages remain InciWeb content | "Incident information: InciWeb (inciweb.wildfire.gov)" | | **OpenFreeMap / OpenMapTiles** | Vector tiles | OpenStreetMap data under ODbL 1.0 | OpenFreeMap © OpenMapTiles | | **Overture Maps Foundation** | Military area names pack | [ODbL 1.0](https://opendatacommons.org/licenses/odbl/1-0/) | Overture Maps Foundation | diff --git a/changelog.d/0000-s2-ghostmaps-layer.md b/changelog.d/0000-s2-ghostmaps-layer.md new file mode 100644 index 000000000..b78d6a042 --- /dev/null +++ b/changelog.d/0000-s2-ghostmaps-layer.md @@ -0,0 +1,7 @@ +- Add the S2 GhostMaps layer: S2 Underground's public KMZ intel products + (Common Intelligence Picture, 30-day kinetic pull, Border Crisis Map, Iran + and Yemen products and the newest incident drop) from + github.com/s2underground/GhostMaps, with one chip per product and report + cards on click. `/api/ghostmaps` watches the repository's branch head every + 15 s and pushes each new commit to open pages, which reload only the + products that changed (Black Pine Defense). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index eacfa1147..1e2fabc51 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3507,6 +3507,7 @@ its criteria cannot be silently ignored. | Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | | Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | | GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | +| S2 GhostMaps 👻 (`s2-ghostmaps`) | S2 Underground's public GhostMaps repository on GitHub (KMZ intel products, keyless, CC BY-NC-SA 4.0) | `src/layers/ghostMaps/` via `src/app/layers/ghostMaps.js` | `/api/ghostmaps/catalog`, `/api/ghostmaps/events` (server-sent events), `/api/ghostmaps/file/{product}` | branch head checked every 15 s; new commits pushed to the page; 5 min catalog backstop | | Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | GNSS Integrity samples a 250 nm circle around the Earth point under the screen @@ -4994,6 +4995,25 @@ are omitted rather than framing the wrong part of the globe. - Pure half: `src/annotations/drawMode.js` (session, minimum vertices, double-click de-dup at 0.5 m, vertex ceiling, degenerate-geometry refusal, ring closing, dateline-safe length/area/centroid, hint text). Regression surface: `src/annotations/drawMode.test.mjs` and `src/data/inputOwnership.test.mjs` (pure), `src/annotations/drawToolBehaviour.test.mjs` (the tool driven against DOM and viewer doubles — leases, borrowed input actions, teardown, the Enter key, preview heights), `src/annotations/drawTool.test.mjs` (structural: where files live, which ids the markup carries, what the docs may claim), and the rendered harness `scripts/qa-draw-tool.mjs`. - The annotation tree is a declared ownership group: `./annotations` → `src/annotations/index.js` in `package.json`, with its module graph listed in `scripts/package-boundaries.json`, so an annotation module reaching outside that graph is a gate failure rather than a surprise in a layer's bundle. `drawMode.js`, `drawTool.js` and their tests are in `scripts/format-scope.json`. +### S2 GhostMaps (October 2026) + +- **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. +- **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. +- **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. + - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. + - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. + - The new catalog goes to `.gev-cache/ghostmaps/catalog.json` and to every open page over server-sent events (`/api/ghostmaps/events`, 25 s heartbeat, at most 64 subscribers). Default products are downloaded at once. + - The layer reloads only products whose blob changed. The new edition's data source is added before the old one is removed. A product that leaves the catalog leaves the globe. + - The row reads `LIVE` while the stream is open. A five-minute catalog read is the backstop. +- **Files.** `/api/ghostmaps/file/{product}` serves only catalog products. Files are downloaded by commit-pinned raw URLs, so a CDN cannot return an older edition. They are cached on disk by blob sha, and blobs no current product uses are pruned. A request whose `?v=` names the current blob is `immutable`. Repository paths never reach the page. +- **Rendering.** The page parses the KMZ with the import module's zip, KMZ and XML helpers into one `CustomDataSource` per product. + - Points become ground-clamped billboards with the depth test off, so they never sink into terrain or 3D tiles. Because the Google 3D map hides the globe, nothing else would stop far-side pins drawing through the planet. Pins are therefore horizon-culled with Street Level's `createHorizonCull` before their first frame and again whenever the camera moves. + - Lines and polygon outlines are ground-clamped polylines. + - Styles, StyleMaps and inline styles are honoured. Icons come only from images packed in the same KMZ and at most 512 KB. Anything else uses a per-product round marker. + - Network links, overlays, models and remote hrefs are never followed. Hidden placemarks are skipped. Bounds are 20,000 placemarks and 250,000 positions per product. +- **Reports.** Clicking a marker opens a card with the product, folder, edition date and the placemark's report fields. `src/layers/ghostMaps/records.js` reads ArcGIS popup tables as text only: `` and internal fields are dropped and values are capped. Description markup is never inserted into the page. The first `http(s)` research link opens from the card. The card is `stateless`, so it comes and goes with the click, as CCTV cards do. It is also `horizonCull`ed, so it hides when its marker turns behind the planet. Picks are registered under `s2-ghostmaps`. +- **Validation.** Unit tests: `src/layers/ghostMaps/*.test.mjs` cover editions, the catalog, ref parsing, description tables and wrapping; `server/providers/ghostMaps.test.mjs` covers commit detection, events, pinned downloads, cache pruning, restart and routing. `node scripts/qa-ghostmaps.mjs --url http://localhost:4173` covers the live load, the open stream, picks and cards (asserted on the overlay host's diagnostics, since the `desynchronized` overlay canvas can read back a stale buffer headless), far-side culling, a synthetic push, chip persistence, disable and phone width. + ### Import Your Own Files (October 2026) - DISPLAY ▸ **Import** reads GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX from the picker or a file drop anywhere in the app. Each file gets show/hide, Fly-to and Remove controls. GeoJSON and GPX use the row's color; KML/KMZ retain their styles and show a hatched swatch. The file bytes remain on the device for the page session. Rendered overlays can appear in map screenshots sent through voice or other existing screenshot tools. diff --git a/package.json b/package.json index 241c59e59..9256107e4 100644 --- a/package.json +++ b/package.json @@ -168,6 +168,7 @@ "./layers/recent-imagery/source": "./src/layers/recentImagery/source.js", "./layers/perimeters": "./src/layers/perimeters/index.js", "./layers/gnss": "./src/layers/gnss/index.js", + "./layers/ghostmaps": "./src/layers/ghostMaps/index.js", "./sources/firms-csv": "./src/data/firmsCsv.js", "./layers/firms": "./src/layers/firms/index.js", "./layers/satellites": "./src/layers/satellites/index.js", @@ -244,6 +245,7 @@ "./layers/earthquakes/source": "./src/layers/earthquakes/source.js", "./layers/perimeters/source": "./src/layers/perimeters/source.js", "./layers/gnss/source": "./src/layers/gnss/source.js", + "./layers/ghostmaps/source": "./src/layers/ghostMaps/source.js", "./layers/firms/source": "./src/layers/firms/source.js", "./layers/installations/source": "./src/layers/installations/source.js", "./layers/launches/source": "./src/layers/launches/source.js", diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 83e961d8b..46d4fa3c4 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -865,6 +865,17 @@ "src/layers/gnss/source.test.mjs", "src/layers/gnss/ownership.test.mjs", "scripts/qa-gnss.mjs", + "server/providers/ghostMaps.js", + "server/providers/ghostMaps.test.mjs", + "src/app/layers/ghostMaps.js", + "src/layers/ghostMaps/index.js", + "src/layers/ghostMaps/catalog.js", + "src/layers/ghostMaps/kml.js", + "src/layers/ghostMaps/records.js", + "src/layers/ghostMaps/source.js", + "src/layers/ghostMaps/catalog.test.mjs", + "src/layers/ghostMaps/records.test.mjs", + "scripts/qa-ghostmaps.mjs", "src/layers/cyclones/index.js", "src/layers/cyclones/rendering.js", "src/layers/cyclones/labels.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index bf33bebc7..ddd3f6bdf 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2285,6 +2285,12 @@ "src/layers/gnss/index.js", "src/layers/gnss/source.js", "src/layers/gnss/records.js", + "src/app/layers/ghostMaps.js", + "src/layers/ghostMaps/index.js", + "src/layers/ghostMaps/catalog.js", + "src/layers/ghostMaps/records.js", + "src/layers/ghostMaps/kml.js", + "src/layers/ghostMaps/source.js", "src/layers/cctv/sourcePolicy.js", "src/sources/reference.js", "src/search/nominatim.js", @@ -2916,6 +2922,12 @@ "src/layers/gnss/index.js", "src/layers/gnss/source.js", "src/layers/gnss/records.js", + "src/app/layers/ghostMaps.js", + "src/layers/ghostMaps/index.js", + "src/layers/ghostMaps/catalog.js", + "src/layers/ghostMaps/records.js", + "src/layers/ghostMaps/kml.js", + "src/layers/ghostMaps/source.js", "src/layers/cctv/sourcePolicy.js", "src/search/nominatim.js", "src/nominatimGeocode.js", @@ -3067,7 +3079,10 @@ "src/layers/airQuality/source.js", "src/ui/statusText.js", "src/ui/layerNames.js", - "src/layers/streetLevel/providers/mapillary/policy.js" + "src/layers/streetLevel/providers/mapillary/policy.js", + "src/data/geoImportImages.js", + "src/data/geoImportPins.js", + "src/data/geoFileImport.js" ], "external": [ "hls.js", @@ -3759,6 +3774,38 @@ ], "external": [] }, + "ghostmaps": { + "exports": ["./layers/ghostmaps"], + "modules": [ + "src/layers/ghostMaps/index.js", + "src/layers/ghostMaps/catalog.js", + "src/layers/ghostMaps/records.js", + "src/layers/ghostMaps/kml.js", + "src/layers/ghostMaps/source.js", + "src/sources/httpBody.js", + "src/data/geoFileImport.js", + "src/i18n/marker.js", + "src/i18n/index.js", + "src/i18n/locale.js", + "src/i18n/locales/es.json", + "src/i18n/locales/ru.json", + "src/data/geoImportImages.js", + "src/data/geoImportPins.js", + "src/layers/streetLevel/tileMath.js", + "src/layers/streetLevel/view.js" + ], + "external": ["cesium"] + }, + "ghostmaps-source": { + "exports": ["./layers/ghostmaps/source"], + "modules": [ + "src/layers/ghostMaps/source.js", + "src/layers/ghostMaps/catalog.js", + "src/sources/httpBody.js", + "src/i18n/marker.js" + ], + "external": [] + }, "perimeters-source": { "exports": ["./layers/perimeters/source"], "modules": [ diff --git a/scripts/qa-ghostmaps.mjs b/scripts/qa-ghostmaps.mjs new file mode 100644 index 000000000..83dc8ed97 --- /dev/null +++ b/scripts/qa-ghostmaps.mjs @@ -0,0 +1,645 @@ +#!/usr/bin/env node +/** + * qa-ghostmaps.mjs — headless proof for the S2 Underground GhostMaps layer + * (id `s2-ghostmaps`): S2's public KMZ intel products, kept current from + * github.com/s2underground/GhostMaps. + * + * Drives the REAL app in headless Chromium against a dev server (keyless). + * + * (i) LIVE — enabling the layer loads the default products (Common + * Intelligence Picture and the 30-day kinetic pull) from the live + * repository through /api/ghostmaps, one data source per product, + * matching the proxy's catalog commit; the row shows one chip per + * catalog product with the defaults active, and the page holds an + * open event stream to /api/ghostmaps/events. + * (ii) PICK — a Common Intelligence Picture marker at the screen centre + * picks as this layer's entity; clicking it paints its report card + * and clicking empty ground clears it. + * (iii) PUSH (synthetic) — the event stream is replaced in the page and a + * catalog naming a new edition of one product is pushed: only that + * product reloads (a new data source replaces the old one, no + * duplicates), the other keeps its data source. A catalog without a + * product removes it. + * (iv) CHIPS — clicking an active product chip removes that product, + * clicking it again restores it; the choice survives a reload. + * (v) DISABLE — every GhostMaps data source leaves the viewer. + * (vi) VIEWPORT — at a 390x844 phone viewport the open row and its chips + * fit the panel width with no horizontal overflow. + * (vii) FAR SIDE — pins skip the depth test, so they are horizon-culled: + * seen from over the Indian Ocean no pin clearly behind the planet is + * shown; back over North America the near-side pins show again. + * + * Screenshots go to qa-shots/ghostmaps-*.png (gitignored). + * + * Run: node scripts/qa-ghostmaps.mjs --url http://localhost:4173 + * Optional: --timeout-ms 180000 for a slow boot, --shots . + * Exits non-zero on any FAIL. Does not commit anything. + */ + +import puppeteer from 'puppeteer'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const argv = process.argv.slice(2); +const getOpt = (name, dflt) => { + const i = argv.indexOf(name); + return i >= 0 && argv[i + 1] ? argv[i + 1] : dflt; +}; +const APP_URL = getOpt('--url', 'http://localhost:4173'); +const HEADFUL = argv.includes('--headful'); +const TIMEOUT_MS = Number(getOpt('--timeout-ms', '90000')); +const SHOTS_DIR = path.resolve( + getOpt('--shots', path.join(__dirname, '..', 'qa-shots')), +); +const LAYER_ID = 's2-ghostmaps'; + +const results = []; +function record(name, ok, detail) { + results.push({ name, ok, detail }); + const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m'; + console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function boot(page) { + await page.goto(`${APP_URL}/?welcome=0`, { + waitUntil: 'domcontentloaded', + timeout: TIMEOUT_MS, + }); + await page.waitForFunction( + () => window.__godsEyeView?.viewer && window.__godsEyeView?.dataManager, + { timeout: TIMEOUT_MS }, + ); + await sleep(5000); + await page.keyboard.press('Escape'); +} + +/** GhostMaps data sources on the viewer: name → entity count. */ +function ghostSources(prefix) { + const { dataSources } = window.__godsEyeView.viewer; + const out = []; + for (let i = 0; i < dataSources.length; i++) { + const ds = dataSources.get(i); + if (ds.name?.startsWith(prefix)) + out.push({ name: ds.name, count: ds.entities.values.length }); + } + return out; +} + +function paintedOverlayPixels() { + const canvas = document.getElementById('world-overlay-canvas'); + if (!canvas) return -1; + const pixels = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let count = 0; + for (let i = 3; i < pixels.length; i += 4) if (pixels[i] > 0) count++; + return count; +} + +async function enableAndSettle(page) { + await page.evaluate((id) => { + const panel = document.getElementById('data-panel'); + if (panel?.classList.contains('collapsed')) + panel.querySelector('[data-collapse-target="data-panel"]')?.click(); + return window.__godsEyeView.dataManager.setEnabled(id, true); + }, LAYER_ID); + await page.waitForFunction( + (id) => { + const stats = window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats(); + return !stats.loading && stats.count > 0; + }, + { timeout: TIMEOUT_MS, polling: 500 }, + LAYER_ID, + ); +} + +const stats = (page) => + page.evaluate( + (id) => window.__godsEyeView.dataManager.layers.get(id).module.getStats(), + LAYER_ID, + ); +const sources = (page) => + page.evaluate(`(${ghostSources})(${JSON.stringify(`${LAYER_ID}:`)})`); + +async function showRow(page) { + await page.evaluate((id) => { + document + .querySelector(`[data-layer-id="${id}"]`) + ?.scrollIntoView({ block: 'center' }); + }, LAYER_ID); + await sleep(400); +} + +async function main() { + fs.mkdirSync(SHOTS_DIR, { recursive: true }); + const browser = await puppeteer.launch({ + headless: !HEADFUL, + args: ['--use-angle=swiftshader', '--enable-unsafe-swiftshader'], + defaultViewport: { width: 1440, height: 900 }, + }); + try { + // ---- (i) LIVE --------------------------------------------------------- + const page = await browser.newPage(); + // Start clean once; a reload must keep what the page stored. + await page.evaluateOnNewDocument(() => { + try { + if (sessionStorage.getItem('qa-ghostmaps-clean')) return; + localStorage.clear(); + sessionStorage.setItem('qa-ghostmaps-clean', '1'); + } catch { + /* sandboxed frames have no storage */ + } + }); + page.on('pageerror', (error) => + console.log(` [page error] ${error.message}`), + ); + await boot(page); + const catalog = await page.evaluate(() => + fetch('/api/ghostmaps/catalog').then((r) => r.json()), + ); + record( + 'LIVE: proxy catalog names a commit and the default products', + /^[0-9a-f]{40}$/.test(catalog.commit || '') && + ['cip', 'kinetic-30d'].every((id) => + catalog.products.some((p) => p.id === id && p.defaultOn), + ), + `${catalog.commit?.slice(0, 7)} "${catalog.message}", ${catalog.products.length} products`, + ); + const t0 = Date.now(); + await enableAndSettle(page); + const live = await stats(page); + const liveSources = await sources(page); + record( + 'LIVE: default products load as one data source each', + liveSources.length === 2 && + liveSources.every((s) => s.count > 0) && + live.products.sort().join() === 'cip,kinetic-30d', + `${liveSources.map((s) => `${s.name}=${s.count}`).join(', ')} in ${Date.now() - t0} ms`, + ); + record( + 'LIVE: layer reports the proxy commit and no error', + live.commit === catalog.commit && !live.error, + `commit ${live.commit?.slice(0, 7)}, error ${live.error}`, + ); + await showRow(page); + const chips = await page.evaluate( + (id) => + [ + ...document.querySelectorAll( + `[data-layer-id="${id}"] .data-toggle-chip`, + ), + ].map((chip) => ({ + id: chip.dataset.chipId, + active: chip.getAttribute('aria-pressed') === 'true', + })), + LAYER_ID, + ); + record( + 'LIVE: one chip per product, defaults pressed', + chips.length === catalog.products.length && + chips.every( + (chip) => + chip.active === + catalog.products.find((p) => p.id === chip.id)?.defaultOn, + ), + chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '), + ); + const stream = await page + .waitForFunction( + (id) => + window.__godsEyeView.dataManager.layers.get(id).module.getStats() + .stream === 'open', + { timeout: 15000, polling: 250 }, + LAYER_ID, + ) + .then( + () => true, + () => false, + ); + record('LIVE: the /api/ghostmaps/events stream is open', stream); + await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') }); + + // ---- (ii) PICK ---------------------------------------------------------- + const target = await page.evaluate((id) => { + const gev = window.__godsEyeView; + const ds = [...Array(gev.viewer.dataSources.length).keys()] + .map((i) => gev.viewer.dataSources.get(i)) + .find((item) => item.name === `${id}:cip`); + const entity = ds.entities.values.find((e) => e.billboard); + const carto = gev.viewer.scene.globe.ellipsoid.cartesianToCartographic( + entity.position.getValue(gev.viewer.clock.currentTime), + ); + try { + gev.viewer.camera.cancelFlight(); + } catch { + /* no flight */ + } + gev.viewer.camera.setView({ + destination: gev.viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: carto.longitude, + latitude: carto.latitude, + height: 40000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + gev.viewer.scene.requestRender?.(); + return entity.id; + }, LAYER_ID); + await page + .waitForFunction( + () => window.__godsEyeView.viewer.scene.globe.tilesLoaded, + { timeout: 30000 }, + ) + .catch(() => {}); + // The horizon cull reveals the pin on the frame after the camera moves. + await page + .waitForFunction( + () => { + const v = window.__godsEyeView.viewer; + v.scene.requestRender?.(); + const c = v.scene.canvas; + return v.scene + .drillPick({ x: c.clientWidth / 2, y: c.clientHeight / 2 }, 8) + .some((p) => + String(p?.id?.id ?? p?.id ?? '').startsWith('ghostmaps:cip:'), + ); + }, + { timeout: 15000, polling: 500 }, + ) + .catch(() => {}); + const probe = await page.evaluate(() => { + const v = window.__godsEyeView.viewer; + const c = v.scene.canvas; + const box = c.getBoundingClientRect(); + const centre = { x: c.clientWidth / 2, y: c.clientHeight / 2 }; + const picked = v.scene.drillPick(centre, 8); + // Empty ground: the first point below the marker (the card is drawn + // above it) that picks nothing at all; markers can be dense in a city. + let empty = null; + for (let r = 120; r < 420 && !empty; r += 60) + for (const a of [2, 1, 3, 0, 4]) { + const p = { + x: centre.x + r * Math.cos((a * Math.PI) / 4), + y: centre.y + r * Math.sin((a * Math.PI) / 4), + }; + if (!v.scene.pick(p)) { + empty = p; + break; + } + } + empty ||= { x: centre.x + 400, y: centre.y + 300 }; + return { + ids: picked.map((p) => String(p?.id?.id ?? p?.id ?? '')), + centre: { x: centre.x + box.left, y: centre.y + box.top }, + empty: { x: empty.x + box.left, y: empty.y + box.top }, + }; + }); + record( + 'PICK: the centred marker picks as a GhostMaps entity', + probe.ids.some((id) => id.startsWith('ghostmaps:cip:')), + `${target} → [${probe.ids.slice(0, 3).join(', ')}]`, + ); + const before = await page.evaluate(paintedOverlayPixels); + await page.mouse.click(probe.centre.x, probe.centre.y); + const painted = await page + .waitForFunction(`(${paintedOverlayPixels})() > ${before + 100}`, { + timeout: 10000, + polling: 250, + }) + .then( + () => true, + () => false, + ); + const after = await page.evaluate(paintedOverlayPixels); + record( + 'PICK: clicking the marker paints its report card', + painted, + `${before} → ${after} pixels`, + ); + await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-card.png') }); + const cardState = () => + page.evaluate((id) => { + const d = window.__gevWorldOverlay?.getDiagnostics?.(); + return { + entries: d?.entriesBySource?.[id] ?? null, + painted: d?.paintedBySource?.[id] ?? null, + }; + }, LAYER_ID); + const selectedState = await cardState(); + record( + 'PICK: the overlay host holds and paints exactly one card', + selectedState.entries === 1 && selectedState.painted === 1, + JSON.stringify(selectedState), + ); + await page.mouse.click(probe.empty.x, probe.empty.y); + // The overlay canvas is `desynchronized`, so headless read-back can + // return a stale buffer; the host's own model is the authority here. + const cleared = await page + .waitForFunction( + (id) => { + const d = window.__gevWorldOverlay.getDiagnostics(); + return ( + d.entriesBySource?.[id] === 0 && !(d.paintedBySource?.[id] > 0) + ); + }, + { timeout: 10000, polling: 250 }, + LAYER_ID, + ) + .then( + () => true, + () => false, + ); + record( + 'PICK: clicking empty ground clears the card', + cleared, + JSON.stringify(await cardState()), + ); + + // ---- (vii) FAR SIDE --------------------------------------------------- + // Pins skip the depth test; with Google 3D tiles nothing else hides the + // far side. A pin clearly behind the planet (sphere test with margin) must + // be hidden, and pins facing the camera must still show. + const horizonCounts = (lon, lat) => + page.evaluate( + async (id, lo, la) => { + const gev = window.__godsEyeView; + const d2r = Math.PI / 180; + try { + gev.viewer.camera.cancelFlight(); + } catch { + /* no flight */ + } + gev.viewer.camera.setView({ + destination: + gev.viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: lo * d2r, + latitude: la * d2r, + height: 18_000_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + gev.viewer.scene.requestRender?.(); + await new Promise((r) => setTimeout(r, 1200)); + const c = gev.viewer.camera.positionWC; + const norm = (v) => Math.hypot(v.x, v.y, v.z); + let far = 0; + let nearShown = 0; + for (let i = 0; i < gev.viewer.dataSources.length; i++) { + const ds = gev.viewer.dataSources.get(i); + if (!ds.name?.startsWith(`${id}:`)) continue; + for (const e of ds.entities.values) { + if (!e.billboard) continue; + const p = e.position.getValue(gev.viewer.clock.currentTime); + // Angle from the camera's nadir to the pin, against the + // geometric horizon angle; the cull's 1 km margin shows pins up + // to about 1° past it, so "clearly behind" is beyond 2°. + const angle = Math.acos( + (p.x * c.x + p.y * c.y + p.z * c.z) / (norm(p) * norm(c)), + ); + const horizon = Math.acos(norm(p) / norm(c)); + const margin = (2 * Math.PI) / 180; + if (e.show && angle > horizon + margin) far++; + if (e.show && angle < horizon - margin) nearShown++; + } + } + return { far, nearShown }; + }, + LAYER_ID, + lon, + lat, + ); + const fromIndianOcean = await horizonCounts(80, -30); + record( + 'FAR SIDE: pins behind the planet are hidden', + fromIndianOcean.far === 0, + JSON.stringify(fromIndianOcean), + ); + await page.screenshot({ + path: path.join(SHOTS_DIR, 'ghostmaps-far-side.png'), + }); + const fromAmerica = await horizonCounts(-98, 39); + record( + 'FAR SIDE: pins facing the camera show again', + fromAmerica.far === 0 && fromAmerica.nearShown > 100, + JSON.stringify(fromAmerica), + ); + + // ---- (iv) CHIPS ------------------------------------------------------- + const clickChip = async (id) => { + // Row controls render on the next frame after the layer settles. + await page.waitForSelector( + `[data-layer-id="${LAYER_ID}"] .data-toggle-chip[data-chip-id="${id}"]`, + { timeout: 15000 }, + ); + return page.evaluate( + (layer, chip) => + document + .querySelector( + `[data-layer-id="${layer}"] .data-toggle-chip[data-chip-id="${chip}"]`, + ) + ?.click(), + LAYER_ID, + id, + ); + }; + await clickChip('kinetic-30d'); + await page.waitForFunction( + (id) => + !window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats() + .products.includes('kinetic-30d'), + { timeout: 15000 }, + LAYER_ID, + ); + const without = await sources(page); + record( + 'CHIPS: turning a product off removes only its data source', + without.length === 1 && without[0].name === `${LAYER_ID}:cip`, + without.map((s) => s.name).join(', '), + ); + await page.reload({ waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__godsEyeView?.dataManager, { + timeout: TIMEOUT_MS, + }); + await sleep(3000); + await enableAndSettle(page); + const kept = await stats(page); + record( + 'CHIPS: the product choice survives a reload', + kept.products.join() === 'cip', + kept.products.join(', '), + ); + await showRow(page); + await clickChip('kinetic-30d'); + await page.waitForFunction( + (id) => + window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats() + .products.includes('kinetic-30d'), + { timeout: 30000 }, + LAYER_ID, + ); + record( + 'CHIPS: turning it back on restores it', + (await sources(page)).length === 2, + ); + + // ---- (v) DISABLE ------------------------------------------------------ + await page.evaluate( + (id) => window.__godsEyeView.dataManager.setEnabled(id, false), + LAYER_ID, + ); + await sleep(500); + record( + 'DISABLE: every GhostMaps data source leaves the viewer', + (await sources(page)).length === 0, + ); + await page.close(); + + // ---- (iii) PUSH (synthetic) ---------------------------------------------- + const push = await browser.newPage(); + await push.evaluateOnNewDocument(() => { + localStorage.clear(); + // A controllable stand-in for the real stream. + class QaEventSource { + constructor(url) { + this.url = url; + this.listeners = new Set(); + (window.__qaStreams ||= []).push(this); + } + addEventListener(type, listener) { + if (type === 'catalog') this.listeners.add(listener); + } + removeEventListener(type, listener) { + this.listeners.delete(listener); + } + close() { + this.closed = true; + } + emit(catalog) { + for (const listener of this.listeners) + listener({ data: JSON.stringify(catalog) }); + } + } + window.EventSource = QaEventSource; + }); + await boot(push); + await enableAndSettle(push); + const identity = () => + push.evaluate((id) => { + const { dataSources } = window.__godsEyeView.viewer; + window.__qaIds ||= new WeakMap(); + window.__qaNext ||= 1; + const out = {}; + for (let i = 0; i < dataSources.length; i++) { + const ds = dataSources.get(i); + if (!ds.name?.startsWith(`${id}:`)) continue; + if (!window.__qaIds.has(ds)) + window.__qaIds.set(ds, window.__qaNext++); + out[ds.name] = window.__qaIds.get(ds); + } + return out; + }, LAYER_ID); + const first = await identity(); + const pushed = Date.now(); + await push.evaluate(async (id) => { + const catalog = await fetch('/api/ghostmaps/catalog').then((r) => + r.json(), + ); + const kinetic = catalog.products.find((p) => p.id === 'kinetic-30d'); + kinetic.version = 'f'.repeat(40); + catalog.message = 'QA push'; + window.__qaStreams + .filter((s) => !s.closed) + .forEach((s) => s.emit(catalog)); + }, LAYER_ID); + await push.waitForFunction( + (id) => { + const layer = window.__godsEyeView.dataManager.layers.get(id).module; + return ( + !layer.getStats().loading && + layer.getRowControls().info.includes('QA push') + ); + }, + { timeout: 20000, polling: 100 }, + LAYER_ID, + ); + await sleep(500); + const second = await identity(); + record( + 'PUSH: a new edition reloads only that product', + Object.keys(second).length === 2 && + second[`${LAYER_ID}:kinetic-30d`] !== + first[`${LAYER_ID}:kinetic-30d`] && + second[`${LAYER_ID}:cip`] === first[`${LAYER_ID}:cip`], + `${JSON.stringify(first)} → ${JSON.stringify(second)} in ${Date.now() - pushed} ms`, + ); + await push.evaluate(async () => { + const catalog = await fetch('/api/ghostmaps/catalog').then((r) => + r.json(), + ); + catalog.products = catalog.products.filter((p) => p.id !== 'kinetic-30d'); + window.__qaStreams + .filter((s) => !s.closed) + .forEach((s) => s.emit(catalog)); + }); + await sleep(1000); + const third = await identity(); + record( + 'PUSH: a product leaving the catalog leaves the globe', + Object.keys(third).join() === `${LAYER_ID}:cip`, + Object.keys(third).join(', '), + ); + await push.close(); + + // ---- (vi) VIEWPORT ---------------------------------------------------- + const phone = await browser.newPage(); + await phone.setViewport({ width: 390, height: 844, isMobile: true }); + await phone.evaluateOnNewDocument(() => localStorage.clear()); + await boot(phone); + await enableAndSettle(phone); + await showRow(phone); + const fit = await phone.evaluate((id) => { + const row = document.querySelector(`[data-layer-id="${id}"]`); + const panel = document.getElementById('data-panel'); + const controls = row?.querySelector('.data-toggle-controls'); + return { + row: Boolean(row), + overflow: controls ? controls.scrollWidth - controls.clientWidth : -1, + pageOverflow: + document.documentElement.scrollWidth - + document.documentElement.clientWidth, + panelRight: panel?.getBoundingClientRect().right, + width: window.innerWidth, + }; + }, LAYER_ID); + record( + 'VIEWPORT: chips fit the phone-width panel', + fit.row && fit.overflow <= 1 && fit.pageOverflow <= 0, + JSON.stringify(fit), + ); + await phone.screenshot({ + path: path.join(SHOTS_DIR, 'ghostmaps-phone.png'), + }); + await phone.close(); + } finally { + await browser.close(); + } + const failed = results.filter((r) => !r.ok); + console.log( + `\n${results.length - failed.length}/${results.length} passed. Screenshots: ${SHOTS_DIR}`, + ); + process.exit(failed.length ? 1 : 0); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/server/providers/ghostMaps.js b/server/providers/ghostMaps.js new file mode 100644 index 000000000..dd30fd986 --- /dev/null +++ b/server/providers/ghostMaps.js @@ -0,0 +1,431 @@ +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { + GHOSTMAPS_MAX_FILE_BYTES, + GHOSTMAPS_REPO, + buildGhostMapsCatalog, + ghostMapsRawUrl, + parseGitRefAdvertisement, +} from '../../src/layers/ghostMaps/catalog.js'; +import { + readResponseBytesCapped, + readResponseJsonCapped, + readResponseTextCapped, +} from './common/http.js'; + +// S2 Underground GhostMaps (public GitHub repository of KMZ intel products). +// The repository is not ours, so there is no webhook: the watcher reads the +// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the +// REST API rate limit) every few seconds, and only a new commit costs REST +// calls (tree + commit). Browsers hear about a new commit over server-sent +// events and fetch only the products whose blob changed. Files are fetched by +// commit-pinned raw URLs and kept on disk by blob sha. + +const DEFAULT_POLL_MS = 15_000; +const MIN_POLL_MS = 5_000; +const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000]; +const TIMEOUT_MS = 15_000; +const FILE_TIMEOUT_MS = 120_000; +const HEARTBEAT_MS = 25_000; +const MAX_SUBSCRIBERS = 64; +const MAX_REF_BYTES = 64 * 1024; +const MAX_TREE_BYTES = 16 * 1024 * 1024; +const MAX_COMMIT_BYTES = 2 * 1024 * 1024; +const USER_AGENT = 'gods-eye-view-ghostmaps/1.0'; +const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`; + +/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */ +export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) { + const seconds = Number(raw); + if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS; + return Math.max(MIN_POLL_MS, Math.round(seconds * 1000)); +} + +/** + * The repository watcher and file cache, independent of HTTP routing. + * @returns {{start: Function, stop: Function, catalog: Function, + * checkNow: Function, file: Function, subscribe: Function}} + */ +export function createGhostMapsWatcher({ + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), + pollMs = ghostMapsPollMs(), + cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'), + token = () => String(process.env.GITHUB_TOKEN || '').trim(), + log = (...args) => console.log(...args), + warn = (...args) => console.warn(...args), +} = {}) { + let state = { + commit: null, + committedAt: null, + message: null, + products: [], + checkedAt: null, + changedAt: null, + error: null, + }; + let timer = null; + let running = false; + let checking = null; + let failures = 0; + let diskLoaded = false; + const listeners = new Set(); + const downloads = new Map(); + + const apiHeaders = () => ({ + Accept: 'application/vnd.github+json', + 'User-Agent': USER_AGENT, + ...(token() ? { Authorization: `Bearer ${token()}` } : {}), + }); + + async function get(url, headers, timeoutMs = TIMEOUT_MS) { + const response = await fetchImpl(url, { + headers, + redirect: 'follow', + signal: AbortSignal.timeout(timeoutMs), + }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), { + status: response.status, + }); + } + return response; + } + + async function readHead() { + const response = await get( + `${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`, + { 'User-Agent': USER_AGENT }, + ); + const sha = parseGitRefAdvertisement( + await readResponseTextCapped(response, MAX_REF_BYTES), + ); + if (!sha) throw new Error('GhostMaps branch head not advertised'); + return sha; + } + + async function readCommit(sha) { + const api = `https://api.github.com/repos/${REPO_PATH}`; + const tree = await readResponseJsonCapped( + await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()), + MAX_TREE_BYTES, + ); + if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed'); + if (tree.truncated) warn('[ghostmaps] repository tree listing truncated'); + let committedAt = null; + let message = null; + try { + const commit = await readResponseJsonCapped( + await get(`${api}/commits/${sha}`, apiHeaders()), + MAX_COMMIT_BYTES, + ); + committedAt = Date.parse(commit?.commit?.committer?.date) || null; + message = String(commit?.commit?.message || '') + .split('\n')[0] + .slice(0, 200); + } catch { + // The commit date is decoration; the tree is what matters. + } + return { products: buildGhostMapsCatalog(tree.tree), committedAt, message }; + } + + async function loadDisk() { + if (diskLoaded) return; + diskLoaded = true; + try { + const saved = JSON.parse( + await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'), + ); + if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products)) + state = { ...state, ...saved, error: null }; + } catch { + /* first run */ + } + } + + async function saveDisk() { + try { + await fsp.mkdir(cacheDir, { recursive: true }); + const { commit, committedAt, message, products, changedAt } = state; + await fsp.writeFile( + path.join(cacheDir, 'catalog.json'), + JSON.stringify({ commit, committedAt, message, products, changedAt }), + ); + } catch (error) { + warn('[ghostmaps] could not save catalog:', error.message); + } + } + + function emit() { + const event = publicCatalog(); + for (const listener of listeners) { + try { + listener(event); + } catch { + /* a broken subscriber must not stop the rest */ + } + } + } + + async function check() { + await loadDisk(); + const head = await readHead(); + state.checkedAt = now(); + if (head === state.commit) return false; + const next = await readCommit(head); + const previous = new Map(state.products.map((p) => [p.id, p.version])); + const changed = next.products + .filter((p) => previous.get(p.id) !== p.version) + .map((p) => p.id); + state = { + ...state, + ...next, + commit: head, + changedAt: now(), + error: null, + }; + log( + `[ghostmaps] commit ${head.slice(0, 8)}${ + next.message ? ` "${next.message}"` : '' + }; updated: ${changed.join(', ') || 'none'}`, + ); + await saveDisk(); + emit(); + // Warm the default products so the first browser load is local. + for (const product of state.products) + if (product.defaultOn) file(product.id).catch(() => {}); + return true; + } + + /** Run one check now, coalescing with one already in flight. */ + function checkNow() { + if (!checking) + checking = check() + .then( + (changed) => { + failures = 0; + return changed; + }, + (error) => { + failures += 1; + state.error = error.message; + state.checkedAt = now(); + warn('[ghostmaps] check failed:', error.message); + emit(); + return false; + }, + ) + .finally(() => { + checking = null; + }); + return checking; + } + + function schedule() { + clearTimeout(timer); + if (!running) return; + const delay = failures + ? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1] + : pollMs; + timer = setTimeout(async () => { + await checkNow(); + schedule(); + }, delay); + timer.unref?.(); + } + + /** Start watching (idempotent); the first check runs immediately. */ + function start() { + if (running) return checking || Promise.resolve(false); + running = true; + const first = checkNow(); + first.then(schedule); + return first; + } + + function stop() { + running = false; + clearTimeout(timer); + timer = null; + } + + function publicCatalog() { + return { + repo: GHOSTMAPS_REPO.url, + branch: GHOSTMAPS_REPO.branch, + commit: state.commit, + committedAt: state.committedAt, + message: state.message, + checkedAt: state.checkedAt, + changedAt: state.changedAt, + pollMs, + error: state.error, + products: state.products.map( + ({ + id, + title, + defaultOn, + fileName, + fileTitle, + version, + size, + dateMs, + }) => ({ + id, + title, + defaultOn, + fileName, + fileTitle, + version, + size, + dateMs, + }), + ), + }; + } + + /** + * The bytes of one product's current edition, from disk or a commit-pinned + * download. Only catalog paths are ever fetched. + */ + async function file(id) { + const product = state.products.find((p) => p.id === id); + if (!product) return null; + const target = path.join(cacheDir, 'files', `${product.version}.kmz`); + try { + return { product, bytes: await fsp.readFile(target) }; + } catch { + /* not cached yet */ + } + const commit = state.commit; + let pending = downloads.get(product.version); + if (!pending) { + pending = (async () => { + const response = await get( + ghostMapsRawUrl(commit, product.path), + { 'User-Agent': USER_AGENT }, + FILE_TIMEOUT_MS, + ); + const bytes = Buffer.from( + await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES), + ); + await fsp.mkdir(path.dirname(target), { recursive: true }); + const temp = `${target}.${process.pid}.tmp`; + await fsp.writeFile(temp, bytes); + await fsp.rename(temp, target); + await pruneFiles(); + return bytes; + })().finally(() => downloads.delete(product.version)); + downloads.set(product.version, pending); + } + return { product, bytes: await pending }; + } + + /** Keep only files that a current product still points to. */ + async function pruneFiles() { + const keep = new Set(state.products.map((p) => `${p.version}.kmz`)); + const dir = path.join(cacheDir, 'files'); + for (const name of await fsp.readdir(dir).catch(() => [])) { + if (!keep.has(name) && !name.endsWith('.tmp')) + await fsp.rm(path.join(dir, name), { force: true }).catch(() => {}); + } + } + + function subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + } + + return { + start, + stop, + checkNow, + catalog: publicCatalog, + file, + subscribe, + subscribers: () => listeners.size, + }; +} + +/** Same-origin GhostMaps routes for dev and preview. */ +export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) { + const json = (res, status, value) => { + if (res.destroyed) return; + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + ...(status === 405 ? { Allow: 'GET' } : {}), + }); + res.end(JSON.stringify(value)); + }; + + function events(req, res) { + if (watcher.subscribers() >= MAX_SUBSCRIBERS) + return json(res, 429, { error: 'too_many_subscribers' }); + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-store', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no', + }); + const send = (catalog) => + res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`); + res.write('retry: 5000\n\n'); + send(watcher.catalog()); + const unsubscribe = watcher.subscribe(send); + const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS); + heartbeat.unref?.(); + const close = () => { + clearInterval(heartbeat); + unsubscribe(); + }; + req.on('close', close); + res.on('error', close); + } + + async function handler(req, res) { + if (req.method !== 'GET') + return json(res, 405, { error: 'method_not_allowed' }); + const url = new URL(req.url || '/', 'http://local'); + const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname); + if (!['/catalog', '/events'].includes(url.pathname) && !match) + return json(res, 404, { error: 'unknown_route' }); + // Watching starts with the first request, so an install that never turns + // the layer on never contacts GitHub. + const first = watcher.start(); + if (url.pathname === '/catalog') { + if (!watcher.catalog().commit) await first; + return json(res, 200, watcher.catalog()); + } + if (url.pathname === '/events') return events(req, res); + try { + if (!watcher.catalog().commit) await first; + const found = await watcher.file(match[1]); + if (!found) return json(res, 404, { error: 'unknown_product' }); + const pinned = url.searchParams.get('v') === found.product.version; + res.writeHead(200, { + 'Content-Type': 'application/vnd.google-earth.kmz', + 'Content-Length': found.bytes.length, + 'X-GhostMaps-Version': found.product.version, + // A versioned URL names one immutable blob. + 'Cache-Control': pinned + ? 'private, max-age=31536000, immutable' + : 'no-store', + }); + return res.end(found.bytes); + } catch { + return json(res, 502, { error: 'ghostmaps_unavailable' }); + } + } + + return { + name: 'ghostmaps', + configureServer({ middlewares }) { + middlewares.use('/api/ghostmaps', handler); + }, + configurePreviewServer({ middlewares }) { + middlewares.use('/api/ghostmaps', handler); + }, + }; +} diff --git a/server/providers/ghostMaps.test.mjs b/server/providers/ghostMaps.test.mjs new file mode 100644 index 000000000..6251bf4de --- /dev/null +++ b/server/providers/ghostMaps.test.mjs @@ -0,0 +1,227 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { + createGhostMapsWatcher, + ghostMapsPollMs, + ghostMapsProxy, +} from './ghostMaps.js'; + +const MASTER = + 'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/'; +const KINETIC = + 'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz'; +const sha = (c) => c.repeat(40); + +/** A fake GitHub: ref advertisement, tree, commit and raw files. */ +function fakeGitHub() { + const repo = { head: sha('1'), trees: new Map(), calls: [] }; + repo.trees.set(sha('1'), [ + { + path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`, + type: 'blob', + size: 4, + sha: sha('a'), + }, + { path: KINETIC, type: 'blob', size: 4, sha: sha('b') }, + ]); + const body = (text, status = 200) => + new Response(text, { + status, + headers: { 'content-length': String(Buffer.byteLength(text)) }, + }); + const fetchImpl = async (url) => { + repo.calls.push(url); + if (url.endsWith('/info/refs?service=git-upload-pack')) + return body( + `001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`, + ); + const tree = /git\/trees\/([0-9a-f]{40})/.exec(url); + if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) })); + if (/\/commits\//.test(url)) + return body( + JSON.stringify({ + commit: { + message: 'Oct 18 Update\nmore', + committer: { date: '2026-10-18T00:00:00Z' }, + }, + }), + ); + if (url.startsWith('https://raw.githubusercontent.com/')) + return body('KMZ!'); + return body('nope', 404); + }; + return { repo, fetchImpl }; +} + +async function withCache(fn) { + const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-')); + try { + await fn(dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +test('a new commit is announced once, naming only the products that changed', () => + withCache(async (cacheDir) => { + const { repo, fetchImpl } = fakeGitHub(); + const watcher = createGhostMapsWatcher({ + fetchImpl, + cacheDir, + log: () => {}, + warn: () => {}, + }); + const events = []; + watcher.subscribe((catalog) => events.push(catalog)); + + assert.equal(await watcher.checkNow(), true); + const first = watcher.catalog(); + assert.equal(first.commit, sha('1')); + assert.equal(first.message, 'Oct 18 Update'); + assert.deepEqual( + first.products.map((p) => p.id), + ['cip', 'kinetic-30d'], + ); + assert.equal(events.length, 1); + + // Same head: one ref read, no REST calls, no event. + repo.calls.length = 0; + assert.equal(await watcher.checkNow(), false); + assert.equal(repo.calls.length, 1); + assert.equal(events.length, 1); + + // A new upload: the CIP gets a newer edition, the 30-day pull is untouched. + repo.head = sha('2'); + repo.trees.set(sha('2'), [ + ...repo.trees.get(sha('1')), + { + path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`, + type: 'blob', + size: 4, + sha: sha('c'), + }, + ]); + assert.equal(await watcher.checkNow(), true); + assert.equal(events.length, 2); + const cip = events[1].products.find((p) => p.id === 'cip'); + assert.equal(cip.version, sha('c')); + assert.match(cip.fileName, /October_18/); + assert.equal( + events[1].products.find((p) => p.id === 'kinetic-30d').version, + sha('b'), + ); + // Repository paths stay on the server. + assert.equal('path' in cip, false); + watcher.stop(); + })); + +test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () => + withCache(async (cacheDir) => { + const { repo, fetchImpl } = fakeGitHub(); + const watcher = createGhostMapsWatcher({ + fetchImpl, + cacheDir, + log: () => {}, + warn: () => {}, + }); + await watcher.checkNow(); + const found = await watcher.file('kinetic-30d'); + assert.equal(found.bytes.toString(), 'KMZ!'); + const raw = repo.calls.filter((url) => + url.includes('raw.githubusercontent'), + ); + assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`))); + assert.ok( + raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')), + ); + const before = repo.calls.length; + await watcher.file('kinetic-30d'); + assert.equal(repo.calls.length, before, 'second read comes from disk'); + assert.equal(await watcher.file('not-a-product'), null); + + // When the edition changes, the old blob leaves the disk cache. + await watcher.file('cip'); + repo.head = sha('2'); + repo.trees.set(sha('2'), [ + { + path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`, + type: 'blob', + size: 4, + sha: sha('c'), + }, + { path: KINETIC, type: 'blob', size: 4, sha: sha('b') }, + ]); + await watcher.checkNow(); + await watcher.file('cip'); + const files = (await readdir(path.join(cacheDir, 'files'))).sort(); + assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]); + watcher.stop(); + })); + +test('the catalog survives a restart from disk, and upstream failures are reported', () => + withCache(async (cacheDir) => { + const { fetchImpl } = fakeGitHub(); + const quiet = { log: () => {}, warn: () => {} }; + await createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet }).checkNow(); + const offline = createGhostMapsWatcher({ + fetchImpl: async () => new Response('down', { status: 503 }), + cacheDir, + ...quiet, + }); + assert.equal(await offline.checkNow(), false); + const catalog = offline.catalog(); + assert.equal(catalog.commit, sha('1')); + assert.equal(catalog.products.length, 2); + assert.match(catalog.error, /HTTP 503/); + })); + +test('poll interval is configurable with a 5 s floor', () => { + assert.equal(ghostMapsPollMs(undefined), 15000); + assert.equal(ghostMapsPollMs('30'), 30000); + assert.equal(ghostMapsPollMs('1'), 5000); + assert.equal(ghostMapsPollMs('soon'), 15000); +}); + +test('routes: unknown paths and methods are refused without contacting GitHub', async () => { + let started = 0; + const watcher = { + start: () => { + started += 1; + return Promise.resolve(false); + }, + catalog: () => ({ commit: sha('1'), products: [] }), + file: async () => null, + subscribe: () => () => {}, + subscribers: () => 0, + }; + let handler; + ghostMapsProxy({ watcher }).configureServer({ + middlewares: { use: (_route, fn) => (handler = fn) }, + }); + const call = async (method, url) => { + const res = { + destroyed: false, + writeHead(status, headers) { + this.status = status; + this.headers = headers; + }, + end(body) { + this.body = String(body ?? ''); + }, + }; + await handler({ method, url }, res); + return res; + }; + assert.equal((await call('GET', '/nope')).status, 404); + assert.equal((await call('POST', '/catalog')).status, 405); + assert.equal((await call('GET', '/file/../../etc')).status, 404); + assert.equal(started, 0); + assert.equal((await call('GET', '/file/cip')).status, 404); + const catalog = await call('GET', '/catalog'); + assert.equal(catalog.status, 200); + assert.equal(JSON.parse(catalog.body).commit, sha('1')); + assert.equal(started, 2); +}); diff --git a/server/providers/local.js b/server/providers/local.js index 5d39c3a01..d9e7b7919 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -26,6 +26,7 @@ import { keySetupEndpoint } from '../standalone/key-setup.js'; import { weatherProxy } from './weather.js'; import { firePerimetersProxy } from './firePerimeters.js'; import { gnssIntegrityProxy } from './gnssIntegrity.js'; +import { ghostMapsProxy } from './ghostMaps.js'; import { cycloneProxy } from './cyclones.js'; import { windProxy } from './wind.js'; @@ -64,6 +65,7 @@ function localProviderPlugins({ realtime, gemini } = {}) { mapillaryProxy(), firePerimetersProxy(), gnssIntegrityProxy(), + ghostMapsProxy(), keySetupEndpoint(), ]; } diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 832c174bc..7c149ab59 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -27,6 +27,7 @@ import { createApplicationEarthquakes } from './layers/earthquakes.js'; import { createApplicationAirQuality } from './layers/airQuality.js'; import { createApplicationFirePerimeters } from './layers/perimeters.js'; import { createApplicationGnssIntegrity } from './layers/gnss.js'; +import { createApplicationGhostMaps } from './layers/ghostMaps.js'; import { createApplicationCables } from './layers/submarineCables.js'; import { createApplicationStreetLevel } from './layers/streetLevel.js'; import { createInfrastructureLayers } from '../data/infrastructure.js'; @@ -126,6 +127,7 @@ export function createApplicationCatalog({ createApplicationGnssIntegrity({ source: sources['gnss-integrity'], }), + createApplicationGhostMaps(), createApplicationAlpr({ surface, source: sources.alpr }), createApplicationStreetLevel({ surface, diff --git a/src/app/constructCatalog.test.mjs b/src/app/constructCatalog.test.mjs index 9e1059c33..eb9e88d31 100644 --- a/src/app/constructCatalog.test.mjs +++ b/src/app/constructCatalog.test.mjs @@ -40,7 +40,7 @@ test('catalogs construct distinct layers and classification from their supplied signal: b.signal, surface: fixtureSurface(b.signal), }); - assert.equal(first.layers.length, 32); + assert.equal(first.layers.length, 33); assert.ok(first.get('local-adsb'), 'Local ADS-B is registered'); assert.deepEqual( first.metadata.find(({ id }) => id === 'local-adsb'), @@ -61,6 +61,7 @@ test('catalogs construct distinct layers and classification from their supplied ); assert.ok(first.get('fire-perimeters')); assert.ok(first.get('gnss-integrity')); + assert.ok(first.get('s2-ghostmaps')); assert.ok(first.get('transit')); const order = first.layers.map(({ id }) => id); assert.deepEqual( @@ -154,7 +155,7 @@ test('Mapillary can be omitted or replaced without changing catalog membership', surface: fixtureSurface(lifetime.signal), }; const catalog = createApplicationCatalog(options); - assert.equal(catalog.layers.length, 32); + assert.equal(catalog.layers.length, 33); assert.deepEqual(catalog.get('street-level').providerIds, []); assert.equal( catalogControlServices(catalog).streetLevelLayer, diff --git a/src/app/layers/ghostMaps.js b/src/app/layers/ghostMaps.js new file mode 100644 index 000000000..5e373c3d8 --- /dev/null +++ b/src/app/layers/ghostMaps.js @@ -0,0 +1,27 @@ +import * as Cesium from 'cesium'; +import { + createGhostMapsLayer, + createGhostMapsSource, +} from '../../layers/ghostMaps/index.js'; +import * as picking from '../../data/pickRegistry.js'; +import * as overlays from '../../overlays/worldOverlay.js'; +import { isPointerFree } from '../../data/inputOwnership.js'; + +/** Wire the S2 Underground GhostMaps layer into the application catalog. */ +export function createApplicationGhostMaps(options) { + return createGhostMapsLayer({ + source: createGhostMapsSource(), + overlayHost: { + setEntries: overlays.setOverlayEntries, + setVisible: overlays.setOverlaySourceVisible, + clearSource: overlays.clearOverlaySource, + hitTest: overlays.hitTestWorldOverlay, + }, + openExternal: (url) => window.open(url, '_blank', 'noopener,noreferrer'), + screenSpaceEventHandlerFactory: (viewer) => + new Cesium.ScreenSpaceEventHandler(viewer.scene.canvas), + picking, + pointer: { isPointerFree }, + ...options, + }); +} diff --git a/src/app/sourceComposition.test.mjs b/src/app/sourceComposition.test.mjs index 422308f1e..bfb031026 100644 --- a/src/app/sourceComposition.test.mjs +++ b/src/app/sourceComposition.test.mjs @@ -37,7 +37,7 @@ test('every acquisition source can be omitted without losing catalog membership' test('all acquisition sources may be absent, and enable fails before initializing or fetching', async (t) => { const actual = catalog({}, t); - assert.equal(actual.layers.length, 32); + assert.equal(actual.layers.length, 33); const manager = new LayerLifecycle( {}, { getSourceAvailability: actual.getSourceAvailability }, diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index a86f43465..e7a523766 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -441,6 +441,14 @@ export const DATA_CREDITS = [ '(earthdata.nasa.gov/firms), ' + 'part of NASA’s Earth Observing System Data and Information System (EOSDIS)', }, + { + key: 's2-ghostmaps', + html: + 'S2 GhostMaps intel products: S2 Underground © by S2A1, ' + + 'github.com/s2underground/GhostMaps, ' + + 'licensed under CC BY-NC-SA 4.0. ' + + 'Shown as published; reports are S2 assessments, not verified fact.', + }, { key: 'drivebc-cctv', html: diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index fc6a5e079..86c60cb94 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -1040,7 +1040,13 @@ export function detachImportedTime(dataSource) { return dataSource; } -function parseXml(text, what) { +/** + * Parse KML or GPX text with the namespace repairs Cesium applies. + * @param {string} text + * @param {string} what Format name for the error message. + * @returns {Document} + */ +export function parseXml(text, what) { // Match the namespace repairs Cesium applies to ordinary exported KML. if (what !== 'GPX') text = String(text).replace(/]*>/i, (root) => { @@ -1073,7 +1079,8 @@ function parseXml(text, what) { return doc; } -function decodeXml(bytes) { +/** Decode XML bytes, honouring a UTF-16 byte-order mark. */ +export function decodeXml(bytes) { const encoding = bytes[0] === 0xff && bytes[1] === 0xfe ? 'utf-16le' diff --git a/src/data/layerState.js b/src/data/layerState.js index dd79eb12a..aa366a0e6 100644 --- a/src/data/layerState.js +++ b/src/data/layerState.js @@ -675,6 +675,11 @@ export const LAYER_STATE_REGISTRY = Object.freeze([ token: 'x', disposition: 'enabled-only', }), + Object.freeze({ + id: 's2-ghostmaps', + token: '5', + disposition: 'enabled-only', + }), Object.freeze({ id: 'satellites', token: 's', diff --git a/src/data/layerState.test.mjs b/src/data/layerState.test.mjs index 77b8f902f..ba5496227 100644 --- a/src/data/layerState.test.mjs +++ b/src/data/layerState.test.mjs @@ -202,8 +202,8 @@ function encode(state) { test('production registry is exact, canonical, and rejects incomplete contracts', async () => { assert.equal(validateLayerStateRegistry(), true); - assert.equal(REGISTERED_LAYER_IDS.length, 31); - assert.equal(new Set(REGISTERED_LAYER_IDS).size, 31); + assert.equal(REGISTERED_LAYER_IDS.length, 32); + assert.equal(new Set(REGISTERED_LAYER_IDS).size, 32); assert.ok(REGISTERED_LAYER_IDS.includes('transit')); assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort()); assert.deepEqual(LEGACY_LAYER_STATE_TOKENS, { @@ -242,19 +242,19 @@ test('production registry is exact, canonical, and rejects incomplete contracts' for (const [id, token] of Object.entries(LEGACY_LAYER_STATE_TOKENS)) { assert.equal(LAYER_STATE_TOKEN_RESERVATIONS[id], token); } - assert.equal(nextLayerStateToken(), '5'); + assert.equal(nextLayerStateToken(), '6'); assert.equal( nextLayerStateToken({ ...LAYER_STATE_TOKEN_RESERVATIONS, - alpha: '5', - bravo: '6', + alpha: '6', + bravo: '7', }), - '7', + '8', ); const digitsExhausted = { ...LAYER_STATE_TOKEN_RESERVATIONS, ...Object.fromEntries( - [...'56789'].map((digit) => [`prior-${digit}`, digit]), + [...'6789'].map((digit) => [`prior-${digit}`, digit]), ), }; assert.equal(nextLayerStateToken(digitsExhausted), '00'); @@ -307,8 +307,8 @@ test('production registry is exact, canonical, and rejects incomplete contracts' assert.equal( validateLayerStateAllocations(LAYER_STATE_TOKEN_RESERVATIONS, { ...LAYER_STATE_TOKEN_RESERVATIONS, - future: '5', - next: '6', + future: '6', + next: '7', }), true, ); @@ -318,7 +318,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' ...LAYER_STATE_TOKEN_RESERVATIONS, future: '00', }), - /next free token 5/, + /next free token 6/, ); const beforeLastDigit = { ...digitsExhausted }; delete beforeLastDigit['prior-9']; @@ -341,10 +341,10 @@ test('production registry is exact, canonical, and rejects incomplete contracts' assert.throws( () => validateLayerStateAllocations( - { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '5' }, - { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '5', competing: '5' }, + { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '6' }, + { ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '6', competing: '6' }, ), - /next free token 6/, + /next free token 7/, ); assert.throws( () => validateLayerStateAllocations({ future: '00' }, { future: '01' }), diff --git a/src/data/layerStateTokenLedger.test.mjs b/src/data/layerStateTokenLedger.test.mjs index 7bcfd96cb..47b55b067 100644 --- a/src/data/layerStateTokenLedger.test.mjs +++ b/src/data/layerStateTokenLedger.test.mjs @@ -128,7 +128,7 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed { ...parseLayerStateTokenReservations(reservationRows) }, { ...LAYER_STATE_TOKEN_RESERVATIONS }, ); - assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 31); + assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 32); assert.deepEqual( { ...LAYER_STATE_TOKEN_RESERVATIONS }, { @@ -136,6 +136,7 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed 'street-level': '0', 'gnss-integrity': '3', 'air-quality': '4', + 's2-ghostmaps': '5', }, ); assert.throws( @@ -161,9 +162,9 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed /Duplicate layer-state token reservation/, ); assert.equal( - parseLayerStateTokenReservations([...reservationRows, ['future', '5']]) + parseLayerStateTokenReservations([...reservationRows, ['future', '6']]) .future, - '5', + '6', ); assert.throws( () => @@ -181,14 +182,14 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed test('published retired digits and competing PRs advance the merge-time allocation', () => { withPublishedBase( - { rows: [...reservationRows, ['retired-layer', '5']] }, + { rows: [...reservationRows, ['retired-layer', '6']] }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '6'); + assert.equal(nextLayerStateToken(published), '7'); assert.equal( validateLayerStateAllocations(published, { ...published, - newcomer: '6', + newcomer: '7', }), true, ); @@ -196,15 +197,15 @@ test('published retired digits and competing PRs advance the merge-time allocati () => validateLayerStateAllocations(published, { ...published, - competing: '5', + competing: '6', }), - /next free token 6/, + /next free token 7/, ); assert.throws( () => validateLayerStateAllocations(published, { ...LAYER_STATE_TOKEN_RESERVATIONS, - newcomer: '5', + newcomer: '6', }), /changed or removed: retired-layer/, ); @@ -212,8 +213,8 @@ test('published retired digits and competing PRs advance the merge-time allocati ); }); -test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => { - withPublishedBase({ rows: [...reservationRows, ['pr-a', '5']] }, (cwd) => { +test('PR B manually replaces provisional 6 with 7 after PR A publishes 6', () => { + withPublishedBase({ rows: [...reservationRows, ['pr-a', '6']] }, (cwd) => { mkdirSync(path.join(cwd, 'scripts')); const fixtureChecker = path.join( cwd, @@ -234,8 +235,8 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-b', '5']], - [{ id: 'pr-b', token: '5' }], + [...reservationRows, ['pr-b', '6']], + [{ id: 'pr-b', token: '6' }], ); const beforeRebase = runCheck(); assert.equal(beforeRebase.status, 1); @@ -243,10 +244,10 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '5'], ['pr-b', '5']], + [...reservationRows, ['pr-a', '6'], ['pr-b', '6']], [ - { id: 'pr-a', token: '5' }, - { id: 'pr-b', token: '5' }, + { id: 'pr-a', token: '6' }, + { id: 'pr-b', token: '6' }, ], ); const staleRebase = runCheck(); @@ -254,16 +255,16 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => assert.match(staleRebase.stderr, /Duplicate layer-state token reservation/); const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '6'); - const validBaselineRows = [...reservationRows, ['pr-a', '5']]; - writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '5' }]); + assert.equal(nextLayerStateToken(published), '7'); + const validBaselineRows = [...reservationRows, ['pr-a', '6']]; + writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '6' }]); const helper = spawnSync( process.execPath, [realpathSync(fixtureNext), 'pr-b'], { cwd, encoding: 'utf8' }, ); assert.equal(helper.status, 0, helper.stderr); - assert.match(helper.stdout, /pr-b: 6/); + assert.match(helper.stdout, /pr-b: 7/); assert.deepEqual( JSON.parse(readFileSync(path.join(cwd, LEDGER_PATH), 'utf8')), validBaselineRows, @@ -271,20 +272,20 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => ); writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '5'], ['pr-b', '6']], + [...reservationRows, ['pr-a', '6'], ['pr-b', '7']], [ - { id: 'pr-a', token: '5' }, - { id: 'pr-b', token: '6' }, + { id: 'pr-a', token: '6' }, + { id: 'pr-b', token: '7' }, ], ); const corrected = runCheck(); assert.equal(corrected.status, 0, corrected.stderr); - assert.match(corrected.stdout, /32 published, 1 new/); + assert.match(corrected.stdout, /33 published, 1 new/); }); }); test('allocation batches cross the last digit and base-36 pair boundaries in order', () => { - const digitRows = [...'56789'].map((digit) => [`retired-${digit}`, digit]); + const digitRows = [...'6789'].map((digit) => [`retired-${digit}`, digit]); const allDigits = parseLayerStateTokenReservations([ ...reservationRows, ...digitRows, @@ -336,7 +337,7 @@ test('allocation batches cross the last digit and base-36 pair boundaries in ord test('an isolated valid two-character fixture round-trips an l field beyond the old 64-character cap', async () => { const cwd = mkdtempSync(path.join(tmpdir(), 'gev-layer-token-width-')); try { - const priorDigits = [...'56789'].map((digit) => [ + const priorDigits = [...'6789'].map((digit) => [ `qa-prior-digit-${digit}`, digit, ]); @@ -373,7 +374,7 @@ test('an isolated valid two-character fixture round-trips an l field beyond the const params = new URLSearchParams([['v', '2']]); codec.encodeLayerStateParams(params, state); assert.ok(params.get('l').length > 64, 'fixture must cross the old cap'); - assert.equal(params.get('l').length, 73); + assert.equal(params.get('l').length, 75); const restored = codec.decodeLayerStateParams(params); assert.deepEqual(restored?.enabledLayerIds, expectedLayerIds); assert.deepEqual(restored?.options, state.options); @@ -391,7 +392,7 @@ test('checker reads a complete future base ledger and rejects retired-token reus }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(Object.keys(published).length, 32); + assert.equal(Object.keys(published).length, 33); assert.equal(published['retired-layer'], '00'); assert.throws( () => diff --git a/src/data/layerStateTokenReservations.json b/src/data/layerStateTokenReservations.json index dd8dcd7c4..7593425ae 100644 --- a/src/data/layerStateTokenReservations.json +++ b/src/data/layerStateTokenReservations.json @@ -20,6 +20,7 @@ ["radio", "r"], ["recent-imagery", "1"], ["rocket-launches", "x"], + ["s2-ghostmaps", "5"], ["satellites", "s"], ["street-level", "0"], ["telegeography-submarine-cables", "u"], diff --git a/src/firstRunExperience.test.mjs b/src/firstRunExperience.test.mjs index 749123acd..acf4d4bfe 100644 --- a/src/firstRunExperience.test.mjs +++ b/src/firstRunExperience.test.mjs @@ -995,10 +995,11 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping // list descriptions; the missions still ride existing tools. // Street Level and GNSS extend the manifest; broadcast caveats describe tracked and analyst aircraft. // Air quality (ECCC AQHI) extends the manifest. - assert.equal(block.length, 30655, 'serialized tool schema length drifted'); + // S2 GhostMaps extends the manifest. + assert.equal(block.length, 30723, 'serialized tool schema length drifted'); assert.equal( crypto.createHash('sha256').update(block).digest('hex'), - '3b1d5e77c10a92f6543a4f9d1ff1870101a0306b65e5b8a6113111c81eda371e', + '5937f0e90d38bf4109136c71bb9561298dd3fca11464ff980a1f982b6c154e5b', 'the first-run missions must ride EXISTING tools: no schema edit, no cache bust', ); const instructions = fs.readFileSync( diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index d86ab843d..90da6d197 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -2937,5 +2937,33 @@ "Gemini access was refused. Check the API key and model access.": "Se denegó el acceso a Gemini. Comprueba la clave de API y el acceso al modelo.", "Gemini Live is unavailable for the configured model.": "Gemini Live no está disponible para el modelo configurado.", "Token request is too large": "La solicitud de credencial es demasiado grande", - "Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío" + "Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío", + "S2 GhostMaps": "S2 GhostMaps", + "Intel Picture": "Panorama de inteligencia", + "Kinetic 30d": "Cinético 30 d", + "Border Crisis": "Crisis fronteriza", + "Iran Picture": "Panorama de Irán", + "Iran Kinetic": "Irán cinético", + "Yemen FLOT": "FLOT de Yemen", + "Latest Drop": "Última publicación", + "GhostMaps file could not be read": "No se pudo leer el archivo de GhostMaps", + "GhostMaps catalog unavailable": "Catálogo de GhostMaps no disponible", + "Malformed GhostMaps catalog": "Catálogo de GhostMaps mal formado", + "{count} source links ↗ · click card to open the first": { + "one": "{count} enlace a la fuente ↗ · haz clic en la tarjeta para abrirlo", + "many": "{count} de enlaces a fuentes ↗ · haz clic en la tarjeta para abrir el primero", + "other": "{count} enlaces a fuentes ↗ · haz clic en la tarjeta para abrir el primero" + }, + "S2 · {name}": "S2 · {name}", + "Report": "Informe", + "Open the source for {title}": "Abrir la fuente de {title}", + "LIVE · GitHub {commit} · {message} · checked every {seconds}s": "EN VIVO · GitHub {commit} · {message} · comprobado cada {seconds} s", + "GitHub {commit} · {message} · reconnecting for live updates": "GitHub {commit} · {message} · reconectando para recibir actualizaciones en vivo", + "Connecting to the S2 GhostMaps repository…": "Conectando con el repositorio S2 GhostMaps…", + "{count} placemarks": { + "one": "{count} marcador", + "many": "{count} de marcadores", + "other": "{count} marcadores" + }, + "S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.": "S2 Underground publica estos productos KMZ en GitHub. Las nuevas publicaciones se detectan en segundos y sustituyen automáticamente a la edición anterior. Haz clic en un marcador para ver su informe." } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index 7ee31fd1c..25bcead93 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3004,5 +3004,35 @@ "Gemini access was refused. Check the API key and model access.": "Доступ к Gemini запрещён. Проверьте ключ API и доступ к модели.", "Gemini Live is unavailable for the configured model.": "Gemini Live недоступен для выбранной модели.", "Token request is too large": "Запрос токена слишком велик", - "Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON" + "Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON", + "S2 GhostMaps": "S2 GhostMaps", + "Intel Picture": "Разведкартина", + "Kinetic 30d": "Боевые 30 дн.", + "Border Crisis": "Кризис на границе", + "Iran Picture": "Иран: картина", + "Iran Kinetic": "Иран: боевые", + "Yemen FLOT": "Йемен: линия фронта", + "Latest Drop": "Новый выпуск", + "GhostMaps file could not be read": "Не удалось прочитать файл GhostMaps", + "GhostMaps catalog unavailable": "Каталог GhostMaps недоступен", + "Malformed GhostMaps catalog": "Повреждённый каталог GhostMaps", + "{count} source links ↗ · click card to open the first": { + "one": "{count} ссылка на источник ↗ · нажмите на карточку, чтобы открыть", + "few": "{count} ссылки на источники ↗ · нажмите на карточку, чтобы открыть первую", + "many": "{count} ссылок на источники ↗ · нажмите на карточку, чтобы открыть первую", + "other": "{count} ссылки на источники ↗ · нажмите на карточку, чтобы открыть первую" + }, + "S2 · {name}": "S2 · {name}", + "Report": "Сводка", + "Open the source for {title}": "Открыть источник: {title}", + "LIVE · GitHub {commit} · {message} · checked every {seconds}s": "В ЭФИРЕ · GitHub {commit} · {message} · проверка каждые {seconds} с", + "GitHub {commit} · {message} · reconnecting for live updates": "GitHub {commit} · {message} · переподключение для обновлений в реальном времени", + "Connecting to the S2 GhostMaps repository…": "Подключение к репозиторию S2 GhostMaps…", + "{count} placemarks": { + "one": "{count} метка", + "few": "{count} метки", + "many": "{count} меток", + "other": "{count} метки" + }, + "S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.": "S2 Underground публикует эти продукты KMZ на GitHub. Новые файлы обнаруживаются за секунды и автоматически заменяют предыдущий выпуск. Нажмите на метку, чтобы открыть её сводку." } diff --git a/src/i18n/same-as-english.json b/src/i18n/same-as-english.json index c159beaab..722f1c942 100644 --- a/src/i18n/same-as-english.json +++ b/src/i18n/same-as-english.json @@ -74,6 +74,8 @@ "Radio", "Reggae", "Rock", + "S2 GhostMaps", + "S2 · {name}", "SAR", "SE", "SECTOR {lat} {lon}", @@ -131,6 +133,8 @@ "NVG", "R&B", "REC", + "S2 GhostMaps", + "S2 · {name}", "VS", "WHOT/BHOT" ] diff --git a/src/layers/ghostMaps/catalog.js b/src/layers/ghostMaps/catalog.js new file mode 100644 index 000000000..4fb68c948 --- /dev/null +++ b/src/layers/ghostMaps/catalog.js @@ -0,0 +1,245 @@ +// S2 Underground GhostMaps catalog: which KMZ in the public repository is the +// current edition of each product. Pure and portable (the server builds the +// catalog, tests pin the rules); no i18n, no DOM, no Cesium. + +export const GHOSTMAPS_REPO = Object.freeze({ + owner: 's2underground', + name: 'GhostMaps', + branch: 'main', + url: 'https://github.com/s2underground/GhostMaps', +}); + +/** Largest KMZ the proxy downloads; matches the browser's import ceiling. */ +export const GHOSTMAPS_MAX_FILE_BYTES = 50 * 1024 * 1024; + +const ARCGIS = 'ArcGIS Data for ATAK (KMZs)/'; +const INCIDENTS = 'Incident Response Products for ATAK (KMZs and Documents)/'; + +/** + * Curated products, in panel order. Each names a series of dated uploads; the + * newest edition wins. Patterns match whole repository paths, so a renamed + * folder fails closed (the product drops out) instead of matching elsewhere. + */ +export const GHOSTMAPS_PRODUCTS = Object.freeze([ + Object.freeze({ + id: 'cip', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Common Intelligence Picture', + defaultOn: true, + pattern: new RegExp( + `^${escape(ARCGIS)}Common Intelligence Picture/Master Database/[^/]+\\.kmz$`, + 'i', + ), + }), + Object.freeze({ + id: 'kinetic-30d', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Kinetic Activity (30 days)', + defaultOn: true, + pattern: new RegExp( + `^${escape(ARCGIS)}Common Intelligence Picture/30-Day Pulls/Kinetic[^/]*\\.kmz$`, + 'i', + ), + }), + Object.freeze({ + id: 'border-crisis', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Border Crisis Map', + defaultOn: false, + pattern: new RegExp( + `^${escape(ARCGIS)}Border Crisis Map/S2Underground_Border_Crisis_Map[^/]*\\.kmz$`, + 'i', + ), + }), + Object.freeze({ + id: 'iran-cip', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Iran Common Intelligence Picture', + defaultOn: false, + pattern: new RegExp( + `^${escape(INCIDENTS)}Iran[^/]*/Iran Common Intelligence Picture/[^/]+\\.kmz$`, + 'i', + ), + }), + Object.freeze({ + id: 'iran-kinetic', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Iran Kinetic Events', + defaultOn: false, + pattern: new RegExp( + `^${escape(INCIDENTS)}Iran[^/]*/Iran Kinetic Events/[^/]+\\.kmz$`, + 'i', + ), + }), + Object.freeze({ + id: 'yemen-flot', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Yemen FLOT', + defaultOn: false, + pattern: new RegExp( + `^${escape(INCIDENTS)}Yemen[^/]*/Yemen FLOT[^/]*\\.kmz$`, + 'i', + ), + }), +]); + +/** The newest incident product outside the curated series and the archive. */ +export const GHOSTMAPS_LATEST_DROP = Object.freeze({ + id: 'latest-drop', + // i18n-ignore: catalog data; the panel shows its own translated label + title: 'Latest incident drop', + defaultOn: false, +}); + +const LATEST_DROP_EXCLUDED = + /\/(?:0_Archive|Basedata|Reference Material)(?:\/|$)/i; + +function escape(text) { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +const MONTHS = [ + 'jan', + 'feb', + 'mar', + 'apr', + 'may', + 'jun', + 'jul', + 'aug', + 'sep', + 'oct', + 'nov', + 'dec', +]; +const DATE_IN_NAME = + /(january|february|march|april|may|june|july|august|september|sept|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)[ _-]?(\d{1,2})(?:st|nd|rd|th)?[ ,_-]+(20\d\d)/i; +const EDITION_IN_NAME = /(?:[ _-]export)?[ _-]v?(\d{1,3})$/i; + +/** + * The date and edition written into an S2 file name, e.g. + * `..._October_16_2026_Export_1.kmz` → 2026-10-16, edition 1. Undated names + * report `dateMs: null` and sort below every dated edition. + * @param {string} fileName + * @returns {{dateMs: ?number, edition: number}} + */ +export function parseGhostMapsEdition(fileName) { + const stem = String(fileName || '').replace(/\.km[lz]$/i, ''); + const match = DATE_IN_NAME.exec(stem); + let dateMs = null; + if (match) { + const month = MONTHS.indexOf(match[1].slice(0, 3).toLowerCase()); + const day = Number(match[2]); + const year = Number(match[3]); + const at = Date.UTC(year, month, day); + // Date.UTC rolls 31 Sep into 1 Oct; only an exact calendar date counts. + if (month >= 0 && new Date(at).getUTCDate() === day) dateMs = at; + } + const tail = match ? stem.slice(match.index + match[0].length) : stem; + const edition = Number(EDITION_IN_NAME.exec(tail)?.[1] ?? 0); + return { dateMs, edition }; +} + +function newer(a, b) { + if (!b) return true; + const ad = a.dateMs ?? -Infinity; + const bd = b.dateMs ?? -Infinity; + if (ad !== bd) return ad > bd; + if (a.edition !== b.edition) return a.edition > b.edition; + return a.path > b.path; +} + +/** `S2Underground_Border_Crisis_Map_June_02_2026_v1.kmz` → readable words. */ +export function ghostMapsFileTitle(fileName) { + return String(fileName || '') + .replace(/\.km[lz]$/i, '') + .replace(/^S2Underground_/i, '') + .replace(/_/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Build the product catalog from a recursive git tree listing. + * @param {Array<{path: string, type: string, sha: string, size?: number}>} tree + * @returns {Array<{id: string, title: string, defaultOn: boolean, + * path: string, fileName: string, fileTitle: string, version: string, + * size: number, dateMs: ?number, edition: number}>} + */ +export function buildGhostMapsCatalog(tree) { + const best = new Map(); + let drop = null; + for (const entry of Array.isArray(tree) ? tree : []) { + if (entry?.type !== 'blob' || typeof entry.path !== 'string') continue; + if (!/\.kmz$/i.test(entry.path)) continue; + if (!/^[0-9a-f]{40}$/.test(entry.sha || '')) continue; + const size = Number(entry.size); + if (!Number.isFinite(size) || size <= 0 || size > GHOSTMAPS_MAX_FILE_BYTES) + continue; + const fileName = entry.path.slice(entry.path.lastIndexOf('/') + 1); + const candidate = { + path: entry.path, + fileName, + version: entry.sha, + size, + ...parseGhostMapsEdition(fileName), + }; + const product = GHOSTMAPS_PRODUCTS.find(({ pattern }) => + pattern.test(entry.path), + ); + if (product) { + if (newer(candidate, best.get(product.id))) + best.set(product.id, candidate); + } else if ( + entry.path.startsWith(INCIDENTS) && + !LATEST_DROP_EXCLUDED.test(`/${entry.path}`) && + candidate.dateMs != null && + newer(candidate, drop) + ) { + drop = candidate; + } + } + const rows = []; + for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { + const file = + product === GHOSTMAPS_LATEST_DROP ? drop : best.get(product.id); + if (!file) continue; + rows.push({ + id: product.id, + title: product.title, + defaultOn: product.defaultOn, + ...file, + fileTitle: ghostMapsFileTitle(file.fileName), + }); + } + return rows; +} + +/** + * Raw download URL pinned to a commit, so a CDN can never hand back an older + * edition for a newer catalog. + * @param {string} commit 40-hex commit sha. + * @param {string} path Repository path. + */ +export function ghostMapsRawUrl(commit, path) { + const encoded = path.split('/').map(encodeURIComponent).join('/'); + return `https://raw.githubusercontent.com/${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}/${commit}/${encoded}`; +} + +/** + * The branch head from a git smart-HTTP ref advertisement + * (`/info/refs?service=git-upload-pack`), or null. + * @param {string} text + * @param {string} [branch] + * @returns {?string} + */ +export function parseGitRefAdvertisement(text, branch = GHOSTMAPS_REPO.branch) { + const ref = `refs/heads/${branch}`; + for (const line of String(text || '').split('\n')) { + // pkt-line: 4 hex length digits, then " " (the first ref line + // also carries a NUL and the capability list). + const match = /^[0-9a-f]{4}([0-9a-f]{40}) ([^\0\s]+)/.exec(line); + if (match && match[2] === ref) return match[1]; + } + return null; +} diff --git a/src/layers/ghostMaps/catalog.test.mjs b/src/layers/ghostMaps/catalog.test.mjs new file mode 100644 index 000000000..74735381d --- /dev/null +++ b/src/layers/ghostMaps/catalog.test.mjs @@ -0,0 +1,165 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + buildGhostMapsCatalog, + ghostMapsFileTitle, + ghostMapsRawUrl, + parseGhostMapsEdition, + parseGitRefAdvertisement, +} from './catalog.js'; + +const ARCGIS = 'ArcGIS Data for ATAK (KMZs)/'; +const INCIDENTS = 'Incident Response Products for ATAK (KMZs and Documents)/'; +let n = 0; +const blob = (path, size = 1000) => ({ + path, + type: 'blob', + size, + sha: (++n).toString(16).padStart(40, '0'), +}); + +test('editions read the date and export/version number written in the name', () => { + assert.deepEqual( + parseGhostMapsEdition( + 'S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz', + ), + { dateMs: Date.UTC(2026, 9, 16), edition: 1 }, + ); + assert.deepEqual( + parseGhostMapsEdition( + 'S2Underground_Border_Crisis_Map_September_21_2026_Export_v2.kmz', + ), + { dateMs: Date.UTC(2026, 8, 21), edition: 2 }, + ); + assert.deepEqual( + parseGhostMapsEdition('Reno Wildfires August 24, 2026.kmz'), + { + dateMs: Date.UTC(2026, 7, 24), + edition: 0, + }, + ); + assert.deepEqual( + parseGhostMapsEdition('Venezuelan_Crisis_Sep_5_2025_v2.kmz'), + { + dateMs: Date.UTC(2025, 8, 5), + edition: 2, + }, + ); + assert.deepEqual( + parseGhostMapsEdition('Kinetic_Activities_Last_30_Days.kmz'), + { + dateMs: null, + edition: 0, + }, + ); + // An impossible date is not a date. + assert.equal(parseGhostMapsEdition('X_September_31_2026.kmz').dateMs, null); +}); + +test('each product keeps its newest edition by date, then edition number', () => { + const master = `${ARCGIS}Common Intelligence Picture/Master Database/`; + const tree = [ + blob( + `${master}S2Underground_Common_Intelligence_Picture_October_9_2026_Export_1.kmz`, + ), + blob( + `${master}S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz`, + ), + blob( + `${master}S2Underground_Common_Intelligence_Picture_October_1_2026_Export_1.kmz`, + ), + blob( + `${master}S2Underground_Common_Intelligence_Picture_September_30_2026_Export_2.kmz`, + ), + blob( + `${ARCGIS}Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz`, + ), + blob( + `${ARCGIS}Border Crisis Map/S2Underground_Border_Crisis_Map_September_21_2026_Export_v1.kmz`, + ), + blob( + `${ARCGIS}Border Crisis Map/S2Underground_Border_Crisis_Map_September_21_2026_Export_v2.kmz`, + ), + // A European-only subset in a subfolder is not the border map. + blob( + `${ARCGIS}Border Crisis Map/European Layers Only/Europe_Migration_Crisis_June_02_2026_v1.kmz`, + ), + blob(`${INCIDENTS}Yemen 2026/Yemen FLOT.kmz`), + blob(`${INCIDENTS}Yemen 2026/Yemen FLOT_October_10_2026.kmz`), + { path: `${INCIDENTS}Yemen 2026`, type: 'tree', sha: 'f'.repeat(40) }, + ]; + const rows = buildGhostMapsCatalog(tree); + const byId = Object.fromEntries(rows.map((row) => [row.id, row])); + assert.deepEqual( + rows.map(({ id }) => id), + ['cip', 'kinetic-30d', 'border-crisis', 'yemen-flot'], + ); + assert.match(byId.cip.fileName, /October_16_2026/); + assert.equal(byId.cip.defaultOn, true); + assert.equal(byId['kinetic-30d'].defaultOn, true); + assert.match(byId['border-crisis'].fileName, /Export_v2/); + assert.equal(byId['yemen-flot'].fileName, 'Yemen FLOT_October_10_2026.kmz'); + assert.equal(byId.cip.version, tree[1].sha); + assert.equal( + byId.cip.fileTitle, + 'Common Intelligence Picture October 16 2026 Export 1', + ); +}); + +test('latest drop is the newest dated incident file outside curated, archived and base data', () => { + const rows = buildGhostMapsCatalog([ + blob(`${INCIDENTS}Reno Wildfires August 24, 2026.kmz`), + blob( + `${INCIDENTS}Spokane Fire Disaster - August, 2026/SpokaneDisasterAid_August_8_2026.kmz`, + ), + blob( + `${INCIDENTS}0_Archive/Texas Floods/TexasFloods_December_1_2026_v1.kmz`, + ), + blob(`${INCIDENTS}Iran 2026/Basedata/Pipelines_December_1_2026.kmz`), + blob( + `${INCIDENTS}Iran 2026/Iran Kinetic Events/Iran_Kinetic_Events_December_2_2026_v1.kmz`, + ), + blob(`${INCIDENTS}Undated Product.kmz`), + ]); + const drop = rows.find(({ id }) => id === 'latest-drop'); + assert.equal(drop.fileName, 'Reno Wildfires August 24, 2026.kmz'); + assert.ok(rows.some(({ id }) => id === 'iran-kinetic')); +}); + +test('oversized, empty, non-KMZ and malformed entries are skipped', () => { + const path = `${ARCGIS}Common Intelligence Picture/Master Database/S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz`; + assert.deepEqual( + buildGhostMapsCatalog([ + { ...blob(path), size: 51 * 1024 * 1024 }, + { ...blob(path), size: 0 }, + { ...blob(path), sha: 'not-a-sha' }, + blob(path.replace(/\.kmz$/, '.zip')), + null, + ]), + [], + ); + assert.deepEqual(buildGhostMapsCatalog(undefined), []); +}); + +test('raw URLs are pinned to the commit and encode every path segment', () => { + assert.equal( + ghostMapsRawUrl('a'.repeat(40), 'Folder (KMZs)/Yemen FLOT.kmz'), + `https://raw.githubusercontent.com/s2underground/GhostMaps/${'a'.repeat(40)}/Folder%20(KMZs)/Yemen%20FLOT.kmz`, + ); + assert.equal(ghostMapsFileTitle('S2Underground_A__B.kmz'), 'A B'); +}); + +test('the branch head comes from the smart-HTTP ref advertisement', () => { + const head = '142d7630bc8f1f2c4b96e16ee865ca50d1ab578d'; + const other = 'b'.repeat(40); + const text = [ + '001e# service=git-upload-pack', + `0159${head} HEAD\0multi_ack symref=HEAD:refs/heads/main`, + `003f${other} refs/heads/main-old`, + `003d${head} refs/heads/main`, + '0000', + ].join('\n'); + assert.equal(parseGitRefAdvertisement(text), head); + assert.equal(parseGitRefAdvertisement(text, 'dev'), null); + assert.equal(parseGitRefAdvertisement(''), null); +}); diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js new file mode 100644 index 000000000..b0aa7c88b --- /dev/null +++ b/src/layers/ghostMaps/index.js @@ -0,0 +1,708 @@ +import * as Cesium from 'cesium'; +import { + decodeXml, + kmzImageResolver, + kmzMainDocument, + parseXml, + readZipEntries, + resolveKmzPath, +} from '../../data/geoFileImport.js'; +import { + formatNumber, + formatUtcDayMonth, + msg, + t, + tn, +} from '../../i18n/index.js'; +import { createHorizonCull } from '../streetLevel/view.js'; +import { readGhostMapsKml } from './kml.js'; +import { fieldLabel, parseDescriptionFields, wrapCardText } from './records.js'; +export * from './catalog.js'; +export * from './records.js'; +export { readGhostMapsKml } from './kml.js'; +export { createGhostMapsSource } from './source.js'; + +export const GHOSTMAPS_LAYER_ID = 's2-ghostmaps'; +export const GHOSTMAPS_OVERLAY_SOURCE_ID = 's2-ghostmaps'; +const PICK_PREFIX = 'ghostmaps:'; +const STORAGE_KEY = 'gev.ghostmaps.products'; +const ACCENT = '#c8ff00'; +const MAX_ICON_BYTES = 512 * 1024; +const CARD_HOST_OPTIONS = Object.freeze({ + cohortLimit: 1, + collisionCapacity: 1, + moving: false, +}); +const MAX_CARD_LINES = 12; + +/** Panel names for the catalog's product ids (the catalog sends English). */ +const PRODUCT_LABELS = Object.freeze({ + cip: msg('Intel Picture'), + 'kinetic-30d': msg('Kinetic 30d'), + 'border-crisis': msg('Border Crisis'), + 'iran-cip': msg('Iran Picture'), + 'iran-kinetic': msg('Iran Kinetic'), + 'yemen-flot': msg('Yemen FLOT'), + 'latest-drop': msg('Latest Drop'), +}); + +/** One distinct fallback marker color per product. */ +const PRODUCT_COLORS = Object.freeze({ + cip: '#c8ff00', + 'kinetic-30d': '#ff5a36', + 'border-crisis': '#ffb547', + 'iran-cip': '#39d0ff', + 'iran-kinetic': '#ff3b8d', + 'yemen-flot': '#5dff9f', + 'latest-drop': '#b48cff', +}); + +function defaultStorage() { + try { + return globalThis.localStorage ?? null; + } catch { + return null; + } +} + +function readStoredSelection(storage) { + try { + const value = JSON.parse(storage?.getItem(STORAGE_KEY) ?? 'null'); + return Array.isArray(value) ? new Set(value.map(String)) : null; + } catch { + return null; + } +} + +function writeStoredSelection(storage, selected) { + try { + storage?.setItem(STORAGE_KEY, JSON.stringify([...selected])); + } catch { + /* private window or blocked storage: the selection lasts this session */ + } +} + +/** A round marker for placemarks whose icon is missing or remote. */ +const markerCache = new Map(); +function markerImage(color) { + if (markerCache.has(color)) return markerCache.get(color); + let url = null; + try { + const canvas = document.createElement('canvas'); + canvas.width = canvas.height = 20; + const ctx = canvas.getContext('2d'); + ctx.beginPath(); + ctx.arc(10, 10, 7, 0, Math.PI * 2); + ctx.fillStyle = color; + ctx.fill(); + ctx.lineWidth = 2; + ctx.strokeStyle = 'rgba(0,0,0,0.85)'; + ctx.stroke(); + url = canvas.toDataURL(); + } catch { + url = null; + } + markerCache.set(color, url); + return url; +} + +const cesiumColor = (rgba, fallback, alpha) => + rgba + ? new Cesium.Color(rgba[0], rgba[1], rgba[2], alpha ?? rgba[3]) + : Cesium.Color.fromCssColorString(fallback).withAlpha(alpha ?? 1); + +function anchorOf(geometry) { + if (geometry.type === 'point') return geometry.position; + const ring = + geometry.type === 'polygon' ? geometry.outer : geometry.positions; + let lon = 0; + let lat = 0; + for (const [x, y] of ring) { + lon += x; + lat += y; + } + return [lon / ring.length, lat / ring.length]; +} + +/** + * S2 Underground GhostMaps: the current editions of S2's public KMZ intel + * products, kept in step with the GitHub repository. The proxy watches the + * branch head and pushes each new commit; only changed products reload. + */ +export function createGhostMapsLayer({ + source, + overlayHost = null, + screenSpaceEventHandlerFactory = null, + picking = null, + pointer = null, + openExternal = null, + storage = defaultStorage(), +} = {}) { + if (typeof source?.getCatalog !== 'function') + throw new TypeError('GhostMaps layer requires a catalog source'); + let _viewer = null; + let _enabled = false; + let _catalog = null; + let _lastError = null; + let _unsubscribe = null; + let _stream = 'closed'; + let _queue = Promise.resolve(); + let _clickHandler = null; + let _rowListener = null; + let _selectedKey = null; + let _selectedCardId = null; + let _selectedLink = null; + let _lifetime = new AbortController(); + let _selection = readStoredSelection(storage); + /** productId → {version, dataSource, urls, count, features} */ + const _loaded = new Map(); + /** productId → in-flight version */ + const _loading = new Map(); + /** productId → error text */ + const _failed = new Map(); + + const notifyRow = () => _rowListener?.(); + const render = () => _viewer?.scene?.requestRender?.(); + // Pins skip the depth test, and with Google 3D tiles the globe is hidden, + // so nothing else stops far-side pins drawing through the planet. + const _horizon = createHorizonCull({ + getViewer: () => _viewer, + items: function* () { + for (const item of _loaded.values()) yield* item.pins; + }, + onChange: render, + }); + const canSelect = () => + overlayHost && screenSpaceEventHandlerFactory && picking; + + function wanted(product) { + return _selection ? _selection.has(product.id) : product.defaultOn; + } + + function selectedIds() { + return (_catalog?.products || []).filter(wanted).map(({ id }) => id); + } + + function unload(id) { + const item = _loaded.get(id); + if (!item) return; + _loaded.delete(id); + _viewer?.dataSources?.remove(item.dataSource, true); + item.urls.forEach((url) => URL.revokeObjectURL(url)); + if (_selectedKey?.startsWith(`${id}:`)) clearSelection(); + } + + /** Build one product's data source off-scene, then swap it in. */ + async function load(product, signal) { + const bytes = await source.getFile(product.id, product.version, { signal }); + const entries = await readZipEntries(bytes); + const main = kmzMainDocument(entries); + if (!main) throw new Error(msg('That KMZ holds no KML document.')); + const urls = []; + const resolvePacked = kmzImageResolver(entries, main, (url) => + urls.push(url), + ); + // Icons only: ArcGIS exports also pack multi-megabyte photo attachments, + // which fall back to the product marker instead of being decoded. + const resolveImage = (href) => { + const path = resolveKmzPath(href, main, entries); + return path && entries.get(path).length <= MAX_ICON_BYTES + ? resolvePacked(href) + : null; + }; + let parsed; + try { + parsed = readGhostMapsKml( + parseXml(decodeXml(entries.get(main)), 'KML'), + resolveImage, + ); + } catch (error) { + urls.forEach((url) => URL.revokeObjectURL(url)); + throw error; + } + entries.clear(); + signal.throwIfAborted(); + const color = PRODUCT_COLORS[product.id] || ACCENT; + const dataSource = new Cesium.CustomDataSource( + `${GHOSTMAPS_LAYER_ID}:${product.id}`, + ); + const features = []; + const pins = []; + dataSource.entities.suspendEvents(); + parsed.features.forEach((feature, index) => { + const key = `${product.id}:${index}`; + const { style } = feature; + feature.geometries.forEach((geometry, part) => { + const id = `${PICK_PREFIX}${key}:${part}`; + if (geometry.type === 'point') { + const position = Cesium.Cartesian3.fromDegrees(...geometry.position); + const entity = dataSource.entities.add({ + id, + position, + billboard: { + image: style.icon || markerImage(color), + scale: Math.min( + 1.5, + Math.max(0.35, (style.iconScale ?? 1) * 0.7), + ), + color: + style.icon && style.iconColor + ? cesiumColor(style.iconColor) + : undefined, + heightReference: Cesium.HeightReference.CLAMP_TO_GROUND, + verticalOrigin: Cesium.VerticalOrigin.CENTER, + // Ground-clamped icons would sink into terrain and 3D tiles; the + // horizon cull hides the far side instead (see `_horizon`). + disableDepthTestDistance: Number.POSITIVE_INFINITY, + }, + }); + pins.push({ + position, + get show() { + return entity.show; + }, + set show(value) { + entity.show = value; + }, + }); + } else if (geometry.type === 'line') { + dataSource.entities.add({ + id, + polyline: { + positions: Cesium.Cartesian3.fromDegreesArray( + geometry.positions.flat(), + ), + width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), + material: cesiumColor(style.lineColor, color), + clampToGround: true, + }, + }); + } else { + const ring = (points) => + Cesium.Cartesian3.fromDegreesArray(points.flat()); + dataSource.entities.add({ + id, + polygon: { + hierarchy: new Cesium.PolygonHierarchy( + ring(geometry.outer), + geometry.holes.map( + (hole) => new Cesium.PolygonHierarchy(ring(hole)), + ), + ), + material: + style.fill === false + ? Cesium.Color.TRANSPARENT + : cesiumColor( + style.polyColor, + color, + style.polyColor ? undefined : 0.25, + ), + classificationType: Cesium.ClassificationType.BOTH, + }, + }); + if (style.outline !== false) + dataSource.entities.add({ + id: `${id}:outline`, + polyline: { + positions: ring([...geometry.outer, geometry.outer[0]]), + width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + material: cesiumColor(style.lineColor, color), + clampToGround: true, + }, + }); + } + }); + features.push({ + name: feature.name, + folder: feature.folder, + description: feature.description, + anchor: anchorOf(feature.geometries[0]), + }); + }); + dataSource.entities.resumeEvents(); + return { version: product.version, dataSource, urls, features, pins }; + } + + /** Bring the globe in line with the catalog and the chip selection. */ + function sync() { + const run = _queue.then(async () => { + if (!_enabled || !_viewer || !_catalog) return; + const signal = _lifetime.signal; + const products = new Map(_catalog.products.map((p) => [p.id, p])); + for (const id of [..._loaded.keys()]) { + const product = products.get(id); + if (!product || !wanted(product)) unload(id); + } + for (const product of _catalog.products) { + if (!wanted(product) || signal.aborted) continue; + if (_loaded.get(product.id)?.version === product.version) continue; + _loading.set(product.id, product.version); + _failed.delete(product.id); + notifyRow(); + try { + const next = await load(product, signal); + if (signal.aborted || !_enabled) { + next.urls.forEach((url) => URL.revokeObjectURL(url)); + return; + } + // Cull before the first frame so far-side pins never flash. + _horizon.update(next.pins); + await _viewer.dataSources.add(next.dataSource); + // The new edition is on the globe before the old one leaves. + unload(product.id); + _loaded.set(product.id, next); + } catch (error) { + if (signal.aborted) return; + console.warn(`[Data:GhostMaps] ${product.id}:`, error); + _failed.set( + product.id, + error?.template || + error?.message || + msg('GhostMaps file could not be read'), + ); + } finally { + _loading.delete(product.id); + notifyRow(); + render(); + } + } + }); + _queue = run.catch(() => {}); + return run; + } + + function applyCatalog(catalog) { + _catalog = catalog; + _lastError = catalog.error || null; + notifyRow(); + return sync(); + } + + function toggleProduct(id) { + // The first change pins the whole current selection, defaults included. + if (!_selection) _selection = new Set(selectedIds()); + if (_selection.has(id)) _selection.delete(id); + else _selection.add(id); + writeStoredSelection(storage, _selection); + notifyRow(); + void sync(); + } + + // ---- selection card ---------------------------------------------------- + function featureFor(key) { + const [productId, index] = key.split(':'); + return { + product: _catalog?.products.find((p) => p.id === productId) || null, + feature: _loaded.get(productId)?.features[Number(index)] || null, + }; + } + + function publishCard() { + if (!canSelect()) return; + const { product, feature } = _selectedKey ? featureFor(_selectedKey) : {}; + if (!feature) { + _selectedKey = null; + _selectedCardId = null; + _selectedLink = null; + overlayHost.setEntries( + GHOSTMAPS_OVERLAY_SOURCE_ID, + [], + CARD_HOST_OPTIONS, + ); + return; + } + const { fields, links } = parseDescriptionFields(feature.description); + _selectedLink = links[0] || null; + const heading = [t(PRODUCT_LABELS[product?.id] || product?.title || '')]; + if (feature.folder) heading.push(fieldLabel(feature.folder)); + if (product?.dateMs) heading.push(formatUtcDayMonth(product.dateMs)); + const details = [heading.filter(Boolean).join(' · ')]; + for (const { label, value } of fields) { + if (details.length >= MAX_CARD_LINES) break; + // ArcGIS repeats the feature type as the first row; the title has it. + if (value === feature.name) continue; + const line = label ? `${fieldLabel(label)}: ${value}` : value; + details.push( + ...wrapCardText(line, 60, Math.min(4, MAX_CARD_LINES - details.length)), + ); + } + if (_selectedLink && openExternal) + details.push( + tn( + links.length, + '{count} source link ↗ · click card to open', + '{count} source links ↗ · click card to open the first', + { count: formatNumber(links.length) }, + ), + ); + const title = t('S2 · {name}', { + name: fieldLabel(feature.name || feature.folder) || t('Report'), + }); + const card = { + id: `ghostmaps-card:${_selectedKey}`, + selected: true, + interactive: Boolean(_selectedLink && openExternal), + title, + details, + accent: PRODUCT_COLORS[product?.id] || ACCENT, + priority: Number.MAX_SAFE_INTEGER, + gapPx: 15, + verticalOnly: true, + placement: 'above', + // A click card comes and goes with the click, as CCTV cards do, rather + // than holding the label arbiter's 2.5 s minimum lifetime; and it hides + // when its marker turns behind the planet. + stateless: true, + horizonCull: true, + position: Cesium.Cartesian3.fromDegrees(...feature.anchor), + }; + if (card.interactive) { + const link = _selectedLink; + card.accessibilityLabel = t('Open the source for {title}', { title }); + card.activate = () => { + openExternal(link); + return true; + }; + } + _selectedCardId = card.id; + overlayHost.setEntries( + GHOSTMAPS_OVERLAY_SOURCE_ID, + [card], + CARD_HOST_OPTIONS, + ); + overlayHost.setVisible?.(GHOSTMAPS_OVERLAY_SOURCE_ID, true); + } + + function clearSelection() { + _selectedKey = null; + _selectedCardId = null; + _selectedLink = null; + // An empty publish, as the perimeter cards deselect: it releases the + // selected card's paint lane, which clearing the source alone left drawn. + overlayHost?.setEntries(GHOSTMAPS_OVERLAY_SOURCE_ID, [], CARD_HOST_OPTIONS); + } + + function pickedKey(picked) { + const pickId = picking.resolvePickId(picked); + if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX)) + return null; + const [productId, index] = pickId.slice(PICK_PREFIX.length).split(':'); + return _loaded.get(productId)?.features[Number(index)] + ? `${productId}:${index}` + : null; + } + + function installClickHandler() { + if (!canSelect() || _clickHandler || !_viewer) return; + picking.registerPickOwner?.( + GHOSTMAPS_LAYER_ID, + (pickedId) => + typeof pickedId === 'string' && pickedId.startsWith(PICK_PREFIX), + ); + _clickHandler = screenSpaceEventHandlerFactory(_viewer); + _clickHandler.setInputAction((click) => { + if (pointer && !pointer.isPointerFree()) return; + const cardHit = overlayHost.hitTest?.( + click.position?.x, + click.position?.y, + { sourceId: GHOSTMAPS_OVERLAY_SOURCE_ID }, + ); + if (cardHit && cardHit.entryId === _selectedCardId) { + if (_selectedLink && openExternal) openExternal(_selectedLink); + return; + } + const picked = _viewer.scene.pick(click.position); + const key = picked ? pickedKey(picked) : null; + if (key) { + _selectedKey = key; + publishCard(); + render(); + return; + } + if (picked) { + const pickId = picking.resolvePickId(picked); + if (pickId && picking.isOwnedByOtherLayer(GHOSTMAPS_LAYER_ID, pickId)) + return; + } + if (_selectedKey) { + clearSelection(); + render(); + } + }, Cesium.ScreenSpaceEventType.LEFT_CLICK); + } + + function removeClickHandler() { + _clickHandler?.destroy(); + _clickHandler = null; + picking?.unregisterPickOwner?.(GHOSTMAPS_LAYER_ID); + } + + function featureCount() { + let count = 0; + for (const item of _loaded.values()) count += item.features.length; + return count; + } + + const layer = { + id: GHOSTMAPS_LAYER_ID, + name: 'S2 GhostMaps', + icon: '👻', + source: 'S2 Underground', + // Pushes arrive within seconds over the event stream; this interval is + // the backstop if the stream is down. + updateInterval: 300000, + + init(viewer) { + if (_viewer) throw new Error('GhostMaps layer is already initialized'); + _viewer = viewer; + }, + + enable() { + _enabled = true; + _lifetime = new AbortController(); + for (const item of _loaded.values()) item.dataSource.show = true; + installClickHandler(); + if (!_unsubscribe && typeof source.subscribe === 'function') + _unsubscribe = source.subscribe( + (catalog) => { + if (_enabled) void applyCatalog(catalog); + }, + (state) => { + if (_stream === state) return; + _stream = state; + notifyRow(); + }, + ); + }, + + disable() { + _enabled = false; + _lifetime.abort(); + _unsubscribe?.(); + _unsubscribe = null; + _stream = 'closed'; + removeClickHandler(); + clearSelection(); + _horizon.stop(); + for (const id of [..._loaded.keys()]) unload(id); + render(); + }, + + async update(_viewerArg, { signal } = {}) { + if (!_enabled || signal?.aborted) return false; + try { + const catalog = await source.getCatalog({ signal }); + if (!_enabled) return false; + await applyCatalog(catalog); + return true; + } catch (error) { + if (signal?.aborted || !_enabled) return false; + console.warn('[Data:GhostMaps] catalog error:', error); + _lastError = error?.message || msg('GhostMaps catalog unavailable'); + notifyRow(); + // Keep the layer on; the event stream or the next interval recovers. + return true; + } + }, + + destroy() { + layer.disable(); + _viewer = null; + _catalog = null; + _rowListener = null; + }, + + setRowControlsListener(listener) { + _rowListener = listener; + }, + + getRowControls() { + const products = _catalog?.products || []; + const commit = _catalog?.commit; + const info = commit + ? _stream === 'open' + ? t('LIVE · GitHub {commit} · {message} · checked every {seconds}s', { + commit: commit.slice(0, 7), + message: _catalog.message || '—', + seconds: formatNumber( + Math.round((_catalog.pollMs || 15000) / 1000), + ), + }) + : t('GitHub {commit} · {message} · reconnecting for live updates', { + commit: commit.slice(0, 7), + message: _catalog.message || '—', + }) + : t('Connecting to the S2 GhostMaps repository…'); + return { + chips: products.map((product) => { + const loaded = _loaded.get(product.id); + const failed = _failed.get(product.id); + return { + id: product.id, + label: t(PRODUCT_LABELS[product.id] || product.title), + active: wanted(product), + busy: _loading.has(product.id), + state: failed ? 'error' : undefined, + color: PRODUCT_COLORS[product.id], + title: [ + product.fileTitle, + loaded + ? tn( + loaded.features.length, + '{count} placemark', + '{count} placemarks', + { + count: formatNumber(loaded.features.length), + }, + ) + : null, + failed ? t(failed) : null, + ] + .filter(Boolean) + .join(' · '), + onClick: () => toggleProduct(product.id), + }; + }), + legend: [], + info, + infoTitle: t( + 'S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.', + ), + }; + }, + + getStats() { + return { + count: featureCount(), + products: [..._loaded.keys()], + commit: _catalog?.commit || null, + lastUpdate: _catalog?.committedAt || _catalog?.changedAt || null, + checkedAt: _catalog?.checkedAt || null, + stream: _stream, + loading: _loading.size > 0, + error: _lastError || [..._failed.values()][0] || null, + fallback: false, + }; + }, + + /** Loaded placemarks as plain records, for tools and QA. */ + getAnalystRecords(maxCount = Infinity) { + if (!_enabled) return []; + const out = []; + for (const [productId, item] of _loaded) { + for (const feature of item.features) { + if (out.length >= maxCount) return out; + out.push({ + product: productId, + name: feature.name, + folder: feature.folder, + lon: feature.anchor[0], + lat: feature.anchor[1], + }); + } + } + return out; + }, + }; + return layer; +} diff --git a/src/layers/ghostMaps/kml.js b/src/layers/ghostMaps/kml.js new file mode 100644 index 000000000..92e0c532f --- /dev/null +++ b/src/layers/ghostMaps/kml.js @@ -0,0 +1,198 @@ +// Read the drawable content of a parsed GhostMaps KML document: placemarks +// with their folder, geometry, resolved style and raw description. Network +// links, overlays, models and remote icons are never followed; an icon is +// either an image packed in the same KMZ or the layer's own marker. +import { kmlColor } from './records.js'; + +export const MAX_GHOSTMAPS_FEATURES = 20_000; +export const MAX_GHOSTMAPS_POSITIONS = 250_000; + +const childrenNamed = (el, name) => + [...(el?.children || [])].filter((child) => child.localName === name); +const firstChild = (el, name) => + [...(el?.children || [])].find((child) => child.localName === name) || null; +const descendants = (el, name) => + [...(el?.getElementsByTagName?.('*') || [])].filter( + (child) => child.localName === name, + ); +const text = (el) => (el?.textContent ?? '').trim(); + +function number(el, fallback) { + const value = Number(text(el)); + return el && Number.isFinite(value) ? value : fallback; +} + +/** `lon,lat[,alt] lon,lat ...` → [[lon, lat], ...]; invalid tuples are dropped. */ +export function parseKmlCoordinates(value) { + const out = []; + for (const tuple of String(value ?? '') + .trim() + .split(/\s+/)) { + const [lon, lat] = tuple.split(',').map(Number); + if ( + Number.isFinite(lon) && + Number.isFinite(lat) && + Math.abs(lat) <= 90 && + Math.abs(lon) <= 180 + ) + out.push([lon, lat]); + } + return out; +} + +function readStyle(styleEl, resolveImage) { + const style = {}; + const icon = firstChild(styleEl, 'IconStyle'); + if (icon) { + const href = text(firstChild(firstChild(icon, 'Icon'), 'href')); + style.icon = href ? resolveImage(href) : null; + style.iconScale = number(firstChild(icon, 'scale'), 1); + style.iconColor = kmlColor(text(firstChild(icon, 'color'))); + } + const line = firstChild(styleEl, 'LineStyle'); + if (line) { + style.lineColor = kmlColor(text(firstChild(line, 'color'))); + style.lineWidth = number(firstChild(line, 'width'), 1); + } + const poly = firstChild(styleEl, 'PolyStyle'); + if (poly) { + style.polyColor = kmlColor(text(firstChild(poly, 'color'))); + style.fill = text(firstChild(poly, 'fill')) !== '0'; + style.outline = text(firstChild(poly, 'outline')) !== '0'; + } + return style; +} + +/** Shared styles by id; a StyleMap resolves to its `normal` pair. */ +function collectStyles(doc, resolveImage) { + const styles = new Map(); + for (const el of descendants(doc, 'Style')) { + const id = el.getAttribute('id'); + if (id) styles.set(id, readStyle(el, resolveImage)); + } + const maps = new Map(); + for (const el of descendants(doc, 'StyleMap')) { + const id = el.getAttribute('id'); + if (!id) continue; + const normal = childrenNamed(el, 'Pair').find( + (pair) => text(firstChild(pair, 'key')) === 'normal', + ); + if (!normal) continue; + const inline = firstChild(normal, 'Style'); + maps.set( + id, + inline + ? readStyle(inline, resolveImage) + : text(firstChild(normal, 'styleUrl')), + ); + } + const resolve = (url, depth = 0) => { + const id = String(url || '').replace(/^#/, ''); + if (!id || url.includes('/') || depth > 2) return {}; + if (styles.has(id)) return styles.get(id); + const mapped = maps.get(id); + if (typeof mapped === 'string') return resolve(mapped, depth + 1); + return mapped || {}; + }; + return resolve; +} + +function readGeometry(el, out, budget) { + switch (el.localName) { + case 'Point': { + const [position] = parseKmlCoordinates( + text(firstChild(el, 'coordinates')), + ); + if (position) { + budget.positions += 1; + out.push({ type: 'point', position }); + } + break; + } + case 'LineString': + case 'LinearRing': { + const positions = parseKmlCoordinates( + text(firstChild(el, 'coordinates')), + ); + if (positions.length >= 2) { + budget.positions += positions.length; + out.push({ type: 'line', positions }); + } + break; + } + case 'Polygon': { + const ring = (boundary) => + parseKmlCoordinates( + text(firstChild(firstChild(boundary, 'LinearRing'), 'coordinates')), + ); + const outer = ring(firstChild(el, 'outerBoundaryIs')); + if (outer.length >= 3) { + const holes = childrenNamed(el, 'innerBoundaryIs') + .map(ring) + .filter((hole) => hole.length >= 3); + budget.positions += + outer.length + holes.reduce((n, hole) => n + hole.length, 0); + out.push({ type: 'polygon', outer, holes }); + } + break; + } + case 'MultiGeometry': + for (const child of el.children) readGeometry(child, out, budget); + break; + default: + // Models, tracks and anything newer are not drawn. + break; + } +} + +/** Name of the nearest enclosing Folder (or Document), for grouping. */ +function folderName(el) { + for (let node = el.parentElement; node; node = node.parentElement) { + if (node.localName === 'Folder' || node.localName === 'Document') { + const name = text(firstChild(node, 'name')); + if (name) return name; + } + } + return ''; +} + +/** + * @param {Document} doc Parsed KML. + * @param {(href: string) => ?string} resolveImage URL for a KMZ-packed image. + * @returns {{features: Array<{name: string, folder: string, + * description: string, style: object, geometries: Array}>, + * positions: number, over: boolean}} + */ +export function readGhostMapsKml(doc, resolveImage = () => null) { + const styleFor = collectStyles(doc, resolveImage); + const features = []; + const budget = { positions: 0 }; + let over = false; + for (const placemark of descendants(doc, 'Placemark')) { + // A hidden placemark (visibility 0) is drawn hidden by Google Earth too. + if (text(firstChild(placemark, 'visibility')) === '0') continue; + const geometries = []; + for (const child of placemark.children) + readGeometry(child, geometries, budget); + if (!geometries.length) continue; + if ( + features.length >= MAX_GHOSTMAPS_FEATURES || + budget.positions > MAX_GHOSTMAPS_POSITIONS + ) { + over = true; + break; + } + const inline = firstChild(placemark, 'Style'); + features.push({ + name: text(firstChild(placemark, 'name')).slice(0, 200), + folder: folderName(placemark).slice(0, 120), + description: text(firstChild(placemark, 'description')), + style: { + ...styleFor(text(firstChild(placemark, 'styleUrl'))), + ...(inline ? readStyle(inline, resolveImage) : {}), + }, + geometries, + }); + } + return { features, positions: budget.positions, over }; +} diff --git a/src/layers/ghostMaps/records.js b/src/layers/ghostMaps/records.js new file mode 100644 index 000000000..7c4eb58bf --- /dev/null +++ b/src/layers/ghostMaps/records.js @@ -0,0 +1,165 @@ +// Pure helpers for GhostMaps placemarks: description tables to plain fields, +// KML colors, and card text wrapping. Strings in, data out — description +// markup is only ever read as text, never inserted into the page. + +const ENTITIES = Object.freeze({ + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", + nbsp: ' ', +}); + +/** Decode the HTML character references ArcGIS and Google Earth emit. */ +export function decodeHtmlEntities(text) { + return String(text ?? '').replace( + /&(#x[0-9a-f]{1,6}|#\d{1,7}|[a-z]{2,8});/gi, + (whole, ref) => { + if (ref[0] === '#') { + const code = + ref[1] === 'x' || ref[1] === 'X' + ? parseInt(ref.slice(2), 16) + : parseInt(ref.slice(1), 10); + return code > 0 && + code <= 0x10ffff && + !(code >= 0xd800 && code <= 0xdfff) + ? String.fromCodePoint(code) + : ''; + } + return ENTITIES[ref.toLowerCase()] ?? whole; + }, + ); +} + +/** Markup to one line of plain text. */ +export function htmlToText(html) { + return decodeHtmlEntities( + String(html ?? '') + .replace(/<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]*>/g, ' '), + ) + .replace(/\s+/g, ' ') + .trim(); +} + +const EMPTY_VALUES = new Set(['', '', 'null', 'n/a', '-', '']); +// ArcGIS writes geometry type and internal ids into every popup table. +const SKIPPED_LABELS = new Set(['shape', 'objectid', 'fid', 'globalid']); +const MAX_FIELDS = 24; +const MAX_VALUE = 600; +const MAX_LINKS = 6; + +/** + * Fields and links from a placemark description. ArcGIS "popup" exports are a + * two-column table (label, value); anything else becomes one unlabelled field. + * @param {string} html Description as written in the KML (CDATA text). + * @returns {{fields: Array<{label: string, value: string}>, links: string[]}} + */ +export function parseDescriptionFields(html) { + const source = String(html ?? '').replace( + /<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi, + ' ', + ); + const fields = []; + const links = []; + const addLinks = (markup) => { + for (const match of markup.matchAll(/href\s*=\s*["']([^"']+)["']/gi)) { + const url = decodeHtmlEntities(match[1]).trim(); + if (/^https?:\/\/[^\s]+$/i.test(url) && !links.includes(url)) + links.push(url); + } + }; + // Innermost rows only: ArcGIS nests the field table inside an outer row, + // and only leaf rows with exactly two cells are label/value pairs. + for (const row of source.matchAll( + /]*>((?:(?!/gi, + )) { + const cells = [ + ...row[1].matchAll(/]*>([\s\S]*?)<\/t[dh]\s*>/gi), + ].map((cell) => cell[1]); + if (cells.length !== 2) continue; + const label = htmlToText(cells[0]); + const value = htmlToText(cells[1]); + if (!label || SKIPPED_LABELS.has(label.toLowerCase())) continue; + if (EMPTY_VALUES.has(value.toLowerCase())) continue; + addLinks(cells[1]); + // A bare research link is carried by `links`; the card does not need + // the URL twice. + if (/^https?:\/\/\S+$/i.test(value)) continue; + if (fields.length < MAX_FIELDS) + fields.push({ label, value: value.slice(0, MAX_VALUE) }); + } + if (!fields.length && !links.length) { + const text = htmlToText(source); + if (text) fields.push({ label: '', value: text.slice(0, MAX_VALUE) }); + addLinks(source); + } + return { fields, links: links.slice(0, MAX_LINKS) }; +} + +/** + * KML `aabbggrr` to `[r, g, b, a]` in 0..1, or null. + * @param {string} text + * @returns {?number[]} + */ +export function kmlColor(text) { + const hex = String(text ?? '') + .trim() + .replace(/^#/, ''); + if (!/^[0-9a-f]{8}$/i.test(hex)) return null; + const byte = (at) => parseInt(hex.slice(at, at + 2), 16) / 255; + return [byte(6), byte(4), byte(2), byte(0)]; +} + +/** + * Greedy word wrap for canvas card lines; overlong words are cut. + * @param {string} text + * @param {number} width Characters per line. + * @param {number} maxLines Lines kept; the last ends with an ellipsis when cut. + * @returns {string[]} + */ +export function wrapCardText(text, width = 56, maxLines = 4) { + const lines = []; + let line = ''; + let cut = false; + for (let word of String(text ?? '') + .split(/\s+/) + .filter(Boolean)) { + while (word.length > width) { + if (line) { + lines.push(line); + line = ''; + } + lines.push(word.slice(0, width)); + word = word.slice(width); + } + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + if (lines.length >= maxLines) { + cut = true; + break; + } + } + if (line && lines.length < maxLines) lines.push(line); + else if (line) cut = true; + const kept = lines.slice(0, maxLines); + if (cut && kept.length) { + const last = kept[kept.length - 1]; + kept[kept.length - 1] = `${last.slice(0, width - 1)}…`; + } + return kept; +} + +/** Field labels ArcGIS writes CamelCase; show them as words. */ +export function fieldLabel(label) { + return String(label ?? '') + .replace(/([a-z])([A-Z0-9])/g, '$1 $2') + .replace(/_/g, ' ') + .trim(); +} diff --git a/src/layers/ghostMaps/records.test.mjs b/src/layers/ghostMaps/records.test.mjs new file mode 100644 index 000000000..86fecb52b --- /dev/null +++ b/src/layers/ghostMaps/records.test.mjs @@ -0,0 +1,106 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + decodeHtmlEntities, + fieldLabel, + htmlToText, + kmlColor, + parseDescriptionFields, + wrapCardText, +} from './records.js'; + +// The shape ArcGIS Pro writes into every S2 placemark description. +const ARCGIS = ` + + + +
HistoricProtestSite
+ + + + + + + + +
SHAPEPoint
InstallationTypeHistoricProtestSite
OperationalStatusACTIVE
Affiliation<Null>
DescriptionLocals gather here & protest — weekly.
Research1https://x.com/a/status/1
Research2bad
Research3<Null>
`; + +test('ArcGIS popup tables become plain label/value fields and http links', () => { + const { fields, links } = parseDescriptionFields(ARCGIS); + assert.deepEqual(fields, [ + { label: 'InstallationType', value: 'HistoricProtestSite' }, + { label: 'OperationalStatus', value: 'ACTIVE' }, + { label: 'Description', value: 'Locals gather here & protest — weekly.' }, + { label: 'Research2', value: 'bad' }, + ]); + // javascript: and other schemes never become links. + assert.deepEqual(links, ['https://x.com/a/status/1']); +}); + +test('free-form descriptions become one unlabelled field; scripts are dropped', () => { + assert.deepEqual( + parseDescriptionFields( + '

Front line
as of 10 Oct

more', + ), + { + fields: [{ label: '', value: 'Front line as of 10 Oct more' }], + links: ['https://s2.example/x'], + }, + ); + assert.deepEqual(parseDescriptionFields(''), { fields: [], links: [] }); + assert.deepEqual(parseDescriptionFields(undefined), { + fields: [], + links: [], + }); +}); + +test('field values and counts are bounded', () => { + const rows = Array.from( + { length: 40 }, + (_, i) => `K${i}${'x'.repeat(900)}`, + ).join(''); + const { fields } = parseDescriptionFields(`${rows}
`); + assert.equal(fields.length, 24); + assert.equal(fields[0].value.length, 600); +}); + +test('entities and tags decode to text', () => { + assert.equal( + decodeHtmlEntities('<b> AB  &bogus;'), + ' AB &bogus;', + ); + assert.equal(decodeHtmlEntities('��'), ''); + assert.equal(htmlToText('One\n two'), 'One two'); +}); + +test('KML aabbggrr colors convert to rgba', () => { + assert.deepEqual(kmlColor('ff0000ff'), [1, 0, 0, 1]); + assert.deepEqual(kmlColor('8000ff00'), [0, 1, 0, 128 / 255]); + assert.equal(kmlColor('red'), null); + assert.equal(kmlColor(''), null); +}); + +test('card text wraps on words, cuts long words and marks truncation', () => { + assert.deepEqual(wrapCardText('one two three four', 9, 4), [ + 'one two', + 'three', + 'four', + ]); + assert.deepEqual(wrapCardText('abcdefghijkl', 5, 4), [ + 'abcde', + 'fghij', + 'kl', + ]); + assert.deepEqual(wrapCardText('aa bb cc dd ee', 5, 2), ['aa bb', 'cc d…']); + assert.deepEqual(wrapCardText('', 5, 2), []); + for (const line of wrapCardText('word '.repeat(200), 20, 3)) + assert.ok(line.length <= 20); +}); + +test('CamelCase field labels read as words', () => { + assert.equal(fieldLabel('OperationalStatus'), 'Operational Status'); + assert.equal(fieldLabel('Research1'), 'Research 1'); + assert.equal(fieldLabel('Kinetic_Activity'), 'Kinetic Activity'); +}); diff --git a/src/layers/ghostMaps/source.js b/src/layers/ghostMaps/source.js new file mode 100644 index 000000000..c52ebda64 --- /dev/null +++ b/src/layers/ghostMaps/source.js @@ -0,0 +1,81 @@ +import { + readResponseBytesCapped, + readResponseJsonCapped, +} from '../../sources/httpBody.js'; +import { GHOSTMAPS_MAX_FILE_BYTES } from './catalog.js'; + +// A portable export (package.json): no i18n here. Failures stay English data; +// the Layers panel shows them translated. +const msg = (text) => text; + +/** + * Same-origin GhostMaps source: the catalog, one product's KMZ bytes, and a + * push subscription that fires whenever the watcher sees a new commit. + */ +export function createGhostMapsSource({ + fetchImpl = (...args) => globalThis.fetch(...args), + EventSourceImpl = globalThis.EventSource, +} = {}) { + return { + async getCatalog({ signal } = {}) { + const response = await fetchImpl('/api/ghostmaps/catalog', { signal }); + if (!response.ok) + throw new Error( + `GhostMaps catalog unavailable (HTTP ${response.status})`, + ); + const catalog = await readResponseJsonCapped( + response, + 1024 * 1024, + signal, + ); + if (!Array.isArray(catalog?.products)) + throw new Error(msg('Malformed GhostMaps catalog')); + return catalog; + }, + + async getFile(id, version, { signal } = {}) { + const response = await fetchImpl( + `/api/ghostmaps/file/${encodeURIComponent(id)}?v=${encodeURIComponent(version)}`, + { signal }, + ); + if (!response.ok) + throw new Error(`GhostMaps file unavailable (HTTP ${response.status})`); + return readResponseBytesCapped( + response, + GHOSTMAPS_MAX_FILE_BYTES, + signal, + ); + }, + + /** + * Call `onCatalog(catalog)` for every catalog the server pushes, and + * `onState('open' | 'connecting' | 'closed')` as the stream changes. The + * browser's EventSource reconnects by itself. Returns an unsubscribe. + */ + subscribe(onCatalog, onState = () => {}) { + if (typeof EventSourceImpl !== 'function') { + onState('closed'); + return () => {}; + } + const events = new EventSourceImpl('/api/ghostmaps/events'); + const states = ['connecting', 'open', 'closed']; + const report = () => onState(states[events.readyState] ?? 'connecting'); + events.onopen = report; + events.onerror = report; + const listener = (event) => { + try { + const catalog = JSON.parse(event.data); + if (Array.isArray(catalog?.products)) onCatalog(catalog); + } catch { + /* ignore a malformed frame; the next one replaces it */ + } + }; + events.addEventListener('catalog', listener); + report(); + return () => { + events.removeEventListener('catalog', listener); + events.close(); + }; + }, + }; +} diff --git a/src/tooling/previewServing.test.mjs b/src/tooling/previewServing.test.mjs index 04c24b92d..a049b40e5 100644 --- a/src/tooling/previewServing.test.mjs +++ b/src/tooling/previewServing.test.mjs @@ -11,6 +11,7 @@ test('data providers have both hooks; credential editing stays development-only' const plugins = localProviderPlugins(); assert.ok(plugins.some(({ name }) => name === 'fire-perimeters')); assert.ok(plugins.some(({ name }) => name === 'gnss-integrity')); + assert.ok(plugins.some(({ name }) => name === 'ghostmaps')); for (const plugin of plugins) { if (plugin.name === 'gev-key-setup') { assert.equal(plugin.configurePreviewServer, undefined); @@ -105,6 +106,7 @@ test('real dev and built-preview servers serve provider JSON and terminate unkno ['/api/firms/status', 200], ['/api/fire-perimeters/inciweb/publication/invalid', 400], ['/api/gnss-integrity?lat=invalid', 400], + ['/api/ghostmaps/unknown', 404], ['/api/terrain/heights?points=invalid', 400], ['/api/overpass', 405], ['/api/overpass/status', 200], diff --git a/src/ui/layerNames.js b/src/ui/layerNames.js index d9674efea..2c918ad95 100644 --- a/src/ui/layerNames.js +++ b/src/ui/layerNames.js @@ -34,6 +34,7 @@ export const LAYER_NAME_KEYS = Object.freeze({ 'recent-imagery': msg('Recent Imagery'), 'fire-perimeters': msg('Fire Perimeters'), 'gnss-integrity': msg('GNSS Integrity'), + 's2-ghostmaps': msg('S2 GhostMaps'), wind: msg('Wind'), 'weather-radar': msg('Rain radar'), 'weather-satellite': msg('Satellite clouds'), diff --git a/src/ui/layerPanel.js b/src/ui/layerPanel.js index 109d96c60..18b71dabf 100644 --- a/src/ui/layerPanel.js +++ b/src/ui/layerPanel.js @@ -73,6 +73,7 @@ const PANEL_GROUPS = [ 'local-firms', 'fire-perimeters', 'gnss-integrity', + 's2-ghostmaps', ], }, { diff --git a/src/voice/actionSchemas.test.mjs b/src/voice/actionSchemas.test.mjs index 550d725e4..b295e6f1b 100644 --- a/src/voice/actionSchemas.test.mjs +++ b/src/voice/actionSchemas.test.mjs @@ -38,7 +38,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea // requested radius and center authority in descriptions only. // Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts. // Air quality (ECCC AQHI) extends the manifest. - '775ba323384b4c1b62fb8415bac52aecc3fc668e20bedcfa57e426dcd1da3969', + // S2 GhostMaps extends the manifest. + '64a7ae9719cbf53f2c5b57739c1e84e8b60dc4ede040decd48a9c06d1ccbcbca', ); }); diff --git a/src/voice/layerManifest.js b/src/voice/layerManifest.js index eb0f09909..14b515fa6 100644 --- a/src/voice/layerManifest.js +++ b/src/voice/layerManifest.js @@ -57,6 +57,20 @@ const layers = [ context: false, noQuery: 'navigation-integrity cells; no countable analyst records', }, + { + id: 's2-ghostmaps', + aliases: [ + 's2 ghostmaps', + 'ghost maps', + 'ghostmaps', + 's2 underground', + 's2 intel', + 'intel picture', + 'common intelligence picture', + ], + context: false, + noQuery: 'published third-party intel placemarks; not analyst records', + }, { id: 'flights', track: true,