feat: add S2 GhostMaps live intel layer

Adds an "S2 GhostMaps" layer (Events group) that renders S2 Underground's
GhostMaps KMZ exports (Common Intelligence Picture, Kinetic 30d, Border
Crisis, Iran CIP/Kinetic, Yemen FLOT, latest incident drop).

- server/providers/ghostMaps.js: watcher polls the repo's git smart-HTTP
  ref advertisement (default every 15s, GHOSTMAPS_POLL_SECONDS), fetches
  the tree only on a new head, caches commit-pinned files by blob sha and
  pushes catalog updates to browsers over SSE (/api/ghostmaps/events).
- src/layers/ghostMaps: KML reader, ArcGIS popup-table parsing, per-product
  data sources, product chips, horizon-culled pins, report cards.
- Unit tests, QA gate (scripts/qa-ghostmaps.mjs), credits (CC BY-NC-SA 4.0),
  i18n, voice manifest, layer-state token 5.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
GPTmadeitandClaude Opus 5.5 committed 2026-10-20 16:24:53 -04:00
1 parent 713285b554
commit 1f77d2290c
36 files changed
+3247 -52

No files matched your search

+2
View File
@@ -27,6 +27,7 @@ import { createApplicationEarthquakes } from './layers/earthquakes.js';
import { createApplicationAirQuality } from './layers/airQuality.js';
import { createApplicationFirePerimeters } from './layers/perimeters.js';
import { createApplicationGnssIntegrity } from './layers/gnss.js';
import { createApplicationGhostMaps } from './layers/ghostMaps.js';
import { createApplicationCables } from './layers/submarineCables.js';
import { createApplicationStreetLevel } from './layers/streetLevel.js';
import { createInfrastructureLayers } from '../data/infrastructure.js';
@@ -126,6 +127,7 @@ export function createApplicationCatalog({
createApplicationGnssIntegrity({
source: sources['gnss-integrity'],
}),
createApplicationGhostMaps(),
createApplicationAlpr({ surface, source: sources.alpr }),
createApplicationStreetLevel({
surface,
+3 -2
View File
@@ -40,7 +40,7 @@ test('catalogs construct distinct layers and classification from their supplied
signal: b.signal,
surface: fixtureSurface(b.signal),
});
assert.equal(first.layers.length, 32);
assert.equal(first.layers.length, 33);
assert.ok(first.get('local-adsb'), 'Local ADS-B is registered');
assert.deepEqual(
first.metadata.find(({ id }) => id === 'local-adsb'),
@@ -61,6 +61,7 @@ test('catalogs construct distinct layers and classification from their supplied
);
assert.ok(first.get('fire-perimeters'));
assert.ok(first.get('gnss-integrity'));
assert.ok(first.get('s2-ghostmaps'));
assert.ok(first.get('transit'));
const order = first.layers.map(({ id }) => id);
assert.deepEqual(
@@ -154,7 +155,7 @@ test('Mapillary can be omitted or replaced without changing catalog membership',
surface: fixtureSurface(lifetime.signal),
};
const catalog = createApplicationCatalog(options);
assert.equal(catalog.layers.length, 32);
assert.equal(catalog.layers.length, 33);
assert.deepEqual(catalog.get('street-level').providerIds, []);
assert.equal(
catalogControlServices(catalog).streetLevelLayer,
+27
View File
@@ -0,0 +1,27 @@
import * as Cesium from 'cesium';
import {
createGhostMapsLayer,
createGhostMapsSource,
} from '../../layers/ghostMaps/index.js';
import * as picking from '../../data/pickRegistry.js';
import * as overlays from '../../overlays/worldOverlay.js';
import { isPointerFree } from '../../data/inputOwnership.js';
/** Wire the S2 Underground GhostMaps layer into the application catalog. */
export function createApplicationGhostMaps(options) {
return createGhostMapsLayer({
source: createGhostMapsSource(),
overlayHost: {
setEntries: overlays.setOverlayEntries,
setVisible: overlays.setOverlaySourceVisible,
clearSource: overlays.clearOverlaySource,
hitTest: overlays.hitTestWorldOverlay,
},
openExternal: (url) => window.open(url, '_blank', 'noopener,noreferrer'),
screenSpaceEventHandlerFactory: (viewer) =>
new Cesium.ScreenSpaceEventHandler(viewer.scene.canvas),
picking,
pointer: { isPointerFree },
...options,
});
}
+1 -1
View File
@@ -37,7 +37,7 @@ test('every acquisition source can be omitted without losing catalog membership'
test('all acquisition sources may be absent, and enable fails before initializing or fetching', async (t) => {
const actual = catalog({}, t);
assert.equal(actual.layers.length, 32);
assert.equal(actual.layers.length, 33);
const manager = new LayerLifecycle(
{},
{ getSourceAvailability: actual.getSourceAvailability },
+8
View File
@@ -441,6 +441,14 @@ export const DATA_CREDITS = [
'(<a href="https://earthdata.nasa.gov/firms" target="_blank" rel="noopener">earthdata.nasa.gov/firms</a>), ' +
'part of NASA’s Earth Observing System Data and Information System (EOSDIS)',
},
{
key: 's2-ghostmaps',
html:
'S2 GhostMaps intel products: S2 Underground © by S2A1, ' +
'<a href="https://github.com/s2underground/GhostMaps" target="_blank" rel="noopener">github.com/s2underground/GhostMaps</a>, ' +
'licensed under <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" rel="noopener">CC BY-NC-SA 4.0</a>. ' +
'Shown as published; reports are S2 assessments, not verified fact.',
},
{
key: 'drivebc-cctv',
html:
+9 -2
View File
@@ -1040,7 +1040,13 @@ export function detachImportedTime(dataSource) {
return dataSource;
}
function parseXml(text, what) {
/**
* Parse KML or GPX text with the namespace repairs Cesium applies.
* @param {string} text
* @param {string} what Format name for the error message.
* @returns {Document}
*/
export function parseXml(text, what) {
// Match the namespace repairs Cesium applies to ordinary exported KML.
if (what !== 'GPX')
text = String(text).replace(/<kml\b[^>]*>/i, (root) => {
@@ -1073,7 +1079,8 @@ function parseXml(text, what) {
return doc;
}
function decodeXml(bytes) {
/** Decode XML bytes, honouring a UTF-16 byte-order mark. */
export function decodeXml(bytes) {
const encoding =
bytes[0] === 0xff && bytes[1] === 0xfe
? 'utf-16le'
+5
View File
@@ -675,6 +675,11 @@ export const LAYER_STATE_REGISTRY = Object.freeze([
token: 'x',
disposition: 'enabled-only',
}),
Object.freeze({
id: 's2-ghostmaps',
token: '5',
disposition: 'enabled-only',
}),
Object.freeze({
id: 'satellites',
token: 's',
+13 -13
View File
@@ -202,8 +202,8 @@ function encode(state) {
test('production registry is exact, canonical, and rejects incomplete contracts', async () => {
assert.equal(validateLayerStateRegistry(), true);
assert.equal(REGISTERED_LAYER_IDS.length, 31);
assert.equal(new Set(REGISTERED_LAYER_IDS).size, 31);
assert.equal(REGISTERED_LAYER_IDS.length, 32);
assert.equal(new Set(REGISTERED_LAYER_IDS).size, 32);
assert.ok(REGISTERED_LAYER_IDS.includes('transit'));
assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort());
assert.deepEqual(LEGACY_LAYER_STATE_TOKENS, {
@@ -242,19 +242,19 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
for (const [id, token] of Object.entries(LEGACY_LAYER_STATE_TOKENS)) {
assert.equal(LAYER_STATE_TOKEN_RESERVATIONS[id], token);
}
assert.equal(nextLayerStateToken(), '5');
assert.equal(nextLayerStateToken(), '6');
assert.equal(
nextLayerStateToken({
...LAYER_STATE_TOKEN_RESERVATIONS,
alpha: '5',
bravo: '6',
alpha: '6',
bravo: '7',
}),
'7',
'8',
);
const digitsExhausted = {
...LAYER_STATE_TOKEN_RESERVATIONS,
...Object.fromEntries(
[...'56789'].map((digit) => [`prior-${digit}`, digit]),
[...'6789'].map((digit) => [`prior-${digit}`, digit]),
),
};
assert.equal(nextLayerStateToken(digitsExhausted), '00');
@@ -307,8 +307,8 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
assert.equal(
validateLayerStateAllocations(LAYER_STATE_TOKEN_RESERVATIONS, {
...LAYER_STATE_TOKEN_RESERVATIONS,
future: '5',
next: '6',
future: '6',
next: '7',
}),
true,
);
@@ -318,7 +318,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
...LAYER_STATE_TOKEN_RESERVATIONS,
future: '00',
}),
/next free token 5/,
/next free token 6/,
);
const beforeLastDigit = { ...digitsExhausted };
delete beforeLastDigit['prior-9'];
@@ -341,10 +341,10 @@ test('production registry is exact, canonical, and rejects incomplete contracts'
assert.throws(
() =>
validateLayerStateAllocations(
{ ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '5' },
{ ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '5', competing: '5' },
{ ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '6' },
{ ...LAYER_STATE_TOKEN_RESERVATIONS, merged: '6', competing: '6' },
),
/next free token 6/,
/next free token 7/,
);
assert.throws(
() => validateLayerStateAllocations({ future: '00' }, { future: '01' }),
+29 -28
View File
@@ -128,7 +128,7 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
{ ...parseLayerStateTokenReservations(reservationRows) },
{ ...LAYER_STATE_TOKEN_RESERVATIONS },
);
assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 31);
assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 32);
assert.deepEqual(
{ ...LAYER_STATE_TOKEN_RESERVATIONS },
{
@@ -136,6 +136,7 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
'street-level': '0',
'gnss-integrity': '3',
'air-quality': '4',
's2-ghostmaps': '5',
},
);
assert.throws(
@@ -161,9 +162,9 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
/Duplicate layer-state token reservation/,
);
assert.equal(
parseLayerStateTokenReservations([...reservationRows, ['future', '5']])
parseLayerStateTokenReservations([...reservationRows, ['future', '6']])
.future,
'5',
'6',
);
assert.throws(
() =>
@@ -181,14 +182,14 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed
test('published retired digits and competing PRs advance the merge-time allocation', () => {
withPublishedBase(
{ rows: [...reservationRows, ['retired-layer', '5']] },
{ rows: [...reservationRows, ['retired-layer', '6']] },
(cwd) => {
const published = readPublishedLayerStateReservations('HEAD', cwd);
assert.equal(nextLayerStateToken(published), '6');
assert.equal(nextLayerStateToken(published), '7');
assert.equal(
validateLayerStateAllocations(published, {
...published,
newcomer: '6',
newcomer: '7',
}),
true,
);
@@ -196,15 +197,15 @@ test('published retired digits and competing PRs advance the merge-time allocati
() =>
validateLayerStateAllocations(published, {
...published,
competing: '5',
competing: '6',
}),
/next free token 6/,
/next free token 7/,
);
assert.throws(
() =>
validateLayerStateAllocations(published, {
...LAYER_STATE_TOKEN_RESERVATIONS,
newcomer: '5',
newcomer: '6',
}),
/changed or removed: retired-layer/,
);
@@ -212,8 +213,8 @@ test('published retired digits and competing PRs advance the merge-time allocati
);
});
test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () => {
withPublishedBase({ rows: [...reservationRows, ['pr-a', '5']] }, (cwd) => {
test('PR B manually replaces provisional 6 with 7 after PR A publishes 6', () => {
withPublishedBase({ rows: [...reservationRows, ['pr-a', '6']] }, (cwd) => {
mkdirSync(path.join(cwd, 'scripts'));
const fixtureChecker = path.join(
cwd,
@@ -234,8 +235,8 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () =>
writeCodecFixture(
cwd,
[...reservationRows, ['pr-b', '5']],
[{ id: 'pr-b', token: '5' }],
[...reservationRows, ['pr-b', '6']],
[{ id: 'pr-b', token: '6' }],
);
const beforeRebase = runCheck();
assert.equal(beforeRebase.status, 1);
@@ -243,10 +244,10 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () =>
writeCodecFixture(
cwd,
[...reservationRows, ['pr-a', '5'], ['pr-b', '5']],
[...reservationRows, ['pr-a', '6'], ['pr-b', '6']],
[
{ id: 'pr-a', token: '5' },
{ id: 'pr-b', token: '5' },
{ id: 'pr-a', token: '6' },
{ id: 'pr-b', token: '6' },
],
);
const staleRebase = runCheck();
@@ -254,16 +255,16 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () =>
assert.match(staleRebase.stderr, /Duplicate layer-state token reservation/);
const published = readPublishedLayerStateReservations('HEAD', cwd);
assert.equal(nextLayerStateToken(published), '6');
const validBaselineRows = [...reservationRows, ['pr-a', '5']];
writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '5' }]);
assert.equal(nextLayerStateToken(published), '7');
const validBaselineRows = [...reservationRows, ['pr-a', '6']];
writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '6' }]);
const helper = spawnSync(
process.execPath,
[realpathSync(fixtureNext), 'pr-b'],
{ cwd, encoding: 'utf8' },
);
assert.equal(helper.status, 0, helper.stderr);
assert.match(helper.stdout, /pr-b: 6/);
assert.match(helper.stdout, /pr-b: 7/);
assert.deepEqual(
JSON.parse(readFileSync(path.join(cwd, LEDGER_PATH), 'utf8')),
validBaselineRows,
@@ -271,20 +272,20 @@ test('PR B manually replaces provisional 5 with 6 after PR A publishes 5', () =>
);
writeCodecFixture(
cwd,
[...reservationRows, ['pr-a', '5'], ['pr-b', '6']],
[...reservationRows, ['pr-a', '6'], ['pr-b', '7']],
[
{ id: 'pr-a', token: '5' },
{ id: 'pr-b', token: '6' },
{ id: 'pr-a', token: '6' },
{ id: 'pr-b', token: '7' },
],
);
const corrected = runCheck();
assert.equal(corrected.status, 0, corrected.stderr);
assert.match(corrected.stdout, /32 published, 1 new/);
assert.match(corrected.stdout, /33 published, 1 new/);
});
});
test('allocation batches cross the last digit and base-36 pair boundaries in order', () => {
const digitRows = [...'56789'].map((digit) => [`retired-${digit}`, digit]);
const digitRows = [...'6789'].map((digit) => [`retired-${digit}`, digit]);
const allDigits = parseLayerStateTokenReservations([
...reservationRows,
...digitRows,
@@ -336,7 +337,7 @@ test('allocation batches cross the last digit and base-36 pair boundaries in ord
test('an isolated valid two-character fixture round-trips an l field beyond the old 64-character cap', async () => {
const cwd = mkdtempSync(path.join(tmpdir(), 'gev-layer-token-width-'));
try {
const priorDigits = [...'56789'].map((digit) => [
const priorDigits = [...'6789'].map((digit) => [
`qa-prior-digit-${digit}`,
digit,
]);
@@ -373,7 +374,7 @@ test('an isolated valid two-character fixture round-trips an l field beyond the
const params = new URLSearchParams([['v', '2']]);
codec.encodeLayerStateParams(params, state);
assert.ok(params.get('l').length > 64, 'fixture must cross the old cap');
assert.equal(params.get('l').length, 73);
assert.equal(params.get('l').length, 75);
const restored = codec.decodeLayerStateParams(params);
assert.deepEqual(restored?.enabledLayerIds, expectedLayerIds);
assert.deepEqual(restored?.options, state.options);
@@ -391,7 +392,7 @@ test('checker reads a complete future base ledger and rejects retired-token reus
},
(cwd) => {
const published = readPublishedLayerStateReservations('HEAD', cwd);
assert.equal(Object.keys(published).length, 32);
assert.equal(Object.keys(published).length, 33);
assert.equal(published['retired-layer'], '00');
assert.throws(
() =>
@@ -20,6 +20,7 @@
["radio", "r"],
["recent-imagery", "1"],
["rocket-launches", "x"],
["s2-ghostmaps", "5"],
["satellites", "s"],
["street-level", "0"],
["telegeography-submarine-cables", "u"],
+3 -2
View File
@@ -995,10 +995,11 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping
// list descriptions; the missions still ride existing tools.
// Street Level and GNSS extend the manifest; broadcast caveats describe tracked and analyst aircraft.
// Air quality (ECCC AQHI) extends the manifest.
assert.equal(block.length, 30655, 'serialized tool schema length drifted');
// S2 GhostMaps extends the manifest.
assert.equal(block.length, 30723, 'serialized tool schema length drifted');
assert.equal(
crypto.createHash('sha256').update(block).digest('hex'),
'3b1d5e77c10a92f6543a4f9d1ff1870101a0306b65e5b8a6113111c81eda371e',
'5937f0e90d38bf4109136c71bb9561298dd3fca11464ff980a1f982b6c154e5b',
'the first-run missions must ride EXISTING tools: no schema edit, no cache bust',
);
const instructions = fs.readFileSync(
+29 -1
View File
@@ -2937,5 +2937,33 @@
"Gemini access was refused. Check the API key and model access.": "Se denegó el acceso a Gemini. Comprueba la clave de API y el acceso al modelo.",
"Gemini Live is unavailable for the configured model.": "Gemini Live no está disponible para el modelo configurado.",
"Token request is too large": "La solicitud de credencial es demasiado grande",
"Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío"
"Use an empty JSON token request": "Usa una solicitud de credencial con un objeto JSON vacío",
"S2 GhostMaps": "S2 GhostMaps",
"Intel Picture": "Panorama de inteligencia",
"Kinetic 30d": "Cinético 30 d",
"Border Crisis": "Crisis fronteriza",
"Iran Picture": "Panorama de Irán",
"Iran Kinetic": "Irán cinético",
"Yemen FLOT": "FLOT de Yemen",
"Latest Drop": "Última publicación",
"GhostMaps file could not be read": "No se pudo leer el archivo de GhostMaps",
"GhostMaps catalog unavailable": "Catálogo de GhostMaps no disponible",
"Malformed GhostMaps catalog": "Catálogo de GhostMaps mal formado",
"{count} source links ↗ · click card to open the first": {
"one": "{count} enlace a la fuente ↗ · haz clic en la tarjeta para abrirlo",
"many": "{count} de enlaces a fuentes ↗ · haz clic en la tarjeta para abrir el primero",
"other": "{count} enlaces a fuentes ↗ · haz clic en la tarjeta para abrir el primero"
},
"S2 · {name}": "S2 · {name}",
"Report": "Informe",
"Open the source for {title}": "Abrir la fuente de {title}",
"LIVE · GitHub {commit} · {message} · checked every {seconds}s": "EN VIVO · GitHub {commit} · {message} · comprobado cada {seconds} s",
"GitHub {commit} · {message} · reconnecting for live updates": "GitHub {commit} · {message} · reconectando para recibir actualizaciones en vivo",
"Connecting to the S2 GhostMaps repository…": "Conectando con el repositorio S2 GhostMaps…",
"{count} placemarks": {
"one": "{count} marcador",
"many": "{count} de marcadores",
"other": "{count} marcadores"
},
"S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.": "S2 Underground publica estos productos KMZ en GitHub. Las nuevas publicaciones se detectan en segundos y sustituyen automáticamente a la edición anterior. Haz clic en un marcador para ver su informe."
}
+31 -1
View File
@@ -3004,5 +3004,35 @@
"Gemini access was refused. Check the API key and model access.": "Доступ к Gemini запрещён. Проверьте ключ API и доступ к модели.",
"Gemini Live is unavailable for the configured model.": "Gemini Live недоступен для выбранной модели.",
"Token request is too large": "Запрос токена слишком велик",
"Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON"
"Use an empty JSON token request": "Используйте запрос токена с пустым объектом JSON",
"S2 GhostMaps": "S2 GhostMaps",
"Intel Picture": "Разведкартина",
"Kinetic 30d": "Боевые 30 дн.",
"Border Crisis": "Кризис на границе",
"Iran Picture": "Иран: картина",
"Iran Kinetic": "Иран: боевые",
"Yemen FLOT": "Йемен: линия фронта",
"Latest Drop": "Новый выпуск",
"GhostMaps file could not be read": "Не удалось прочитать файл GhostMaps",
"GhostMaps catalog unavailable": "Каталог GhostMaps недоступен",
"Malformed GhostMaps catalog": "Повреждённый каталог GhostMaps",
"{count} source links ↗ · click card to open the first": {
"one": "{count} ссылка на источник ↗ · нажмите на карточку, чтобы открыть",
"few": "{count} ссылки на источники ↗ · нажмите на карточку, чтобы открыть первую",
"many": "{count} ссылок на источники ↗ · нажмите на карточку, чтобы открыть первую",
"other": "{count} ссылки на источники ↗ · нажмите на карточку, чтобы открыть первую"
},
"S2 · {name}": "S2 · {name}",
"Report": "Сводка",
"Open the source for {title}": "Открыть источник: {title}",
"LIVE · GitHub {commit} · {message} · checked every {seconds}s": "В ЭФИРЕ · GitHub {commit} · {message} · проверка каждые {seconds} с",
"GitHub {commit} · {message} · reconnecting for live updates": "GitHub {commit} · {message} · переподключение для обновлений в реальном времени",
"Connecting to the S2 GhostMaps repository…": "Подключение к репозиторию S2 GhostMaps…",
"{count} placemarks": {
"one": "{count} метка",
"few": "{count} метки",
"many": "{count} меток",
"other": "{count} метки"
},
"S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.": "S2 Underground публикует эти продукты KMZ на GitHub. Новые файлы обнаруживаются за секунды и автоматически заменяют предыдущий выпуск. Нажмите на метку, чтобы открыть её сводку."
}
+4
View File
@@ -74,6 +74,8 @@
"Radio",
"Reggae",
"Rock",
"S2 GhostMaps",
"S2 · {name}",
"SAR",
"SE",
"SECTOR {lat} {lon}",
@@ -131,6 +133,8 @@
"NVG",
"R&B",
"REC",
"S2 GhostMaps",
"S2 · {name}",
"VS",
"WHOT/BHOT"
]
+245
View File
@@ -0,0 +1,245 @@
// S2 Underground GhostMaps catalog: which KMZ in the public repository is the
// current edition of each product. Pure and portable (the server builds the
// catalog, tests pin the rules); no i18n, no DOM, no Cesium.
export const GHOSTMAPS_REPO = Object.freeze({
owner: 's2underground',
name: 'GhostMaps',
branch: 'main',
url: 'https://github.com/s2underground/GhostMaps',
});
/** Largest KMZ the proxy downloads; matches the browser's import ceiling. */
export const GHOSTMAPS_MAX_FILE_BYTES = 50 * 1024 * 1024;
const ARCGIS = 'ArcGIS Data for ATAK (KMZs)/';
const INCIDENTS = 'Incident Response Products for ATAK (KMZs and Documents)/';
/**
* Curated products, in panel order. Each names a series of dated uploads; the
* newest edition wins. Patterns match whole repository paths, so a renamed
* folder fails closed (the product drops out) instead of matching elsewhere.
*/
export const GHOSTMAPS_PRODUCTS = Object.freeze([
Object.freeze({
id: 'cip',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Common Intelligence Picture',
defaultOn: true,
pattern: new RegExp(
`^${escape(ARCGIS)}Common Intelligence Picture/Master Database/[^/]+\\.kmz$`,
'i',
),
}),
Object.freeze({
id: 'kinetic-30d',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Kinetic Activity (30 days)',
defaultOn: true,
pattern: new RegExp(
`^${escape(ARCGIS)}Common Intelligence Picture/30-Day Pulls/Kinetic[^/]*\\.kmz$`,
'i',
),
}),
Object.freeze({
id: 'border-crisis',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Border Crisis Map',
defaultOn: false,
pattern: new RegExp(
`^${escape(ARCGIS)}Border Crisis Map/S2Underground_Border_Crisis_Map[^/]*\\.kmz$`,
'i',
),
}),
Object.freeze({
id: 'iran-cip',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Iran Common Intelligence Picture',
defaultOn: false,
pattern: new RegExp(
`^${escape(INCIDENTS)}Iran[^/]*/Iran Common Intelligence Picture/[^/]+\\.kmz$`,
'i',
),
}),
Object.freeze({
id: 'iran-kinetic',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Iran Kinetic Events',
defaultOn: false,
pattern: new RegExp(
`^${escape(INCIDENTS)}Iran[^/]*/Iran Kinetic Events/[^/]+\\.kmz$`,
'i',
),
}),
Object.freeze({
id: 'yemen-flot',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Yemen FLOT',
defaultOn: false,
pattern: new RegExp(
`^${escape(INCIDENTS)}Yemen[^/]*/Yemen FLOT[^/]*\\.kmz$`,
'i',
),
}),
]);
/** The newest incident product outside the curated series and the archive. */
export const GHOSTMAPS_LATEST_DROP = Object.freeze({
id: 'latest-drop',
// i18n-ignore: catalog data; the panel shows its own translated label
title: 'Latest incident drop',
defaultOn: false,
});
const LATEST_DROP_EXCLUDED =
/\/(?:0_Archive|Basedata|Reference Material)(?:\/|$)/i;
function escape(text) {
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
const MONTHS = [
'jan',
'feb',
'mar',
'apr',
'may',
'jun',
'jul',
'aug',
'sep',
'oct',
'nov',
'dec',
];
const DATE_IN_NAME =
/(january|february|march|april|may|june|july|august|september|sept|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)[ _-]?(\d{1,2})(?:st|nd|rd|th)?[ ,_-]+(20\d\d)/i;
const EDITION_IN_NAME = /(?:[ _-]export)?[ _-]v?(\d{1,3})$/i;
/**
* The date and edition written into an S2 file name, e.g.
* `..._October_16_2026_Export_1.kmz` → 2026-10-16, edition 1. Undated names
* report `dateMs: null` and sort below every dated edition.
* @param {string} fileName
* @returns {{dateMs: ?number, edition: number}}
*/
export function parseGhostMapsEdition(fileName) {
const stem = String(fileName || '').replace(/\.km[lz]$/i, '');
const match = DATE_IN_NAME.exec(stem);
let dateMs = null;
if (match) {
const month = MONTHS.indexOf(match[1].slice(0, 3).toLowerCase());
const day = Number(match[2]);
const year = Number(match[3]);
const at = Date.UTC(year, month, day);
// Date.UTC rolls 31 Sep into 1 Oct; only an exact calendar date counts.
if (month >= 0 && new Date(at).getUTCDate() === day) dateMs = at;
}
const tail = match ? stem.slice(match.index + match[0].length) : stem;
const edition = Number(EDITION_IN_NAME.exec(tail)?.[1] ?? 0);
return { dateMs, edition };
}
function newer(a, b) {
if (!b) return true;
const ad = a.dateMs ?? -Infinity;
const bd = b.dateMs ?? -Infinity;
if (ad !== bd) return ad > bd;
if (a.edition !== b.edition) return a.edition > b.edition;
return a.path > b.path;
}
/** `S2Underground_Border_Crisis_Map_June_02_2026_v1.kmz` → readable words. */
export function ghostMapsFileTitle(fileName) {
return String(fileName || '')
.replace(/\.km[lz]$/i, '')
.replace(/^S2Underground_/i, '')
.replace(/_/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
/**
* Build the product catalog from a recursive git tree listing.
* @param {Array<{path: string, type: string, sha: string, size?: number}>} tree
* @returns {Array<{id: string, title: string, defaultOn: boolean,
* path: string, fileName: string, fileTitle: string, version: string,
* size: number, dateMs: ?number, edition: number}>}
*/
export function buildGhostMapsCatalog(tree) {
const best = new Map();
let drop = null;
for (const entry of Array.isArray(tree) ? tree : []) {
if (entry?.type !== 'blob' || typeof entry.path !== 'string') continue;
if (!/\.kmz$/i.test(entry.path)) continue;
if (!/^[0-9a-f]{40}$/.test(entry.sha || '')) continue;
const size = Number(entry.size);
if (!Number.isFinite(size) || size <= 0 || size > GHOSTMAPS_MAX_FILE_BYTES)
continue;
const fileName = entry.path.slice(entry.path.lastIndexOf('/') + 1);
const candidate = {
path: entry.path,
fileName,
version: entry.sha,
size,
...parseGhostMapsEdition(fileName),
};
const product = GHOSTMAPS_PRODUCTS.find(({ pattern }) =>
pattern.test(entry.path),
);
if (product) {
if (newer(candidate, best.get(product.id)))
best.set(product.id, candidate);
} else if (
entry.path.startsWith(INCIDENTS) &&
!LATEST_DROP_EXCLUDED.test(`/${entry.path}`) &&
candidate.dateMs != null &&
newer(candidate, drop)
) {
drop = candidate;
}
}
const rows = [];
for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) {
const file =
product === GHOSTMAPS_LATEST_DROP ? drop : best.get(product.id);
if (!file) continue;
rows.push({
id: product.id,
title: product.title,
defaultOn: product.defaultOn,
...file,
fileTitle: ghostMapsFileTitle(file.fileName),
});
}
return rows;
}
/**
* Raw download URL pinned to a commit, so a CDN can never hand back an older
* edition for a newer catalog.
* @param {string} commit 40-hex commit sha.
* @param {string} path Repository path.
*/
export function ghostMapsRawUrl(commit, path) {
const encoded = path.split('/').map(encodeURIComponent).join('/');
return `https://raw.githubusercontent.com/${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}/${commit}/${encoded}`;
}
/**
* The branch head from a git smart-HTTP ref advertisement
* (`/info/refs?service=git-upload-pack`), or null.
* @param {string} text
* @param {string} [branch]
* @returns {?string}
*/
export function parseGitRefAdvertisement(text, branch = GHOSTMAPS_REPO.branch) {
const ref = `refs/heads/${branch}`;
for (const line of String(text || '').split('\n')) {
// pkt-line: 4 hex length digits, then "<sha> <ref>" (the first ref line
// also carries a NUL and the capability list).
const match = /^[0-9a-f]{4}([0-9a-f]{40}) ([^\0\s]+)/.exec(line);
if (match && match[2] === ref) return match[1];
}
return null;
}
+165
View File
@@ -0,0 +1,165 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
buildGhostMapsCatalog,
ghostMapsFileTitle,
ghostMapsRawUrl,
parseGhostMapsEdition,
parseGitRefAdvertisement,
} from './catalog.js';
const ARCGIS = 'ArcGIS Data for ATAK (KMZs)/';
const INCIDENTS = 'Incident Response Products for ATAK (KMZs and Documents)/';
let n = 0;
const blob = (path, size = 1000) => ({
path,
type: 'blob',
size,
sha: (++n).toString(16).padStart(40, '0'),
});
test('editions read the date and export/version number written in the name', () => {
assert.deepEqual(
parseGhostMapsEdition(
'S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz',
),
{ dateMs: Date.UTC(2026, 9, 16), edition: 1 },
);
assert.deepEqual(
parseGhostMapsEdition(
'S2Underground_Border_Crisis_Map_September_21_2026_Export_v2.kmz',
),
{ dateMs: Date.UTC(2026, 8, 21), edition: 2 },
);
assert.deepEqual(
parseGhostMapsEdition('Reno Wildfires August 24, 2026.kmz'),
{
dateMs: Date.UTC(2026, 7, 24),
edition: 0,
},
);
assert.deepEqual(
parseGhostMapsEdition('Venezuelan_Crisis_Sep_5_2025_v2.kmz'),
{
dateMs: Date.UTC(2025, 8, 5),
edition: 2,
},
);
assert.deepEqual(
parseGhostMapsEdition('Kinetic_Activities_Last_30_Days.kmz'),
{
dateMs: null,
edition: 0,
},
);
// An impossible date is not a date.
assert.equal(parseGhostMapsEdition('X_September_31_2026.kmz').dateMs, null);
});
test('each product keeps its newest edition by date, then edition number', () => {
const master = `${ARCGIS}Common Intelligence Picture/Master Database/`;
const tree = [
blob(
`${master}S2Underground_Common_Intelligence_Picture_October_9_2026_Export_1.kmz`,
),
blob(
`${master}S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz`,
),
blob(
`${master}S2Underground_Common_Intelligence_Picture_October_1_2026_Export_1.kmz`,
),
blob(
`${master}S2Underground_Common_Intelligence_Picture_September_30_2026_Export_2.kmz`,
),
blob(
`${ARCGIS}Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz`,
),
blob(
`${ARCGIS}Border Crisis Map/S2Underground_Border_Crisis_Map_September_21_2026_Export_v1.kmz`,
),
blob(
`${ARCGIS}Border Crisis Map/S2Underground_Border_Crisis_Map_September_21_2026_Export_v2.kmz`,
),
// A European-only subset in a subfolder is not the border map.
blob(
`${ARCGIS}Border Crisis Map/European Layers Only/Europe_Migration_Crisis_June_02_2026_v1.kmz`,
),
blob(`${INCIDENTS}Yemen 2026/Yemen FLOT.kmz`),
blob(`${INCIDENTS}Yemen 2026/Yemen FLOT_October_10_2026.kmz`),
{ path: `${INCIDENTS}Yemen 2026`, type: 'tree', sha: 'f'.repeat(40) },
];
const rows = buildGhostMapsCatalog(tree);
const byId = Object.fromEntries(rows.map((row) => [row.id, row]));
assert.deepEqual(
rows.map(({ id }) => id),
['cip', 'kinetic-30d', 'border-crisis', 'yemen-flot'],
);
assert.match(byId.cip.fileName, /October_16_2026/);
assert.equal(byId.cip.defaultOn, true);
assert.equal(byId['kinetic-30d'].defaultOn, true);
assert.match(byId['border-crisis'].fileName, /Export_v2/);
assert.equal(byId['yemen-flot'].fileName, 'Yemen FLOT_October_10_2026.kmz');
assert.equal(byId.cip.version, tree[1].sha);
assert.equal(
byId.cip.fileTitle,
'Common Intelligence Picture October 16 2026 Export 1',
);
});
test('latest drop is the newest dated incident file outside curated, archived and base data', () => {
const rows = buildGhostMapsCatalog([
blob(`${INCIDENTS}Reno Wildfires August 24, 2026.kmz`),
blob(
`${INCIDENTS}Spokane Fire Disaster - August, 2026/SpokaneDisasterAid_August_8_2026.kmz`,
),
blob(
`${INCIDENTS}0_Archive/Texas Floods/TexasFloods_December_1_2026_v1.kmz`,
),
blob(`${INCIDENTS}Iran 2026/Basedata/Pipelines_December_1_2026.kmz`),
blob(
`${INCIDENTS}Iran 2026/Iran Kinetic Events/Iran_Kinetic_Events_December_2_2026_v1.kmz`,
),
blob(`${INCIDENTS}Undated Product.kmz`),
]);
const drop = rows.find(({ id }) => id === 'latest-drop');
assert.equal(drop.fileName, 'Reno Wildfires August 24, 2026.kmz');
assert.ok(rows.some(({ id }) => id === 'iran-kinetic'));
});
test('oversized, empty, non-KMZ and malformed entries are skipped', () => {
const path = `${ARCGIS}Common Intelligence Picture/Master Database/S2Underground_Common_Intelligence_Picture_October_16_2026_Export_1.kmz`;
assert.deepEqual(
buildGhostMapsCatalog([
{ ...blob(path), size: 51 * 1024 * 1024 },
{ ...blob(path), size: 0 },
{ ...blob(path), sha: 'not-a-sha' },
blob(path.replace(/\.kmz$/, '.zip')),
null,
]),
[],
);
assert.deepEqual(buildGhostMapsCatalog(undefined), []);
});
test('raw URLs are pinned to the commit and encode every path segment', () => {
assert.equal(
ghostMapsRawUrl('a'.repeat(40), 'Folder (KMZs)/Yemen FLOT.kmz'),
`https://raw.githubusercontent.com/s2underground/GhostMaps/${'a'.repeat(40)}/Folder%20(KMZs)/Yemen%20FLOT.kmz`,
);
assert.equal(ghostMapsFileTitle('S2Underground_A__B.kmz'), 'A B');
});
test('the branch head comes from the smart-HTTP ref advertisement', () => {
const head = '142d7630bc8f1f2c4b96e16ee865ca50d1ab578d';
const other = 'b'.repeat(40);
const text = [
'001e# service=git-upload-pack',
`0159${head} HEAD\0multi_ack symref=HEAD:refs/heads/main`,
`003f${other} refs/heads/main-old`,
`003d${head} refs/heads/main`,
'0000',
].join('\n');
assert.equal(parseGitRefAdvertisement(text), head);
assert.equal(parseGitRefAdvertisement(text, 'dev'), null);
assert.equal(parseGitRefAdvertisement(''), null);
});
+708
View File
@@ -0,0 +1,708 @@
import * as Cesium from 'cesium';
import {
decodeXml,
kmzImageResolver,
kmzMainDocument,
parseXml,
readZipEntries,
resolveKmzPath,
} from '../../data/geoFileImport.js';
import {
formatNumber,
formatUtcDayMonth,
msg,
t,
tn,
} from '../../i18n/index.js';
import { createHorizonCull } from '../streetLevel/view.js';
import { readGhostMapsKml } from './kml.js';
import { fieldLabel, parseDescriptionFields, wrapCardText } from './records.js';
export * from './catalog.js';
export * from './records.js';
export { readGhostMapsKml } from './kml.js';
export { createGhostMapsSource } from './source.js';
export const GHOSTMAPS_LAYER_ID = 's2-ghostmaps';
export const GHOSTMAPS_OVERLAY_SOURCE_ID = 's2-ghostmaps';
const PICK_PREFIX = 'ghostmaps:';
const STORAGE_KEY = 'gev.ghostmaps.products';
const ACCENT = '#c8ff00';
const MAX_ICON_BYTES = 512 * 1024;
const CARD_HOST_OPTIONS = Object.freeze({
cohortLimit: 1,
collisionCapacity: 1,
moving: false,
});
const MAX_CARD_LINES = 12;
/** Panel names for the catalog's product ids (the catalog sends English). */
const PRODUCT_LABELS = Object.freeze({
cip: msg('Intel Picture'),
'kinetic-30d': msg('Kinetic 30d'),
'border-crisis': msg('Border Crisis'),
'iran-cip': msg('Iran Picture'),
'iran-kinetic': msg('Iran Kinetic'),
'yemen-flot': msg('Yemen FLOT'),
'latest-drop': msg('Latest Drop'),
});
/** One distinct fallback marker color per product. */
const PRODUCT_COLORS = Object.freeze({
cip: '#c8ff00',
'kinetic-30d': '#ff5a36',
'border-crisis': '#ffb547',
'iran-cip': '#39d0ff',
'iran-kinetic': '#ff3b8d',
'yemen-flot': '#5dff9f',
'latest-drop': '#b48cff',
});
function defaultStorage() {
try {
return globalThis.localStorage ?? null;
} catch {
return null;
}
}
function readStoredSelection(storage) {
try {
const value = JSON.parse(storage?.getItem(STORAGE_KEY) ?? 'null');
return Array.isArray(value) ? new Set(value.map(String)) : null;
} catch {
return null;
}
}
function writeStoredSelection(storage, selected) {
try {
storage?.setItem(STORAGE_KEY, JSON.stringify([...selected]));
} catch {
/* private window or blocked storage: the selection lasts this session */
}
}
/** A round marker for placemarks whose icon is missing or remote. */
const markerCache = new Map();
function markerImage(color) {
if (markerCache.has(color)) return markerCache.get(color);
let url = null;
try {
const canvas = document.createElement('canvas');
canvas.width = canvas.height = 20;
const ctx = canvas.getContext('2d');
ctx.beginPath();
ctx.arc(10, 10, 7, 0, Math.PI * 2);
ctx.fillStyle = color;
ctx.fill();
ctx.lineWidth = 2;
ctx.strokeStyle = 'rgba(0,0,0,0.85)';
ctx.stroke();
url = canvas.toDataURL();
} catch {
url = null;
}
markerCache.set(color, url);
return url;
}
const cesiumColor = (rgba, fallback, alpha) =>
rgba
? new Cesium.Color(rgba[0], rgba[1], rgba[2], alpha ?? rgba[3])
: Cesium.Color.fromCssColorString(fallback).withAlpha(alpha ?? 1);
function anchorOf(geometry) {
if (geometry.type === 'point') return geometry.position;
const ring =
geometry.type === 'polygon' ? geometry.outer : geometry.positions;
let lon = 0;
let lat = 0;
for (const [x, y] of ring) {
lon += x;
lat += y;
}
return [lon / ring.length, lat / ring.length];
}
/**
* S2 Underground GhostMaps: the current editions of S2's public KMZ intel
* products, kept in step with the GitHub repository. The proxy watches the
* branch head and pushes each new commit; only changed products reload.
*/
export function createGhostMapsLayer({
source,
overlayHost = null,
screenSpaceEventHandlerFactory = null,
picking = null,
pointer = null,
openExternal = null,
storage = defaultStorage(),
} = {}) {
if (typeof source?.getCatalog !== 'function')
throw new TypeError('GhostMaps layer requires a catalog source');
let _viewer = null;
let _enabled = false;
let _catalog = null;
let _lastError = null;
let _unsubscribe = null;
let _stream = 'closed';
let _queue = Promise.resolve();
let _clickHandler = null;
let _rowListener = null;
let _selectedKey = null;
let _selectedCardId = null;
let _selectedLink = null;
let _lifetime = new AbortController();
let _selection = readStoredSelection(storage);
/** productId → {version, dataSource, urls, count, features} */
const _loaded = new Map();
/** productId → in-flight version */
const _loading = new Map();
/** productId → error text */
const _failed = new Map();
const notifyRow = () => _rowListener?.();
const render = () => _viewer?.scene?.requestRender?.();
// Pins skip the depth test, and with Google 3D tiles the globe is hidden,
// so nothing else stops far-side pins drawing through the planet.
const _horizon = createHorizonCull({
getViewer: () => _viewer,
items: function* () {
for (const item of _loaded.values()) yield* item.pins;
},
onChange: render,
});
const canSelect = () =>
overlayHost && screenSpaceEventHandlerFactory && picking;
function wanted(product) {
return _selection ? _selection.has(product.id) : product.defaultOn;
}
function selectedIds() {
return (_catalog?.products || []).filter(wanted).map(({ id }) => id);
}
function unload(id) {
const item = _loaded.get(id);
if (!item) return;
_loaded.delete(id);
_viewer?.dataSources?.remove(item.dataSource, true);
item.urls.forEach((url) => URL.revokeObjectURL(url));
if (_selectedKey?.startsWith(`${id}:`)) clearSelection();
}
/** Build one product's data source off-scene, then swap it in. */
async function load(product, signal) {
const bytes = await source.getFile(product.id, product.version, { signal });
const entries = await readZipEntries(bytes);
const main = kmzMainDocument(entries);
if (!main) throw new Error(msg('That KMZ holds no KML document.'));
const urls = [];
const resolvePacked = kmzImageResolver(entries, main, (url) =>
urls.push(url),
);
// Icons only: ArcGIS exports also pack multi-megabyte photo attachments,
// which fall back to the product marker instead of being decoded.
const resolveImage = (href) => {
const path = resolveKmzPath(href, main, entries);
return path && entries.get(path).length <= MAX_ICON_BYTES
? resolvePacked(href)
: null;
};
let parsed;
try {
parsed = readGhostMapsKml(
parseXml(decodeXml(entries.get(main)), 'KML'),
resolveImage,
);
} catch (error) {
urls.forEach((url) => URL.revokeObjectURL(url));
throw error;
}
entries.clear();
signal.throwIfAborted();
const color = PRODUCT_COLORS[product.id] || ACCENT;
const dataSource = new Cesium.CustomDataSource(
`${GHOSTMAPS_LAYER_ID}:${product.id}`,
);
const features = [];
const pins = [];
dataSource.entities.suspendEvents();
parsed.features.forEach((feature, index) => {
const key = `${product.id}:${index}`;
const { style } = feature;
feature.geometries.forEach((geometry, part) => {
const id = `${PICK_PREFIX}${key}:${part}`;
if (geometry.type === 'point') {
const position = Cesium.Cartesian3.fromDegrees(...geometry.position);
const entity = dataSource.entities.add({
id,
position,
billboard: {
image: style.icon || markerImage(color),
scale: Math.min(
1.5,
Math.max(0.35, (style.iconScale ?? 1) * 0.7),
),
color:
style.icon && style.iconColor
? cesiumColor(style.iconColor)
: undefined,
heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
verticalOrigin: Cesium.VerticalOrigin.CENTER,
// Ground-clamped icons would sink into terrain and 3D tiles; the
// horizon cull hides the far side instead (see `_horizon`).
disableDepthTestDistance: Number.POSITIVE_INFINITY,
},
});
pins.push({
position,
get show() {
return entity.show;
},
set show(value) {
entity.show = value;
},
});
} else if (geometry.type === 'line') {
dataSource.entities.add({
id,
polyline: {
positions: Cesium.Cartesian3.fromDegreesArray(
geometry.positions.flat(),
),
width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)),
material: cesiumColor(style.lineColor, color),
clampToGround: true,
},
});
} else {
const ring = (points) =>
Cesium.Cartesian3.fromDegreesArray(points.flat());
dataSource.entities.add({
id,
polygon: {
hierarchy: new Cesium.PolygonHierarchy(
ring(geometry.outer),
geometry.holes.map(
(hole) => new Cesium.PolygonHierarchy(ring(hole)),
),
),
material:
style.fill === false
? Cesium.Color.TRANSPARENT
: cesiumColor(
style.polyColor,
color,
style.polyColor ? undefined : 0.25,
),
classificationType: Cesium.ClassificationType.BOTH,
},
});
if (style.outline !== false)
dataSource.entities.add({
id: `${id}:outline`,
polyline: {
positions: ring([...geometry.outer, geometry.outer[0]]),
width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)),
material: cesiumColor(style.lineColor, color),
clampToGround: true,
},
});
}
});
features.push({
name: feature.name,
folder: feature.folder,
description: feature.description,
anchor: anchorOf(feature.geometries[0]),
});
});
dataSource.entities.resumeEvents();
return { version: product.version, dataSource, urls, features, pins };
}
/** Bring the globe in line with the catalog and the chip selection. */
function sync() {
const run = _queue.then(async () => {
if (!_enabled || !_viewer || !_catalog) return;
const signal = _lifetime.signal;
const products = new Map(_catalog.products.map((p) => [p.id, p]));
for (const id of [..._loaded.keys()]) {
const product = products.get(id);
if (!product || !wanted(product)) unload(id);
}
for (const product of _catalog.products) {
if (!wanted(product) || signal.aborted) continue;
if (_loaded.get(product.id)?.version === product.version) continue;
_loading.set(product.id, product.version);
_failed.delete(product.id);
notifyRow();
try {
const next = await load(product, signal);
if (signal.aborted || !_enabled) {
next.urls.forEach((url) => URL.revokeObjectURL(url));
return;
}
// Cull before the first frame so far-side pins never flash.
_horizon.update(next.pins);
await _viewer.dataSources.add(next.dataSource);
// The new edition is on the globe before the old one leaves.
unload(product.id);
_loaded.set(product.id, next);
} catch (error) {
if (signal.aborted) return;
console.warn(`[Data:GhostMaps] ${product.id}:`, error);
_failed.set(
product.id,
error?.template ||
error?.message ||
msg('GhostMaps file could not be read'),
);
} finally {
_loading.delete(product.id);
notifyRow();
render();
}
}
});
_queue = run.catch(() => {});
return run;
}
function applyCatalog(catalog) {
_catalog = catalog;
_lastError = catalog.error || null;
notifyRow();
return sync();
}
function toggleProduct(id) {
// The first change pins the whole current selection, defaults included.
if (!_selection) _selection = new Set(selectedIds());
if (_selection.has(id)) _selection.delete(id);
else _selection.add(id);
writeStoredSelection(storage, _selection);
notifyRow();
void sync();
}
// ---- selection card ----------------------------------------------------
function featureFor(key) {
const [productId, index] = key.split(':');
return {
product: _catalog?.products.find((p) => p.id === productId) || null,
feature: _loaded.get(productId)?.features[Number(index)] || null,
};
}
function publishCard() {
if (!canSelect()) return;
const { product, feature } = _selectedKey ? featureFor(_selectedKey) : {};
if (!feature) {
_selectedKey = null;
_selectedCardId = null;
_selectedLink = null;
overlayHost.setEntries(
GHOSTMAPS_OVERLAY_SOURCE_ID,
[],
CARD_HOST_OPTIONS,
);
return;
}
const { fields, links } = parseDescriptionFields(feature.description);
_selectedLink = links[0] || null;
const heading = [t(PRODUCT_LABELS[product?.id] || product?.title || '')];
if (feature.folder) heading.push(fieldLabel(feature.folder));
if (product?.dateMs) heading.push(formatUtcDayMonth(product.dateMs));
const details = [heading.filter(Boolean).join(' · ')];
for (const { label, value } of fields) {
if (details.length >= MAX_CARD_LINES) break;
// ArcGIS repeats the feature type as the first row; the title has it.
if (value === feature.name) continue;
const line = label ? `${fieldLabel(label)}: ${value}` : value;
details.push(
...wrapCardText(line, 60, Math.min(4, MAX_CARD_LINES - details.length)),
);
}
if (_selectedLink && openExternal)
details.push(
tn(
links.length,
'{count} source link ↗ · click card to open',
'{count} source links ↗ · click card to open the first',
{ count: formatNumber(links.length) },
),
);
const title = t('S2 · {name}', {
name: fieldLabel(feature.name || feature.folder) || t('Report'),
});
const card = {
id: `ghostmaps-card:${_selectedKey}`,
selected: true,
interactive: Boolean(_selectedLink && openExternal),
title,
details,
accent: PRODUCT_COLORS[product?.id] || ACCENT,
priority: Number.MAX_SAFE_INTEGER,
gapPx: 15,
verticalOnly: true,
placement: 'above',
// A click card comes and goes with the click, as CCTV cards do, rather
// than holding the label arbiter's 2.5 s minimum lifetime; and it hides
// when its marker turns behind the planet.
stateless: true,
horizonCull: true,
position: Cesium.Cartesian3.fromDegrees(...feature.anchor),
};
if (card.interactive) {
const link = _selectedLink;
card.accessibilityLabel = t('Open the source for {title}', { title });
card.activate = () => {
openExternal(link);
return true;
};
}
_selectedCardId = card.id;
overlayHost.setEntries(
GHOSTMAPS_OVERLAY_SOURCE_ID,
[card],
CARD_HOST_OPTIONS,
);
overlayHost.setVisible?.(GHOSTMAPS_OVERLAY_SOURCE_ID, true);
}
function clearSelection() {
_selectedKey = null;
_selectedCardId = null;
_selectedLink = null;
// An empty publish, as the perimeter cards deselect: it releases the
// selected card's paint lane, which clearing the source alone left drawn.
overlayHost?.setEntries(GHOSTMAPS_OVERLAY_SOURCE_ID, [], CARD_HOST_OPTIONS);
}
function pickedKey(picked) {
const pickId = picking.resolvePickId(picked);
if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX))
return null;
const [productId, index] = pickId.slice(PICK_PREFIX.length).split(':');
return _loaded.get(productId)?.features[Number(index)]
? `${productId}:${index}`
: null;
}
function installClickHandler() {
if (!canSelect() || _clickHandler || !_viewer) return;
picking.registerPickOwner?.(
GHOSTMAPS_LAYER_ID,
(pickedId) =>
typeof pickedId === 'string' && pickedId.startsWith(PICK_PREFIX),
);
_clickHandler = screenSpaceEventHandlerFactory(_viewer);
_clickHandler.setInputAction((click) => {
if (pointer && !pointer.isPointerFree()) return;
const cardHit = overlayHost.hitTest?.(
click.position?.x,
click.position?.y,
{ sourceId: GHOSTMAPS_OVERLAY_SOURCE_ID },
);
if (cardHit && cardHit.entryId === _selectedCardId) {
if (_selectedLink && openExternal) openExternal(_selectedLink);
return;
}
const picked = _viewer.scene.pick(click.position);
const key = picked ? pickedKey(picked) : null;
if (key) {
_selectedKey = key;
publishCard();
render();
return;
}
if (picked) {
const pickId = picking.resolvePickId(picked);
if (pickId && picking.isOwnedByOtherLayer(GHOSTMAPS_LAYER_ID, pickId))
return;
}
if (_selectedKey) {
clearSelection();
render();
}
}, Cesium.ScreenSpaceEventType.LEFT_CLICK);
}
function removeClickHandler() {
_clickHandler?.destroy();
_clickHandler = null;
picking?.unregisterPickOwner?.(GHOSTMAPS_LAYER_ID);
}
function featureCount() {
let count = 0;
for (const item of _loaded.values()) count += item.features.length;
return count;
}
const layer = {
id: GHOSTMAPS_LAYER_ID,
name: 'S2 GhostMaps',
icon: '👻',
source: 'S2 Underground',
// Pushes arrive within seconds over the event stream; this interval is
// the backstop if the stream is down.
updateInterval: 300000,
init(viewer) {
if (_viewer) throw new Error('GhostMaps layer is already initialized');
_viewer = viewer;
},
enable() {
_enabled = true;
_lifetime = new AbortController();
for (const item of _loaded.values()) item.dataSource.show = true;
installClickHandler();
if (!_unsubscribe && typeof source.subscribe === 'function')
_unsubscribe = source.subscribe(
(catalog) => {
if (_enabled) void applyCatalog(catalog);
},
(state) => {
if (_stream === state) return;
_stream = state;
notifyRow();
},
);
},
disable() {
_enabled = false;
_lifetime.abort();
_unsubscribe?.();
_unsubscribe = null;
_stream = 'closed';
removeClickHandler();
clearSelection();
_horizon.stop();
for (const id of [..._loaded.keys()]) unload(id);
render();
},
async update(_viewerArg, { signal } = {}) {
if (!_enabled || signal?.aborted) return false;
try {
const catalog = await source.getCatalog({ signal });
if (!_enabled) return false;
await applyCatalog(catalog);
return true;
} catch (error) {
if (signal?.aborted || !_enabled) return false;
console.warn('[Data:GhostMaps] catalog error:', error);
_lastError = error?.message || msg('GhostMaps catalog unavailable');
notifyRow();
// Keep the layer on; the event stream or the next interval recovers.
return true;
}
},
destroy() {
layer.disable();
_viewer = null;
_catalog = null;
_rowListener = null;
},
setRowControlsListener(listener) {
_rowListener = listener;
},
getRowControls() {
const products = _catalog?.products || [];
const commit = _catalog?.commit;
const info = commit
? _stream === 'open'
? t('LIVE · GitHub {commit} · {message} · checked every {seconds}s', {
commit: commit.slice(0, 7),
message: _catalog.message || '—',
seconds: formatNumber(
Math.round((_catalog.pollMs || 15000) / 1000),
),
})
: t('GitHub {commit} · {message} · reconnecting for live updates', {
commit: commit.slice(0, 7),
message: _catalog.message || '—',
})
: t('Connecting to the S2 GhostMaps repository…');
return {
chips: products.map((product) => {
const loaded = _loaded.get(product.id);
const failed = _failed.get(product.id);
return {
id: product.id,
label: t(PRODUCT_LABELS[product.id] || product.title),
active: wanted(product),
busy: _loading.has(product.id),
state: failed ? 'error' : undefined,
color: PRODUCT_COLORS[product.id],
title: [
product.fileTitle,
loaded
? tn(
loaded.features.length,
'{count} placemark',
'{count} placemarks',
{
count: formatNumber(loaded.features.length),
},
)
: null,
failed ? t(failed) : null,
]
.filter(Boolean)
.join(' · '),
onClick: () => toggleProduct(product.id),
};
}),
legend: [],
info,
infoTitle: t(
'S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.',
),
};
},
getStats() {
return {
count: featureCount(),
products: [..._loaded.keys()],
commit: _catalog?.commit || null,
lastUpdate: _catalog?.committedAt || _catalog?.changedAt || null,
checkedAt: _catalog?.checkedAt || null,
stream: _stream,
loading: _loading.size > 0,
error: _lastError || [..._failed.values()][0] || null,
fallback: false,
};
},
/** Loaded placemarks as plain records, for tools and QA. */
getAnalystRecords(maxCount = Infinity) {
if (!_enabled) return [];
const out = [];
for (const [productId, item] of _loaded) {
for (const feature of item.features) {
if (out.length >= maxCount) return out;
out.push({
product: productId,
name: feature.name,
folder: feature.folder,
lon: feature.anchor[0],
lat: feature.anchor[1],
});
}
}
return out;
},
};
return layer;
}
+198
View File
@@ -0,0 +1,198 @@
// Read the drawable content of a parsed GhostMaps KML document: placemarks
// with their folder, geometry, resolved style and raw description. Network
// links, overlays, models and remote icons are never followed; an icon is
// either an image packed in the same KMZ or the layer's own marker.
import { kmlColor } from './records.js';
export const MAX_GHOSTMAPS_FEATURES = 20_000;
export const MAX_GHOSTMAPS_POSITIONS = 250_000;
const childrenNamed = (el, name) =>
[...(el?.children || [])].filter((child) => child.localName === name);
const firstChild = (el, name) =>
[...(el?.children || [])].find((child) => child.localName === name) || null;
const descendants = (el, name) =>
[...(el?.getElementsByTagName?.('*') || [])].filter(
(child) => child.localName === name,
);
const text = (el) => (el?.textContent ?? '').trim();
function number(el, fallback) {
const value = Number(text(el));
return el && Number.isFinite(value) ? value : fallback;
}
/** `lon,lat[,alt] lon,lat ...` → [[lon, lat], ...]; invalid tuples are dropped. */
export function parseKmlCoordinates(value) {
const out = [];
for (const tuple of String(value ?? '')
.trim()
.split(/\s+/)) {
const [lon, lat] = tuple.split(',').map(Number);
if (
Number.isFinite(lon) &&
Number.isFinite(lat) &&
Math.abs(lat) <= 90 &&
Math.abs(lon) <= 180
)
out.push([lon, lat]);
}
return out;
}
function readStyle(styleEl, resolveImage) {
const style = {};
const icon = firstChild(styleEl, 'IconStyle');
if (icon) {
const href = text(firstChild(firstChild(icon, 'Icon'), 'href'));
style.icon = href ? resolveImage(href) : null;
style.iconScale = number(firstChild(icon, 'scale'), 1);
style.iconColor = kmlColor(text(firstChild(icon, 'color')));
}
const line = firstChild(styleEl, 'LineStyle');
if (line) {
style.lineColor = kmlColor(text(firstChild(line, 'color')));
style.lineWidth = number(firstChild(line, 'width'), 1);
}
const poly = firstChild(styleEl, 'PolyStyle');
if (poly) {
style.polyColor = kmlColor(text(firstChild(poly, 'color')));
style.fill = text(firstChild(poly, 'fill')) !== '0';
style.outline = text(firstChild(poly, 'outline')) !== '0';
}
return style;
}
/** Shared styles by id; a StyleMap resolves to its `normal` pair. */
function collectStyles(doc, resolveImage) {
const styles = new Map();
for (const el of descendants(doc, 'Style')) {
const id = el.getAttribute('id');
if (id) styles.set(id, readStyle(el, resolveImage));
}
const maps = new Map();
for (const el of descendants(doc, 'StyleMap')) {
const id = el.getAttribute('id');
if (!id) continue;
const normal = childrenNamed(el, 'Pair').find(
(pair) => text(firstChild(pair, 'key')) === 'normal',
);
if (!normal) continue;
const inline = firstChild(normal, 'Style');
maps.set(
id,
inline
? readStyle(inline, resolveImage)
: text(firstChild(normal, 'styleUrl')),
);
}
const resolve = (url, depth = 0) => {
const id = String(url || '').replace(/^#/, '');
if (!id || url.includes('/') || depth > 2) return {};
if (styles.has(id)) return styles.get(id);
const mapped = maps.get(id);
if (typeof mapped === 'string') return resolve(mapped, depth + 1);
return mapped || {};
};
return resolve;
}
function readGeometry(el, out, budget) {
switch (el.localName) {
case 'Point': {
const [position] = parseKmlCoordinates(
text(firstChild(el, 'coordinates')),
);
if (position) {
budget.positions += 1;
out.push({ type: 'point', position });
}
break;
}
case 'LineString':
case 'LinearRing': {
const positions = parseKmlCoordinates(
text(firstChild(el, 'coordinates')),
);
if (positions.length >= 2) {
budget.positions += positions.length;
out.push({ type: 'line', positions });
}
break;
}
case 'Polygon': {
const ring = (boundary) =>
parseKmlCoordinates(
text(firstChild(firstChild(boundary, 'LinearRing'), 'coordinates')),
);
const outer = ring(firstChild(el, 'outerBoundaryIs'));
if (outer.length >= 3) {
const holes = childrenNamed(el, 'innerBoundaryIs')
.map(ring)
.filter((hole) => hole.length >= 3);
budget.positions +=
outer.length + holes.reduce((n, hole) => n + hole.length, 0);
out.push({ type: 'polygon', outer, holes });
}
break;
}
case 'MultiGeometry':
for (const child of el.children) readGeometry(child, out, budget);
break;
default:
// Models, tracks and anything newer are not drawn.
break;
}
}
/** Name of the nearest enclosing Folder (or Document), for grouping. */
function folderName(el) {
for (let node = el.parentElement; node; node = node.parentElement) {
if (node.localName === 'Folder' || node.localName === 'Document') {
const name = text(firstChild(node, 'name'));
if (name) return name;
}
}
return '';
}
/**
* @param {Document} doc Parsed KML.
* @param {(href: string) => ?string} resolveImage URL for a KMZ-packed image.
* @returns {{features: Array<{name: string, folder: string,
* description: string, style: object, geometries: Array<object>}>,
* positions: number, over: boolean}}
*/
export function readGhostMapsKml(doc, resolveImage = () => null) {
const styleFor = collectStyles(doc, resolveImage);
const features = [];
const budget = { positions: 0 };
let over = false;
for (const placemark of descendants(doc, 'Placemark')) {
// A hidden placemark (visibility 0) is drawn hidden by Google Earth too.
if (text(firstChild(placemark, 'visibility')) === '0') continue;
const geometries = [];
for (const child of placemark.children)
readGeometry(child, geometries, budget);
if (!geometries.length) continue;
if (
features.length >= MAX_GHOSTMAPS_FEATURES ||
budget.positions > MAX_GHOSTMAPS_POSITIONS
) {
over = true;
break;
}
const inline = firstChild(placemark, 'Style');
features.push({
name: text(firstChild(placemark, 'name')).slice(0, 200),
folder: folderName(placemark).slice(0, 120),
description: text(firstChild(placemark, 'description')),
style: {
...styleFor(text(firstChild(placemark, 'styleUrl'))),
...(inline ? readStyle(inline, resolveImage) : {}),
},
geometries,
});
}
return { features, positions: budget.positions, over };
}
+165
View File
@@ -0,0 +1,165 @@
// Pure helpers for GhostMaps placemarks: description tables to plain fields,
// KML colors, and card text wrapping. Strings in, data out — description
// markup is only ever read as text, never inserted into the page.
const ENTITIES = Object.freeze({
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
});
/** Decode the HTML character references ArcGIS and Google Earth emit. */
export function decodeHtmlEntities(text) {
return String(text ?? '').replace(
/&(#x[0-9a-f]{1,6}|#\d{1,7}|[a-z]{2,8});/gi,
(whole, ref) => {
if (ref[0] === '#') {
const code =
ref[1] === 'x' || ref[1] === 'X'
? parseInt(ref.slice(2), 16)
: parseInt(ref.slice(1), 10);
return code > 0 &&
code <= 0x10ffff &&
!(code >= 0xd800 && code <= 0xdfff)
? String.fromCodePoint(code)
: '';
}
return ENTITIES[ref.toLowerCase()] ?? whole;
},
);
}
/** Markup to one line of plain text. */
export function htmlToText(html) {
return decodeHtmlEntities(
String(html ?? '')
.replace(/<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi, ' ')
.replace(/<br\s*\/?>/gi, ' ')
.replace(/<[^>]*>/g, ' '),
)
.replace(/\s+/g, ' ')
.trim();
}
const EMPTY_VALUES = new Set(['', '<null>', 'null', 'n/a', '-', '<blank>']);
// ArcGIS writes geometry type and internal ids into every popup table.
const SKIPPED_LABELS = new Set(['shape', 'objectid', 'fid', 'globalid']);
const MAX_FIELDS = 24;
const MAX_VALUE = 600;
const MAX_LINKS = 6;
/**
* Fields and links from a placemark description. ArcGIS "popup" exports are a
* two-column table (label, value); anything else becomes one unlabelled field.
* @param {string} html Description as written in the KML (CDATA text).
* @returns {{fields: Array<{label: string, value: string}>, links: string[]}}
*/
export function parseDescriptionFields(html) {
const source = String(html ?? '').replace(
/<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi,
' ',
);
const fields = [];
const links = [];
const addLinks = (markup) => {
for (const match of markup.matchAll(/href\s*=\s*["']([^"']+)["']/gi)) {
const url = decodeHtmlEntities(match[1]).trim();
if (/^https?:\/\/[^\s]+$/i.test(url) && !links.includes(url))
links.push(url);
}
};
// Innermost rows only: ArcGIS nests the field table inside an outer row,
// and only leaf rows with exactly two cells are label/value pairs.
for (const row of source.matchAll(
/<tr\b[^>]*>((?:(?!<tr\b)[\s\S])*?)<\/tr\s*>/gi,
)) {
const cells = [
...row[1].matchAll(/<t[dh]\b[^>]*>([\s\S]*?)<\/t[dh]\s*>/gi),
].map((cell) => cell[1]);
if (cells.length !== 2) continue;
const label = htmlToText(cells[0]);
const value = htmlToText(cells[1]);
if (!label || SKIPPED_LABELS.has(label.toLowerCase())) continue;
if (EMPTY_VALUES.has(value.toLowerCase())) continue;
addLinks(cells[1]);
// A bare research link is carried by `links`; the card does not need
// the URL twice.
if (/^https?:\/\/\S+$/i.test(value)) continue;
if (fields.length < MAX_FIELDS)
fields.push({ label, value: value.slice(0, MAX_VALUE) });
}
if (!fields.length && !links.length) {
const text = htmlToText(source);
if (text) fields.push({ label: '', value: text.slice(0, MAX_VALUE) });
addLinks(source);
}
return { fields, links: links.slice(0, MAX_LINKS) };
}
/**
* KML `aabbggrr` to `[r, g, b, a]` in 0..1, or null.
* @param {string} text
* @returns {?number[]}
*/
export function kmlColor(text) {
const hex = String(text ?? '')
.trim()
.replace(/^#/, '');
if (!/^[0-9a-f]{8}$/i.test(hex)) return null;
const byte = (at) => parseInt(hex.slice(at, at + 2), 16) / 255;
return [byte(6), byte(4), byte(2), byte(0)];
}
/**
* Greedy word wrap for canvas card lines; overlong words are cut.
* @param {string} text
* @param {number} width Characters per line.
* @param {number} maxLines Lines kept; the last ends with an ellipsis when cut.
* @returns {string[]}
*/
export function wrapCardText(text, width = 56, maxLines = 4) {
const lines = [];
let line = '';
let cut = false;
for (let word of String(text ?? '')
.split(/\s+/)
.filter(Boolean)) {
while (word.length > width) {
if (line) {
lines.push(line);
line = '';
}
lines.push(word.slice(0, width));
word = word.slice(width);
}
if (!line) line = word;
else if (line.length + 1 + word.length <= width) line += ` ${word}`;
else {
lines.push(line);
line = word;
}
if (lines.length >= maxLines) {
cut = true;
break;
}
}
if (line && lines.length < maxLines) lines.push(line);
else if (line) cut = true;
const kept = lines.slice(0, maxLines);
if (cut && kept.length) {
const last = kept[kept.length - 1];
kept[kept.length - 1] = `${last.slice(0, width - 1)}…`;
}
return kept;
}
/** Field labels ArcGIS writes CamelCase; show them as words. */
export function fieldLabel(label) {
return String(label ?? '')
.replace(/([a-z])([A-Z0-9])/g, '$1 $2')
.replace(/_/g, ' ')
.trim();
}
+106
View File
@@ -0,0 +1,106 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
decodeHtmlEntities,
fieldLabel,
htmlToText,
kmlColor,
parseDescriptionFields,
wrapCardText,
} from './records.js';
// The shape ArcGIS Pro writes into every S2 placemark description.
const ARCGIS = `<html xmlns:fo="http://www.w3.org/1999/XSL/Format"><head>
<META http-equiv="Content-Type" content="text/html"></head>
<body style="margin:0px"><table style="font-family:Arial">
<tr style="text-align:center;font-weight:bold;background:#9CBCE2"><td>HistoricProtestSite</td></tr>
<tr><td><table style="font-family:Arial">
<tr><td>SHAPE</td><td>Point</td></tr>
<tr bgcolor="#D4E4F3"><td>InstallationType</td><td>HistoricProtestSite</td></tr>
<tr><td>OperationalStatus</td><td>ACTIVE</td></tr>
<tr bgcolor="#D4E4F3"><td>Affiliation</td><td>&lt;Null&gt;</td></tr>
<tr><td>Description</td><td>Locals gather here &amp; protest &#8212; weekly.</td></tr>
<tr><td>Research1</td><td><a target="_blank" href="https://x.com/a/status/1">https://x.com/a/status/1</a></td></tr>
<tr><td>Research2</td><td><a href="javascript:alert(1)">bad</a></td></tr>
<tr><td>Research3</td><td>&lt;Null&gt;</td></tr>
</table></td></tr></table></body><script type="text/javascript">
function changeImage(a, b) { document.getElementById('x').innerHTML = b; }
</script></html>`;
test('ArcGIS popup tables become plain label/value fields and http links', () => {
const { fields, links } = parseDescriptionFields(ARCGIS);
assert.deepEqual(fields, [
{ label: 'InstallationType', value: 'HistoricProtestSite' },
{ label: 'OperationalStatus', value: 'ACTIVE' },
{ label: 'Description', value: 'Locals gather here & protest — weekly.' },
{ label: 'Research2', value: 'bad' },
]);
// javascript: and other schemes never become links.
assert.deepEqual(links, ['https://x.com/a/status/1']);
});
test('free-form descriptions become one unlabelled field; scripts are dropped', () => {
assert.deepEqual(
parseDescriptionFields(
'<p>Front line<br>as of 10 Oct</p><script>steal()</script><a href="https://s2.example/x">more</a>',
),
{
fields: [{ label: '', value: 'Front line as of 10 Oct more' }],
links: ['https://s2.example/x'],
},
);
assert.deepEqual(parseDescriptionFields(''), { fields: [], links: [] });
assert.deepEqual(parseDescriptionFields(undefined), {
fields: [],
links: [],
});
});
test('field values and counts are bounded', () => {
const rows = Array.from(
{ length: 40 },
(_, i) => `<tr><td>K${i}</td><td>${'x'.repeat(900)}</td></tr>`,
).join('');
const { fields } = parseDescriptionFields(`<table>${rows}</table>`);
assert.equal(fields.length, 24);
assert.equal(fields[0].value.length, 600);
});
test('entities and tags decode to text', () => {
assert.equal(
decodeHtmlEntities('&lt;b&gt; &#x41;&#66; &nbsp;&bogus;'),
'<b> AB &bogus;',
);
assert.equal(decodeHtmlEntities('&#xD800;&#0;'), '');
assert.equal(htmlToText('<style>a{}</style><b>One</b>\n two'), 'One two');
});
test('KML aabbggrr colors convert to rgba', () => {
assert.deepEqual(kmlColor('ff0000ff'), [1, 0, 0, 1]);
assert.deepEqual(kmlColor('8000ff00'), [0, 1, 0, 128 / 255]);
assert.equal(kmlColor('red'), null);
assert.equal(kmlColor(''), null);
});
test('card text wraps on words, cuts long words and marks truncation', () => {
assert.deepEqual(wrapCardText('one two three four', 9, 4), [
'one two',
'three',
'four',
]);
assert.deepEqual(wrapCardText('abcdefghijkl', 5, 4), [
'abcde',
'fghij',
'kl',
]);
assert.deepEqual(wrapCardText('aa bb cc dd ee', 5, 2), ['aa bb', 'cc d…']);
assert.deepEqual(wrapCardText('', 5, 2), []);
for (const line of wrapCardText('word '.repeat(200), 20, 3))
assert.ok(line.length <= 20);
});
test('CamelCase field labels read as words', () => {
assert.equal(fieldLabel('OperationalStatus'), 'Operational Status');
assert.equal(fieldLabel('Research1'), 'Research 1');
assert.equal(fieldLabel('Kinetic_Activity'), 'Kinetic Activity');
});
+81
View File
@@ -0,0 +1,81 @@
import {
readResponseBytesCapped,
readResponseJsonCapped,
} from '../../sources/httpBody.js';
import { GHOSTMAPS_MAX_FILE_BYTES } from './catalog.js';
// A portable export (package.json): no i18n here. Failures stay English data;
// the Layers panel shows them translated.
const msg = (text) => text;
/**
* Same-origin GhostMaps source: the catalog, one product's KMZ bytes, and a
* push subscription that fires whenever the watcher sees a new commit.
*/
export function createGhostMapsSource({
fetchImpl = (...args) => globalThis.fetch(...args),
EventSourceImpl = globalThis.EventSource,
} = {}) {
return {
async getCatalog({ signal } = {}) {
const response = await fetchImpl('/api/ghostmaps/catalog', { signal });
if (!response.ok)
throw new Error(
`GhostMaps catalog unavailable (HTTP ${response.status})`,
);
const catalog = await readResponseJsonCapped(
response,
1024 * 1024,
signal,
);
if (!Array.isArray(catalog?.products))
throw new Error(msg('Malformed GhostMaps catalog'));
return catalog;
},
async getFile(id, version, { signal } = {}) {
const response = await fetchImpl(
`/api/ghostmaps/file/${encodeURIComponent(id)}?v=${encodeURIComponent(version)}`,
{ signal },
);
if (!response.ok)
throw new Error(`GhostMaps file unavailable (HTTP ${response.status})`);
return readResponseBytesCapped(
response,
GHOSTMAPS_MAX_FILE_BYTES,
signal,
);
},
/**
* Call `onCatalog(catalog)` for every catalog the server pushes, and
* `onState('open' | 'connecting' | 'closed')` as the stream changes. The
* browser's EventSource reconnects by itself. Returns an unsubscribe.
*/
subscribe(onCatalog, onState = () => {}) {
if (typeof EventSourceImpl !== 'function') {
onState('closed');
return () => {};
}
const events = new EventSourceImpl('/api/ghostmaps/events');
const states = ['connecting', 'open', 'closed'];
const report = () => onState(states[events.readyState] ?? 'connecting');
events.onopen = report;
events.onerror = report;
const listener = (event) => {
try {
const catalog = JSON.parse(event.data);
if (Array.isArray(catalog?.products)) onCatalog(catalog);
} catch {
/* ignore a malformed frame; the next one replaces it */
}
};
events.addEventListener('catalog', listener);
report();
return () => {
events.removeEventListener('catalog', listener);
events.close();
};
},
};
}
+2
View File
@@ -11,6 +11,7 @@ test('data providers have both hooks; credential editing stays development-only'
const plugins = localProviderPlugins();
assert.ok(plugins.some(({ name }) => name === 'fire-perimeters'));
assert.ok(plugins.some(({ name }) => name === 'gnss-integrity'));
assert.ok(plugins.some(({ name }) => name === 'ghostmaps'));
for (const plugin of plugins) {
if (plugin.name === 'gev-key-setup') {
assert.equal(plugin.configurePreviewServer, undefined);
@@ -105,6 +106,7 @@ test('real dev and built-preview servers serve provider JSON and terminate unkno
['/api/firms/status', 200],
['/api/fire-perimeters/inciweb/publication/invalid', 400],
['/api/gnss-integrity?lat=invalid', 400],
['/api/ghostmaps/unknown', 404],
['/api/terrain/heights?points=invalid', 400],
['/api/overpass', 405],
['/api/overpass/status', 200],
+1
View File
@@ -34,6 +34,7 @@ export const LAYER_NAME_KEYS = Object.freeze({
'recent-imagery': msg('Recent Imagery'),
'fire-perimeters': msg('Fire Perimeters'),
'gnss-integrity': msg('GNSS Integrity'),
's2-ghostmaps': msg('S2 GhostMaps'),
wind: msg('Wind'),
'weather-radar': msg('Rain radar'),
'weather-satellite': msg('Satellite clouds'),
+1
View File
@@ -73,6 +73,7 @@ const PANEL_GROUPS = [
'local-firms',
'fire-perimeters',
'gnss-integrity',
's2-ghostmaps',
],
},
{
+2 -1
View File
@@ -38,7 +38,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// requested radius and center authority in descriptions only.
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
// Air quality (ECCC AQHI) extends the manifest.
'775ba323384b4c1b62fb8415bac52aecc3fc668e20bedcfa57e426dcd1da3969',
// S2 GhostMaps extends the manifest.
'64a7ae9719cbf53f2c5b57739c1e84e8b60dc4ede040decd48a9c06d1ccbcbca',
);
});
+14
View File
@@ -57,6 +57,20 @@ const layers = [
context: false,
noQuery: 'navigation-integrity cells; no countable analyst records',
},
{
id: 's2-ghostmaps',
aliases: [
's2 ghostmaps',
'ghost maps',
'ghostmaps',
's2 underground',
's2 intel',
'intel picture',
'common intelligence picture',
],
context: false,
noQuery: 'published third-party intel placemarks; not analyst records',
},
{
id: 'flights',
track: true,