Add adjustable event age filtering to S2 GhostMaps

This commit is contained in:
Dylan Maniatakes committed 2026-11-03 13:15:33 -05:00
1 parent acc9531b3f
commit 36fceb5675
17 files changed
+820 -117

No files matched your search

+1
View File
@@ -5663,6 +5663,7 @@ request viewbox and continues searching without that geographic preference.
- **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked.
- **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere.
- **Event age filter.** The same filter applies to every selected S2 product, including Border Crisis, Iran Picture, Iran Kinetic, Yemen FLOT and Latest Drop. S2 products default to seven days, with row buttons for 1, 3, 7, 14 or 30 days and **All markers**. The choice persists independently of product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage uses seven days. Only the placemark's `Date of Event` (or `EventDate`/`Date`) determines its age, never report dates, dates in prose or KMZ edition/upload dates. Undated, blank or unparseable event dates and future dates stay visible. ArcGIS `M/D/YYYY` dates and optional AM/PM times, plus ISO dates/timestamps, are supported; values without a zone use UTC and date-only values use midnight UTC. The exact cutoff is included. Events age out every minute and during catalog updates, even without a new file or a successful catalog fetch. Widening the window or choosing All markers restores omitted events from the cached KMZ. Removed entities and selected cards disappear together; surviving pick ids stay stable. Product defaults and the update stream are unchanged.
- **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub.
- It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min.
- Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit.
+70 -6
View File
@@ -28,6 +28,8 @@
* (vii) FAR SIDE — pins skip the depth test, so they are horizon-culled:
* seen from over the Indian Ocean no pin clearly behind the planet is
* shown; back over North America the near-side pins show again.
* (viii) AGE — seven days is the default; shorter/longer windows and All
* markers update the globe, and the age choice survives a reload.
*
* Screenshots go to qa-shots/ghostmaps-*.png (gitignored).
*
@@ -201,12 +203,15 @@ async function main() {
);
record(
'LIVE: one chip per product, defaults pressed',
chips.length === catalog.products.length &&
chips.every(
(chip) =>
chip.active ===
catalog.products.find((p) => p.id === chip.id)?.defaultOn,
),
chips.filter((chip) => !chip.id.startsWith('age-')).length ===
catalog.products.length &&
chips
.filter((chip) => !chip.id.startsWith('age-'))
.every(
(chip) =>
chip.active ===
catalog.products.find((p) => p.id === chip.id)?.defaultOn,
),
chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '),
);
const stream = await page
@@ -224,6 +229,65 @@ async function main() {
record('LIVE: the /api/ghostmaps/events stream is open', stream);
await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') });
const ageChips = chips.filter((chip) => chip.id.startsWith('age-'));
record(
'AGE: seven days is the default with an All markers option',
live.ageDays === 7 &&
ageChips.length === 6 &&
ageChips
.filter((chip) => chip.active)
.map((chip) => chip.id)
.join() === 'age-7' &&
ageChips.some((chip) => chip.id === 'age-all'),
);
const selectAge = async (days) => {
await showRow(page);
const selector = `[data-layer-id="${LAYER_ID}"] [data-chip-id="age-${days ?? 'all'}"]`;
await page.waitForSelector(selector, { timeout: 15000 });
// Dispatch through the rendered button, matching the product-chip gate.
await page.$eval(selector, (button) => button.click());
await page.waitForFunction(
(id, age) => {
const s = window.__godsEyeView.dataManager.layers
.get(id)
.module.getStats();
return s.ageDays === age && !s.loading;
},
{ timeout: TIMEOUT_MS },
LAYER_ID,
days,
);
return stats(page);
};
const all = await selectAge(null);
record(
'AGE: All markers restores old events without duplicate data sources',
all.count > live.count && (await sources(page)).length === 2,
`${live.count} → ${all.count}`,
);
await page.reload({ waitUntil: 'domcontentloaded' });
await page.waitForFunction(() => window.__godsEyeView?.dataManager, {
timeout: TIMEOUT_MS,
});
await enableAndSettle(page);
const persistedAge = await stats(page);
record(
'AGE: All markers survives a reload',
persistedAge.ageDays === null && persistedAge.count === all.count,
);
const thirty = await selectAge(30);
const one = await selectAge(1);
const seven = await selectAge(7);
record(
'AGE: shorter and longer windows change the count and seven days restores the default',
one.count <= live.count &&
thirty.count >= live.count &&
thirty.count <= all.count &&
seven.count === live.count &&
seven.ageDays === 7,
`1d=${one.count}, 7d=${seven.count}, 30d=${thirty.count}, all=${all.count}`,
);
// ---- (ii) PICK ----------------------------------------------------------
const target = await page.evaluate((id) => {
const gev = window.__godsEyeView;
+6 -1
View File
@@ -3306,5 +3306,10 @@
"counted over the bounding box of {area}, not its outline": "عُدّ ضمن المستطيل المحيط ({area})، لا ضمن الحدود",
"Search cancelled": "أُلغي البحث",
"Sea Marks": "علامات بحرية",
"Ski Pistes": "مسارات التزلج"
"Ski Pistes": "مسارات التزلج",
"{days}d": "{days}ي",
"All markers": "كل العلامات",
"Show markers of every age": "عرض العلامات من جميع الأعمار",
"Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة",
"Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ"
}
+6 -1
View File
@@ -3006,5 +3006,10 @@
"counted over the bounding box of {area}, not its outline": "über das umgebende Rechteck gezählt ({area}), nicht über den Umriss",
"Search cancelled": "Suche abgebrochen",
"Sea Marks": "Seezeichen",
"Ski Pistes": "Skipisten"
"Ski Pistes": "Skipisten",
"{days}d": "{days}T",
"All markers": "Alle Markierungen",
"Show markers of every age": "Markierungen jeden Alters anzeigen",
"Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben",
"Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben"
}
+6 -1
View File
@@ -3081,5 +3081,10 @@
"counted over the bounding box of {area}, not its outline": "contado en el rectángulo que lo rodea ({area}), no en su contorno",
"Search cancelled": "Búsqueda cancelada",
"Sea Marks": "Señales marítimas",
"Ski Pistes": "Pistas de esquí"
"Ski Pistes": "Pistas de esquí",
"{days}d": "{days}d",
"All markers": "Todos los marcadores",
"Show markers of every age": "Mostrar marcadores de cualquier antigüedad",
"Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha",
"Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha"
}
+6 -1
View File
@@ -3081,5 +3081,10 @@
"counted over the bounding box of {area}, not its outline": "compté sur le rectangle englobant ({area}), pas sur le contour",
"Search cancelled": "Recherche annulée",
"Sea Marks": "Balisage maritime",
"Ski Pistes": "Pistes de ski"
"Ski Pistes": "Pistes de ski",
"{days}d": "{days}j",
"All markers": "Tous les marqueurs",
"Show markers of every age": "Afficher les marqueurs de tout âge",
"Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés",
"Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés"
}
+6 -1
View File
@@ -3081,5 +3081,10 @@
"counted over the bounding box of {area}, not its outline": "contato sul riquadro di delimitazione ({area}), non sul contorno",
"Search cancelled": "Ricerca annullata",
"Sea Marks": "Segnalamenti marittimi",
"Ski Pistes": "Piste da sci"
"Ski Pistes": "Piste da sci",
"{days}d": "{days}g",
"All markers": "Tutti gli indicatori",
"Show markers of every age": "Mostra gli indicatori di qualsiasi età",
"Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti",
"Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti"
}
+6 -1
View File
@@ -2931,5 +2931,10 @@
"counted over the bounding box of {area}, not its outline": "外接矩形 ({area}) で数えた結果で、輪郭ではありません",
"Search cancelled": "検索をキャンセルしました",
"Sea Marks": "航路標識",
"Ski Pistes": "スキーゲレンデ"
"Ski Pistes": "スキーゲレンデ",
"{days}d": "{days}日",
"All markers": "すべてのマーカー",
"Show markers of every age": "日付にかかわらずすべてのマーカーを表示",
"Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数:{days}日。日付のないマーカーは保持",
"Event age: {days}d · undated kept": "経過日数:{days}日以内 · 日付なしは保持"
}
+6 -1
View File
@@ -3081,5 +3081,10 @@
"counted over the bounding box of {area}, not its outline": "contado na caixa delimitadora ({area}), não no contorno",
"Search cancelled": "Busca cancelada",
"Sea Marks": "Sinais marítimos",
"Ski Pistes": "Pistas de esqui"
"Ski Pistes": "Pistas de esqui",
"{days}d": "{days}d",
"All markers": "Todos os marcadores",
"Show markers of every age": "Mostrar marcadores de qualquer idade",
"Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos",
"Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos"
}
+6 -1
View File
@@ -3156,5 +3156,10 @@
"counted over the bounding box of {area}, not its outline": "подсчитано по ограничивающему прямоугольнику ({area}), а не по контуру",
"Search cancelled": "Поиск отменён",
"Sea Marks": "Морские знаки",
"Ski Pistes": "Горнолыжные трассы"
"Ski Pistes": "Горнолыжные трассы",
"{days}d": "{days}д",
"All markers": "Все метки",
"Show markers of every age": "Показать метки любого возраста",
"Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены",
"Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены"
}
+6 -1
View File
@@ -3156,5 +3156,10 @@
"counted over the bounding box of {area}, not its outline": "підраховано за обмежувальним прямокутником ({area}), а не за контуром",
"Search cancelled": "Пошук скасовано",
"Sea Marks": "Морські знаки",
"Ski Pistes": "Гірськолижні траси"
"Ski Pistes": "Гірськолижні траси",
"{days}d": "{days}д",
"All markers": "Усі позначки",
"Show markers of every age": "Показати позначки будь-якого віку",
"Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено",
"Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено"
}
+6 -1
View File
@@ -2931,5 +2931,10 @@
"counted over the bounding box of {area}, not its outline": "按外接矩形({area})计数,而非其轮廓",
"Search cancelled": "搜索已取消",
"Sea Marks": "航标",
"Ski Pistes": "滑雪道"
"Ski Pistes": "滑雪道",
"{days}d": "{days}天",
"All markers": "所有标记",
"Show markers of every age": "显示所有时间的标记",
"Maximum event age: {days}d; undated markers are kept": "事件最长保留时间:{days}天;保留无日期标记",
"Event age: {days}d · undated kept": "事件时间:{days}天内 · 保留无日期标记"
}
+6 -1
View File
@@ -2931,5 +2931,10 @@
"counted over the bounding box of {area}, not its outline": "按外接矩形({area})計數,而非其輪廓",
"Search cancelled": "搜索已取消",
"Sea Marks": "航標",
"Ski Pistes": "滑雪道"
"Ski Pistes": "滑雪道",
"{days}d": "{days}天",
"All markers": "所有標記",
"Show markers of every age": "顯示所有時間的標記",
"Maximum event age: {days}d; undated markers are kept": "事件最長保留時間:{days}天;保留無日期標記",
"Event age: {days}d · undated kept": "事件時間:{days}天內 · 保留無日期標記"
}
+191 -80
View File
@@ -16,7 +16,15 @@ import {
} from '../../i18n/index.js';
import { createHorizonCull } from '../streetLevel/view.js';
import { readGhostMapsKml } from './kml.js';
import { fieldLabel, parseDescriptionFields, wrapCardText } from './records.js';
import {
fieldLabel,
GHOSTMAPS_AGE_OPTIONS,
GHOSTMAPS_DEFAULT_AGE_DAYS,
ghostMapsEventDate,
isGhostMapsEventExpired,
parseDescriptionFields,
wrapCardText,
} from './records.js';
export * from './catalog.js';
export * from './records.js';
export { readGhostMapsKml } from './kml.js';
@@ -26,6 +34,7 @@ export const GHOSTMAPS_LAYER_ID = 's2-ghostmaps';
export const GHOSTMAPS_OVERLAY_SOURCE_ID = 's2-ghostmaps';
const PICK_PREFIX = 'ghostmaps:';
const STORAGE_KEY = 'gev.ghostmaps.products';
const AGE_STORAGE_KEY = 'gev.ghostmaps.ageDays';
const ACCENT = '#c8ff00';
const MAX_ICON_BYTES = 512 * 1024;
const CARD_HOST_OPTIONS = Object.freeze({
@@ -82,6 +91,19 @@ function writeStoredSelection(storage, selected) {
}
}
function readStoredAge(storage) {
try {
const raw = storage?.getItem(AGE_STORAGE_KEY);
if (raw !== null && raw !== undefined) {
const value = JSON.parse(raw);
if (GHOSTMAPS_AGE_OPTIONS.includes(value)) return value;
}
} catch {
/* unavailable or invalid storage uses the default */
}
return GHOSTMAPS_DEFAULT_AGE_DAYS;
}
/** A round marker for placemarks whose icon is missing or remote. */
const markerCache = new Map();
function markerImage(color) {
@@ -137,6 +159,7 @@ export function createGhostMapsLayer({
pointer = null,
openExternal = null,
storage = defaultStorage(),
now = () => Date.now(),
} = {}) {
if (typeof source?.getCatalog !== 'function')
throw new TypeError('GhostMaps layer requires a catalog source');
@@ -154,7 +177,9 @@ export function createGhostMapsLayer({
let _selectedLink = null;
let _lifetime = new AbortController();
let _selection = readStoredSelection(storage);
/** productId → {version, dataSource, urls, count, features} */
let _ageDays = readStoredAge(storage);
let _expiryTimer = null;
/** productId → {version, dataSource, urls, features: Map<index, record>, pins} */
const _loaded = new Map();
/** productId → in-flight version */
const _loading = new Map();
@@ -192,8 +217,31 @@ export function createGhostMapsLayer({
if (_selectedKey?.startsWith(`${id}:`)) clearSelection();
}
// Remove expired entities, rather than hiding them: horizon culling must
// never make an old pin visible again. Map keys keep surviving picks stable.
function expireEvents() {
const at = now();
let changed = false;
for (const [productId, item] of _loaded) {
for (const [index, feature] of item.features) {
if (!isGhostMapsEventExpired(feature.dateMs, at, _ageDays)) continue;
for (const entity of feature.entities)
item.dataSource.entities.remove(entity);
item.features.delete(index);
if (_selectedKey === `${productId}:${index}`) clearSelection();
changed = true;
}
item.pins = item.pins.filter((pin) => item.features.has(pin.index));
}
if (changed) {
notifyRow();
render();
}
}
/** Build one product's data source off-scene, then swap it in. */
async function load(product, signal) {
const ageDays = _ageDays;
const bytes = await source.getFile(product.id, product.version, { signal });
signal.throwIfAborted();
const entries = await readZipEntries(bytes);
@@ -223,12 +271,16 @@ export function createGhostMapsLayer({
const dataSource = new Cesium.CustomDataSource(
`${GHOSTMAPS_LAYER_ID}:${product.id}`,
);
const features = [];
const features = new Map();
const pins = [];
const at = now();
dataSource.entities.suspendEvents();
parsed.features.forEach((feature, index) => {
const dateMs = ghostMapsEventDate(feature.description);
if (isGhostMapsEventExpired(dateMs, at, ageDays)) return;
const key = `${product.id}:${index}`;
const { style } = feature;
const entities = [];
feature.geometries.forEach((geometry, part) => {
const id = `${PICK_PREFIX}${key}:${part}`;
if (geometry.type === 'point') {
@@ -255,7 +307,9 @@ export function createGhostMapsLayer({
disableDepthTestDistance: Number.POSITIVE_INFINITY,
},
});
entities.push(entity);
pins.push({
index,
position,
get show() {
return entity.show;
@@ -265,61 +319,76 @@ export function createGhostMapsLayer({
},
});
} else if (geometry.type === 'line') {
dataSource.entities.add({
id,
polyline: {
positions: Cesium.Cartesian3.fromDegreesArray(
geometry.positions.flat(),
),
width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)),
material: cesiumColor(style.lineColor, color),
clampToGround: true,
},
});
} else {
const ring = (points) =>
Cesium.Cartesian3.fromDegreesArray(points.flat());
dataSource.entities.add({
id,
polygon: {
hierarchy: new Cesium.PolygonHierarchy(
ring(geometry.outer),
geometry.holes.map(
(hole) => new Cesium.PolygonHierarchy(ring(hole)),
),
),
material:
style.fill === false
? Cesium.Color.TRANSPARENT
: cesiumColor(
style.polyColor,
color,
style.polyColor ? undefined : 0.25,
),
classificationType: Cesium.ClassificationType.BOTH,
},
});
if (style.outline !== false)
entities.push(
dataSource.entities.add({
id: `${id}:outline`,
id,
polyline: {
positions: ring([...geometry.outer, geometry.outer[0]]),
width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)),
positions: Cesium.Cartesian3.fromDegreesArray(
geometry.positions.flat(),
),
width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)),
material: cesiumColor(style.lineColor, color),
clampToGround: true,
},
});
}),
);
} else {
const ring = (points) =>
Cesium.Cartesian3.fromDegreesArray(points.flat());
entities.push(
dataSource.entities.add({
id,
polygon: {
hierarchy: new Cesium.PolygonHierarchy(
ring(geometry.outer),
geometry.holes.map(
(hole) => new Cesium.PolygonHierarchy(ring(hole)),
),
),
material:
style.fill === false
? Cesium.Color.TRANSPARENT
: cesiumColor(
style.polyColor,
color,
style.polyColor ? undefined : 0.25,
),
classificationType: Cesium.ClassificationType.BOTH,
},
}),
);
if (style.outline !== false)
entities.push(
dataSource.entities.add({
id: `${id}:outline`,
polyline: {
positions: ring([...geometry.outer, geometry.outer[0]]),
width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)),
material: cesiumColor(style.lineColor, color),
clampToGround: true,
},
}),
);
}
});
features.push({
features.set(index, {
name: feature.name,
folder: feature.folder,
description: feature.description,
dateMs,
entities,
anchor: anchorOf(feature.geometries[0]),
});
});
dataSource.entities.resumeEvents();
return { version: product.version, dataSource, urls, features, pins };
return {
version: product.version,
ageDays,
dataSource,
urls,
features,
pins,
};
} catch (error) {
urls.forEach((url) => URL.revokeObjectURL(url));
throw error;
@@ -330,6 +399,7 @@ export function createGhostMapsLayer({
function sync() {
const run = _queue.then(async () => {
if (!_enabled || !_viewer || !_catalog) return;
expireEvents();
const signal = _lifetime.signal;
const products = new Map(_catalog.products.map((p) => [p.id, p]));
for (const id of [..._loaded.keys()]) {
@@ -338,13 +408,15 @@ export function createGhostMapsLayer({
}
for (const product of _catalog.products) {
if (!wanted(product) || signal.aborted) continue;
if (_loaded.get(product.id)?.version === product.version) continue;
const loaded = _loaded.get(product.id);
if (loaded?.version === product.version && loaded.ageDays === _ageDays)
continue;
_loading.set(product.id, product.version);
_failed.delete(product.id);
notifyRow();
try {
const next = await load(product, signal);
if (signal.aborted || !_enabled) {
if (signal.aborted || !_enabled || next.ageDays !== _ageDays) {
next.urls.forEach((url) => URL.revokeObjectURL(url));
return;
}
@@ -391,12 +463,28 @@ export function createGhostMapsLayer({
void sync();
}
function setAgeDays(ageDays) {
if (!GHOSTMAPS_AGE_OPTIONS.includes(ageDays) || ageDays === _ageDays)
return;
_ageDays = ageDays;
try {
storage?.setItem(AGE_STORAGE_KEY, JSON.stringify(ageDays));
} catch {
/* private window or blocked storage: keep the session choice */
}
expireEvents();
notifyRow();
// A wider window must restore previously omitted placemarks from the
// cached KMZ, even though its upstream version has not changed.
return sync();
}
// ---- selection card ----------------------------------------------------
function featureFor(key) {
const [productId, index] = key.split(':');
return {
product: _catalog?.products.find((p) => p.id === productId) || null,
feature: _loaded.get(productId)?.features[Number(index)] || null,
feature: _loaded.get(productId)?.features.get(Number(index)) || null,
};
}
@@ -490,7 +578,7 @@ export function createGhostMapsLayer({
if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX))
return null;
const [productId, index] = pickId.slice(PICK_PREFIX.length).split(':');
return _loaded.get(productId)?.features[Number(index)]
return _loaded.get(productId)?.features.has(Number(index))
? `${productId}:${index}`
: null;
}
@@ -542,7 +630,7 @@ export function createGhostMapsLayer({
function featureCount() {
let count = 0;
for (const item of _loaded.values()) count += item.features.length;
for (const item of _loaded.values()) count += item.features.size;
return count;
}
@@ -563,6 +651,8 @@ export function createGhostMapsLayer({
enable() {
_enabled = true;
_lifetime = new AbortController();
if (_expiryTimer === null)
_expiryTimer = setInterval(expireEvents, 60_000);
for (const item of _loaded.values()) item.dataSource.show = true;
installClickHandler();
if (!_unsubscribe && typeof source.subscribe === 'function')
@@ -580,6 +670,8 @@ export function createGhostMapsLayer({
disable() {
_enabled = false;
clearInterval(_expiryTimer);
_expiryTimer = null;
_lifetime.abort();
_unsubscribe?.();
_unsubscribe = null;
@@ -593,6 +685,7 @@ export function createGhostMapsLayer({
async update(_viewerArg, { signal } = {}) {
if (!_enabled || signal?.aborted) return false;
expireEvents();
try {
const catalog = await source.getCatalog({ signal });
if (!_enabled) return false;
@@ -637,37 +730,54 @@ export function createGhostMapsLayer({
})
: t('Connecting to the S2 GhostMaps repository…');
return {
chips: products.map((product) => {
const loaded = _loaded.get(product.id);
const failed = _failed.get(product.id);
return {
id: product.id,
label: t(PRODUCT_LABELS[product.id] || product.title),
active: wanted(product),
busy: _loading.has(product.id),
state: failed ? 'error' : undefined,
color: PRODUCT_COLORS[product.id],
title: [
product.fileTitle,
loaded
? tn(
loaded.features.length,
'{count} placemark',
'{count} placemarks',
{
count: formatNumber(loaded.features.length),
},
)
: null,
failed ? t(failed) : null,
]
.filter(Boolean)
.join(' · '),
onClick: () => toggleProduct(product.id),
};
}),
chips: [
...GHOSTMAPS_AGE_OPTIONS.map((days) => ({
id: `age-${days ?? 'all'}`,
label:
days === null
? t('All markers')
: t('{days}d', { days: formatNumber(days) }),
title:
days === null
? t('Show markers of every age')
: t('Maximum event age: {days}d; undated markers are kept', {
days: formatNumber(days),
}),
active: days === _ageDays,
onClick: () => setAgeDays(days),
})),
...products.map((product) => {
const loaded = _loaded.get(product.id);
const failed = _failed.get(product.id);
return {
id: product.id,
label: t(PRODUCT_LABELS[product.id] || product.title),
active: wanted(product),
busy: _loading.has(product.id),
state: failed ? 'error' : undefined,
color: PRODUCT_COLORS[product.id],
title: [
product.fileTitle,
loaded
? tn(
loaded.features.size,
'{count} placemark',
'{count} placemarks',
{
count: formatNumber(loaded.features.size),
},
)
: null,
failed ? t(failed) : null,
]
.filter(Boolean)
.join(' · '),
onClick: () => toggleProduct(product.id),
};
}),
],
legend: [],
info,
info: `${_ageDays === null ? t('All markers') : t('Event age: {days}d · undated kept', { days: formatNumber(_ageDays) })} · ${info}`,
infoTitle: t(
'S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.',
),
@@ -677,6 +787,7 @@ export function createGhostMapsLayer({
getStats() {
return {
count: featureCount(),
ageDays: _ageDays,
products: [..._loaded.keys()],
commit: _catalog?.commit || null,
lastUpdate: _catalog?.committedAt || _catalog?.changedAt || null,
@@ -693,7 +804,7 @@ export function createGhostMapsLayer({
if (!_enabled) return [];
const out = [];
for (const [productId, item] of _loaded) {
for (const feature of item.features) {
for (const feature of item.features.values()) {
if (out.length >= maxCount) return out;
out.push({
product: productId,
+308 -9
View File
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
import test from 'node:test';
import * as Cesium from 'cesium';
import { createGhostMapsLayer } from './index.js';
import { GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP } from './catalog.js';
function element(localName, children = [], value = '', attributes = {}) {
const node = {
@@ -22,7 +23,7 @@ function element(localName, children = [], value = '', attributes = {}) {
return node;
}
function documentFixture() {
function documentFixture(reports = [{ name: 'Test report' }]) {
return element('document', [
element('kml', [
element('Document', [
@@ -36,11 +37,14 @@ function documentFixture() {
'',
{ id: 'pin' },
),
element('Placemark', [
element('name', [], 'Test report'),
element('styleUrl', [], '#pin'),
element('Point', [element('coordinates', [], '12,34')]),
]),
...reports.map(({ name, description = '', geometry }) =>
element('Placemark', [
element('name', [], name),
element('description', [], description),
element('styleUrl', [], '#pin'),
geometry || element('Point', [element('coordinates', [], '12,34')]),
]),
),
]),
]),
]);
@@ -83,11 +87,21 @@ function fixtureKmz() {
return new Uint8Array(Buffer.concat([...local, directory, end]));
}
function fixture(t, { beforeFileReturns, onCreateUrl } = {}) {
function fixture(
t,
{
beforeFileReturns,
onCreateUrl,
reports,
now,
products,
controls = {},
} = {},
) {
const previous = Object.getOwnPropertyDescriptor(globalThis, 'DOMParser');
globalThis.DOMParser = class {
parseFromString() {
return documentFixture();
return documentFixture(reports);
}
};
t.after(() => {
@@ -119,9 +133,13 @@ function fixture(t, { beforeFileReturns, onCreateUrl } = {}) {
};
const layer = createGhostMapsLayer({
storage: null,
now,
...controls,
source: {
getCatalog: async () => ({
products: [{ id: 'cip', version: 'a'.repeat(40), defaultOn: true }],
products: products ?? [
{ id: 'cip', version: 'a'.repeat(40), defaultOn: true },
],
}),
getFile: async () => {
beforeFileReturns?.(layer);
@@ -178,3 +196,284 @@ test('successful load keeps icons until the product is disabled', async (t) => {
assert.deepEqual(revoked, created);
assert.equal(mounted.size, 0);
});
const dated = (name, value, label = 'Date') => ({
name,
description: `<table><tr><td>${label}</td><td>${value}</td></tr></table>`,
});
const ageChip = (layer, days) =>
layer.getRowControls().chips.find(({ id }) => id === `age-${days ?? 'all'}`);
test('age controls default to seven days and widening restores filtered and expired reports', async (t) => {
let at = Date.UTC(2026, 10, 3, 12);
const { layer, viewer, mounted } = fixture(t, {
now: () => at,
reports: [
dated('Old', '10/1/2026'),
dated('Ten days', '10/24/2026'),
dated('Recent', '11/2/2026'),
{ name: 'Undated' },
],
});
await layer.update(viewer);
assert.equal(layer.getStats().ageDays, 7);
assert.equal(ageChip(layer, 7).active, true);
assert.equal(layer.getStats().count, 2);
await ageChip(layer, 14).onClick();
assert.equal(layer.getStats().count, 3);
await ageChip(layer, null).onClick();
assert.equal(layer.getStats().count, 4);
at += 100 * 86400_000;
await layer.update(viewer);
assert.equal(layer.getStats().count, 4, 'all markers never ages out');
await ageChip(layer, 1).onClick();
assert.equal(layer.getStats().count, 1);
assert.equal(layer.getAnalystRecords()[0].name, 'Undated');
await ageChip(layer, null).onClick();
assert.equal(layer.getStats().count, 4);
assert.equal(mounted.size, 1, 'filter changes never duplicate data sources');
const ageChips = layer
.getRowControls()
.chips.filter(({ id }) => id.startsWith('age-'));
assert.equal(ageChips.filter(({ active }) => active).length, 1);
assert.equal(ageChip(layer, null).active, true);
});
test('age preferences persist independently of product choices and reject invalid storage', async (t) => {
const values = new Map([['gev.ghostmaps.products', '["cip"]']]);
const storage = {
getItem: (key) => values.get(key) ?? null,
setItem: (key, value) => values.set(key, value),
};
await t.test('save All markers without changing products', async (t) => {
const { layer, viewer } = fixture(t, { controls: { storage } });
await layer.update(viewer);
await ageChip(layer, null).onClick();
assert.equal(values.get('gev.ghostmaps.ageDays'), 'null');
assert.equal(values.get('gev.ghostmaps.products'), '["cip"]');
});
await t.test('restore All markers and save thirty days', async (t) => {
const { layer } = fixture(t, { controls: { storage } });
assert.equal(layer.getStats().ageDays, null);
await ageChip(layer, 30).onClick();
assert.equal(values.get('gev.ghostmaps.ageDays'), '30');
});
await t.test('restore thirty days', (t) => {
const { layer } = fixture(t, { controls: { storage } });
assert.equal(layer.getStats().ageDays, 30);
});
for (const value of ['garbage', '"7"', '-1', '0', '2', '{}']) {
await t.test(`reject ${value}`, (t) => {
values.set('gev.ghostmaps.ageDays', value);
const { layer } = fixture(t, { controls: { storage } });
assert.equal(layer.getStats().ageDays, 7);
});
}
});
test('blocked storage still permits changing the age within the session', async (t) => {
const storage = {
getItem() {
throw new Error('blocked');
},
setItem() {
throw new Error('blocked');
},
};
const { layer, viewer } = fixture(t, { controls: { storage } });
await layer.update(viewer);
assert.equal(layer.getStats().ageDays, 7);
await ageChip(layer, null).onClick();
assert.equal(layer.getStats().ageDays, null);
});
test('a filter changed during a pending load never installs the superseded filter', async (t) => {
let changed = false;
const { layer, viewer, mounted, revoked, created } = fixture(t, {
now: () => Date.UTC(2026, 10, 3),
reports: [dated('Old', '10/1/2026')],
beforeFileReturns: (layer) => {
if (changed) return;
changed = true;
ageChip(layer, null).onClick();
},
});
await layer.update(viewer);
await layer.update(viewer);
assert.equal(layer.getStats().count, 1);
assert.equal(layer.getStats().ageDays, null);
assert.equal(mounted.size, 1);
assert.equal(
revoked.length,
created.length - 1,
'discarded load releases icons',
);
test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => {
for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) {
await t.test(product.id, async (t) => {
const { layer, viewer, mounted } = fixture(t, {
now: () => Date.UTC(2026, 10, 3, 12),
products: [{ ...product, version: 'a'.repeat(40) }],
reports: [
dated(
'Old event',
'10/1/2026',
product.id === 'cip' ? 'Date of Event' : 'Date',
),
dated('Recent event', '11/2/2026'),
{ name: 'Undated reference feature' },
],
});
await layer.update(viewer);
if (!product.defaultOn) {
layer
.getRowControls()
.chips.find(({ id }) => id === product.id)
.onClick();
await layer.update(viewer);
}
assert.deepEqual(layer.getStats().products, [product.id]);
assert.equal(layer.getStats().count, 2);
assert.deepEqual(
layer.getAnalystRecords().map(({ name }) => name),
['Recent event', 'Undated reference feature'],
);
await ageChip(layer, null).onClick();
assert.equal(
layer.getStats().count,
3,
"All markers restores this product's old events",
);
await ageChip(layer, 7).onClick();
assert.equal(layer.getStats().count, 2);
assert.equal(mounted.size, 1);
});
}
});
});
test('default products filter old event dates while keeping recent, future and undated markers', async (t) => {
const { layer, viewer, mounted } = fixture(t, {
now: () => Date.UTC(2026, 10, 3, 12),
reports: [
dated('Old event', '10/20/2026'),
dated('Recent event', '11/2/2026'),
dated('At cutoff', '10/27/2026 12:00:00 PM'),
dated('Before cutoff', '10/27/2026 11:59:59 AM'),
dated('Future event', '11/4/2026'),
{ name: 'Undated site' },
dated('Blank date', '&lt;Null&gt;'),
dated('Invalid date', '2/30/2026'),
dated('Only a report date', '10/1/2026', 'Date Of Report'),
{
name: 'Old event reported recently',
description:
'<table><tr><td>Date of Event</td><td>10/1/2026</td></tr><tr><td>Date Of Report</td><td>11/2/2026</td></tr></table>',
},
],
});
await layer.update(viewer);
assert.deepEqual(layer.getStats().products, ['cip']);
assert.deepEqual(
layer.getAnalystRecords().map((r) => r.name),
[
'Recent event',
'At cutoff',
'Future event',
'Undated site',
'Blank date',
'Invalid date',
'Only a report date',
],
);
assert.equal(layer.getStats().count, 7);
assert.equal([...mounted][0].entities.values.length, 7);
// Filtering cannot shift pick ids onto a different report.
assert.ok([...mounted][0].entities.getById('ghostmaps:cip:1:0'));
assert.equal(
[...mounted][0].entities.getById('ghostmaps:cip:0:0'),
undefined,
);
});
test('unchanged cached products age out on the timer, clear selected cards and keep surviving picks', async (t) => {
t.mock.timers.enable({ apis: ['setInterval'] });
let at = Date.UTC(2026, 10, 3, 12);
let click;
const cards = [];
const { layer, viewer, mounted } = fixture(t, {
now: () => at,
reports: [
dated('Expiring', '10/27/2026 12:00:00 PM'),
dated('Surviving', '11/2/2026'),
],
controls: {
overlayHost: { setEntries: (_id, entries) => cards.push(entries) },
screenSpaceEventHandlerFactory: () => ({
setInputAction: (fn) => {
click = fn;
},
destroy() {},
}),
picking: { resolvePickId: (picked) => picked },
},
});
await layer.update(viewer);
const ds = [...mounted][0];
viewer.scene.pick = () => 'ghostmaps:cip:0:0';
click({ position: { x: 0, y: 0 } });
assert.equal(cards.at(-1).length, 1);
at += 60_000;
t.mock.timers.tick(60_000);
assert.equal(layer.getStats().count, 1);
assert.equal(ds.entities.values.length, 1);
assert.equal(ds.entities.getById('ghostmaps:cip:0:0'), undefined);
assert.ok(ds.entities.getById('ghostmaps:cip:1:0'));
assert.deepEqual(cards.at(-1), []);
await layer.update(viewer);
assert.equal([...mounted][0], ds, 'same-version KMZ is not reloaded');
viewer.scene.pick = () => 'ghostmaps:cip:1:0';
click({ position: { x: 0, y: 0 } });
assert.match(cards.at(-1)[0].title, /Surviving/);
layer.disable();
at += 14 * 86400_000;
t.mock.timers.tick(60_000);
assert.equal(mounted.size, 0);
layer.enable();
await layer.update(viewer);
assert.equal(layer.getStats().count, 0);
});
test('expiry removes every geometry and polygon outline belonging to an old report', async (t) => {
t.mock.timers.enable({ apis: ['setInterval'] });
let at = Date.UTC(2026, 10, 3);
const polygon = element('Polygon', [
element('outerBoundaryIs', [
element('LinearRing', [
element('coordinates', [], '12,34 13,34 13,35 12,34'),
]),
]),
]);
const { layer, viewer, mounted } = fixture(t, {
now: () => at,
reports: [
{
...dated('Multi-part', '10/27/2026'),
geometry: element('MultiGeometry', [
element('Point', [element('coordinates', [], '12,34')]),
element('LineString', [element('coordinates', [], '12,34 13,35')]),
polygon,
]),
},
],
});
await layer.update(viewer);
const ds = [...mounted][0];
assert.equal(ds.entities.values.length, 4);
at += 60_000;
t.mock.timers.tick(60_000);
assert.equal(ds.entities.values.length, 0);
assert.equal(layer.getStats().count, 0);
});
+101 -11
View File
@@ -1,4 +1,4 @@
// Pure helpers for GhostMaps placemarks: description tables to plain fields,
// Pure helpers for GhostMaps placemarks: report fields and event dates,
// KML colors, and card text wrapping. Strings in, data out — description
// markup is only ever read as text, never inserted into the page.
@@ -51,6 +51,104 @@ const MAX_FIELDS = 24;
const MAX_VALUE = 600;
const MAX_LINKS = 6;
// Use the same leaf rows for report cards and event dates, without the card's
// field-count/value-length limits affecting the age filter.
function* descriptionRows(source) {
for (const row of source.matchAll(
/<tr\b[^>]*>((?:(?!<tr\b)[\s\S])*?)<\/tr\s*>/gi,
)) {
const cells = [
...row[1].matchAll(/<t[dh]\b[^>]*>([\s\S]*?)<\/t[dh]\s*>/gi),
].map((cell) => cell[1]);
if (cells.length === 2)
yield {
label: htmlToText(cells[0]),
value: htmlToText(cells[1]),
markup: cells[1],
};
}
}
export const GHOSTMAPS_DEFAULT_AGE_DAYS = 7;
export const GHOSTMAPS_AGE_OPTIONS = Object.freeze([1, 3, 7, 14, 30, null]);
/** ArcGIS US dates (optionally with a time) and ISO dates, interpreted in UTC
* when no zone is supplied. Reject calendar rollovers instead of guessing. */
export function parseGhostMapsEventDate(value) {
const input = String(value ?? '').trim();
const us =
/^(\d{1,2})\/(\d{1,2})\/(\d{4})(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM))?$/i.exec(
input,
);
const iso =
/^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:\d{2})?)?$/.exec(
input,
);
if (!us && !iso) return null;
const year = Number(us ? us[3] : iso[1]);
const month = Number(us ? us[1] : iso[2]);
const day = Number(us ? us[2] : iso[3]);
let hour = Number(us?.[4] ?? iso?.[4] ?? 0);
const minute = Number(us?.[5] ?? iso?.[5] ?? 0);
const second = Number(us?.[6] ?? iso?.[6] ?? 0);
if (us?.[7]) {
if (hour < 1 || hour > 12) return null;
hour = (hour % 12) + (us[7].toUpperCase() === 'PM' ? 12 : 0);
}
const at = Date.UTC(year, month - 1, day, hour, minute, second);
const date = new Date(at);
if (
year < 1900 ||
month < 1 ||
month > 12 ||
day < 1 ||
date.getUTCDate() !== day ||
hour > 23 ||
minute > 59 ||
second > 59
)
return null;
if (iso?.[4]) {
const zoned = /(?:Z|[+-]\d{2}:\d{2})$/.test(input) ? input : `${input}Z`;
const parsed = Date.parse(zoned);
return Number.isFinite(parsed) ? parsed : null;
}
return at;
}
/** The event's date, never its report/upload date or dates mentioned in prose. */
export function ghostMapsEventDate(description) {
const rows = [
...descriptionRows(
String(description ?? '').replace(
/<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi,
' ',
),
),
];
for (const name of ['dateofevent', 'eventdate', 'date']) {
for (const { label, value } of rows) {
if (label.toLowerCase().replace(/[\s_]+/g, '') !== name) continue;
const at = parseGhostMapsEventDate(value);
if (at !== null) return at;
}
}
return null;
}
/** Unknown and future dates stay visible; only a confirmed old event expires. */
export function isGhostMapsEventExpired(
dateMs,
now = Date.now(),
ageDays = GHOSTMAPS_DEFAULT_AGE_DAYS,
) {
return (
ageDays !== null &&
Number.isFinite(dateMs) &&
dateMs < now - ageDays * 86400_000
);
}
/**
* Fields and links from a placemark description. ArcGIS "popup" exports are a
* two-column table (label, value); anything else becomes one unlabelled field.
@@ -73,18 +171,10 @@ export function parseDescriptionFields(html) {
};
// Innermost rows only: ArcGIS nests the field table inside an outer row,
// and only leaf rows with exactly two cells are label/value pairs.
for (const row of source.matchAll(
/<tr\b[^>]*>((?:(?!<tr\b)[\s\S])*?)<\/tr\s*>/gi,
)) {
const cells = [
...row[1].matchAll(/<t[dh]\b[^>]*>([\s\S]*?)<\/t[dh]\s*>/gi),
].map((cell) => cell[1]);
if (cells.length !== 2) continue;
const label = htmlToText(cells[0]);
const value = htmlToText(cells[1]);
for (const { label, value, markup } of descriptionRows(source)) {
if (!label || SKIPPED_LABELS.has(label.toLowerCase())) continue;
if (EMPTY_VALUES.has(value.toLowerCase())) continue;
addLinks(cells[1]);
addLinks(markup);
// A bare research link is carried by `links`; the card does not need
// the URL twice.
if (/^https?:\/\/\S+$/i.test(value)) continue;
+83
View File
@@ -7,8 +7,83 @@ import {
kmlColor,
parseDescriptionFields,
wrapCardText,
ghostMapsEventDate,
isGhostMapsEventExpired,
parseGhostMapsEventDate,
} from './records.js';
test('event dates parse ArcGIS calendar dates and timestamps consistently in UTC', () => {
for (const [value, iso] of [
['10/28/2026', '2026-10-28T00:00:00Z'],
['10/28/2026 6:12:00 PM', '2026-10-28T18:12:00Z'],
['1/17/2026 9:46:41 AM', '2026-01-17T09:46:41Z'],
['10/28/2026 12:00 AM', '2026-10-28T00:00:00Z'],
['10/28/2026 12:00 PM', '2026-10-28T12:00:00Z'],
['2026-10-28', '2026-10-28T00:00:00Z'],
['2026-10-28T12:34:56', '2026-10-28T12:34:56Z'],
['2026-10-28T12:34:56.123-04:00', '2026-10-28T16:34:56.123Z'],
['2/29/2024', '2024-02-29T00:00:00Z'],
])
assert.equal(parseGhostMapsEventDate(value), Date.parse(iso), value);
for (const value of [
'',
'<Null>',
'N/A',
'unknown',
'2/29/2026',
'9/31/2026',
'2026-02-30',
'13/1/2026',
'1/0/2026',
'10/28/2026 13:00 PM',
'2026-10-28T24:00:00Z',
'2026-10-28T12:60:00Z',
]) {
assert.equal(parseGhostMapsEventDate(value), null, value);
}
});
test('event date fields take precedence over report dates and prose, even after long report tables', () => {
const row = (label, value) => `<tr><td>${label}</td><td>${value}</td></tr>`;
const html = `<table><tr><td><table>${
row('Date Of Report', '11/2/2026') +
row('Description', 'Incident mentioned on 11/1/2026') +
Array.from({ length: 30 }, (_, i) => row(`Field${i}`, 'value')).join('') +
row('Date', '10/30/2026') +
row('Date of Event', '10/28/2026')
}</table></td></tr></table>`;
assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 28));
assert.equal(
ghostMapsEventDate(
`<table>${row('Date', '&lt;Null&gt;')}${row('Date Of Report', '11/2/2026')}</table>`,
),
null,
);
assert.equal(
ghostMapsEventDate(`<table>${row('Date_Of_Event', '10/28/2026')}</table>`),
Date.UTC(2026, 9, 28),
);
assert.equal(ghostMapsEventDate('Occurred 10/28/2026'), null);
assert.equal(
ghostMapsEventDate(
`<script><table>${row('Date', '1/1/2020')}</table></script>`,
),
null,
);
});
test('seven-day expiry keeps the exact boundary, future dates and unknown dates', () => {
const at = Date.UTC(2026, 10, 3, 12);
assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12), at), false);
assert.equal(
isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12) - 1, at),
true,
);
assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 10, 4), at), false);
for (const value of [null, undefined, NaN])
assert.equal(isGhostMapsEventExpired(value, at), false);
});
// The shape ArcGIS Pro writes into every S2 placemark description.
const ARCGIS = `<html xmlns:fo="http://www.w3.org/1999/XSL/Format"><head>
<META http-equiv="Content-Type" content="text/html"></head>
@@ -27,6 +102,14 @@ const ARCGIS = `<html xmlns:fo="http://www.w3.org/1999/XSL/Format"><head>
function changeImage(a, b) { document.getElementById('x').innerHTML = b; }
</script></html>`;
test('custom ages change the cutoff and all markers disables expiry', () => {
const at = Date.UTC(2026, 10, 3, 12);
const event = at - 10 * 86400_000;
assert.equal(isGhostMapsEventExpired(event, at, 7), true);
assert.equal(isGhostMapsEventExpired(event, at, 14), false);
assert.equal(isGhostMapsEventExpired(event, at, null), false);
});
test('ArcGIS popup tables become plain label/value fields and http links', () => {
const { fields, links } = parseDescriptionFields(ARCGIS);
assert.deepEqual(fields, [