From 36fceb56752297c6fb567e6766437bfbf9108e02 Mon Sep 17 00:00:00 2001 From: Dylan Maniatakes Date: Tue, 3 Nov 2026 11:35:13 -0500 Subject: [PATCH 1/4] Add adjustable event age filtering to S2 GhostMaps --- docs/CURRENT-STATE.md | 1 + scripts/qa-ghostmaps.mjs | 76 +++++- src/i18n/locales/ar.json | 7 +- src/i18n/locales/de.json | 7 +- src/i18n/locales/es.json | 7 +- src/i18n/locales/fr.json | 7 +- src/i18n/locales/it.json | 7 +- src/i18n/locales/ja.json | 7 +- src/i18n/locales/pt.json | 7 +- src/i18n/locales/ru.json | 7 +- src/i18n/locales/uk.json | 7 +- src/i18n/locales/zh-CN.json | 7 +- src/i18n/locales/zh-Hant.json | 7 +- src/layers/ghostMaps/index.js | 271 ++++++++++++++------ src/layers/ghostMaps/lifecycle.test.mjs | 317 +++++++++++++++++++++++- src/layers/ghostMaps/records.js | 112 ++++++++- src/layers/ghostMaps/records.test.mjs | 83 +++++++ 17 files changed, 820 insertions(+), 117 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 59557e46b..827035fe0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5663,6 +5663,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. +- **Event age filter.** The same filter applies to every selected S2 product, including Border Crisis, Iran Picture, Iran Kinetic, Yemen FLOT and Latest Drop. S2 products default to seven days, with row buttons for 1, 3, 7, 14 or 30 days and **All markers**. The choice persists independently of product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage uses seven days. Only the placemark's `Date of Event` (or `EventDate`/`Date`) determines its age, never report dates, dates in prose or KMZ edition/upload dates. Undated, blank or unparseable event dates and future dates stay visible. ArcGIS `M/D/YYYY` dates and optional AM/PM times, plus ISO dates/timestamps, are supported; values without a zone use UTC and date-only values use midnight UTC. The exact cutoff is included. Events age out every minute and during catalog updates, even without a new file or a successful catalog fetch. Widening the window or choosing All markers restores omitted events from the cached KMZ. Removed entities and selected cards disappear together; surviving pick ids stay stable. Product defaults and the update stream are unchanged. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/scripts/qa-ghostmaps.mjs b/scripts/qa-ghostmaps.mjs index 83dc8ed97..24edcc60b 100644 --- a/scripts/qa-ghostmaps.mjs +++ b/scripts/qa-ghostmaps.mjs @@ -28,6 +28,8 @@ * (vii) FAR SIDE — pins skip the depth test, so they are horizon-culled: * seen from over the Indian Ocean no pin clearly behind the planet is * shown; back over North America the near-side pins show again. + * (viii) AGE — seven days is the default; shorter/longer windows and All + * markers update the globe, and the age choice survives a reload. * * Screenshots go to qa-shots/ghostmaps-*.png (gitignored). * @@ -201,12 +203,15 @@ async function main() { ); record( 'LIVE: one chip per product, defaults pressed', - chips.length === catalog.products.length && - chips.every( - (chip) => - chip.active === - catalog.products.find((p) => p.id === chip.id)?.defaultOn, - ), + chips.filter((chip) => !chip.id.startsWith('age-')).length === + catalog.products.length && + chips + .filter((chip) => !chip.id.startsWith('age-')) + .every( + (chip) => + chip.active === + catalog.products.find((p) => p.id === chip.id)?.defaultOn, + ), chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '), ); const stream = await page @@ -224,6 +229,65 @@ async function main() { record('LIVE: the /api/ghostmaps/events stream is open', stream); await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') }); + const ageChips = chips.filter((chip) => chip.id.startsWith('age-')); + record( + 'AGE: seven days is the default with an All markers option', + live.ageDays === 7 && + ageChips.length === 6 && + ageChips + .filter((chip) => chip.active) + .map((chip) => chip.id) + .join() === 'age-7' && + ageChips.some((chip) => chip.id === 'age-all'), + ); + const selectAge = async (days) => { + await showRow(page); + const selector = `[data-layer-id="${LAYER_ID}"] [data-chip-id="age-${days ?? 'all'}"]`; + await page.waitForSelector(selector, { timeout: 15000 }); + // Dispatch through the rendered button, matching the product-chip gate. + await page.$eval(selector, (button) => button.click()); + await page.waitForFunction( + (id, age) => { + const s = window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats(); + return s.ageDays === age && !s.loading; + }, + { timeout: TIMEOUT_MS }, + LAYER_ID, + days, + ); + return stats(page); + }; + const all = await selectAge(null); + record( + 'AGE: All markers restores old events without duplicate data sources', + all.count > live.count && (await sources(page)).length === 2, + `${live.count} → ${all.count}`, + ); + await page.reload({ waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__godsEyeView?.dataManager, { + timeout: TIMEOUT_MS, + }); + await enableAndSettle(page); + const persistedAge = await stats(page); + record( + 'AGE: All markers survives a reload', + persistedAge.ageDays === null && persistedAge.count === all.count, + ); + const thirty = await selectAge(30); + const one = await selectAge(1); + const seven = await selectAge(7); + record( + 'AGE: shorter and longer windows change the count and seven days restores the default', + one.count <= live.count && + thirty.count >= live.count && + thirty.count <= all.count && + seven.count === live.count && + seven.ageDays === 7, + `1d=${one.count}, 7d=${seven.count}, 30d=${thirty.count}, all=${all.count}`, + ); + // ---- (ii) PICK ---------------------------------------------------------- const target = await page.evaluate((id) => { const gev = window.__godsEyeView; diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index ca04bf306..345fbc0ab 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3306,5 +3306,10 @@ "counted over the bounding box of {area}, not its outline": "عُدّ ضمن المستطيل المحيط ({area})، لا ضمن الحدود", "Search cancelled": "أُلغي البحث", "Sea Marks": "علامات بحرية", - "Ski Pistes": "مسارات التزلج" + "Ski Pistes": "مسارات التزلج", + "{days}d": "{days}ي", + "All markers": "كل العلامات", + "Show markers of every age": "عرض العلامات من جميع الأعمار", + "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index f990eae81..b59916432 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3006,5 +3006,10 @@ "counted over the bounding box of {area}, not its outline": "über das umgebende Rechteck gezählt ({area}), nicht über den Umriss", "Search cancelled": "Suche abgebrochen", "Sea Marks": "Seezeichen", - "Ski Pistes": "Skipisten" + "Ski Pistes": "Skipisten", + "{days}d": "{days}T", + "All markers": "Alle Markierungen", + "Show markers of every age": "Markierungen jeden Alters anzeigen", + "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index fcdd09185..47768e808 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contado en el rectángulo que lo rodea ({area}), no en su contorno", "Search cancelled": "Búsqueda cancelada", "Sea Marks": "Señales marítimas", - "Ski Pistes": "Pistas de esquí" + "Ski Pistes": "Pistas de esquí", + "{days}d": "{days}d", + "All markers": "Todos los marcadores", + "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", + "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index fd5728596..103bf00a9 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "compté sur le rectangle englobant ({area}), pas sur le contour", "Search cancelled": "Recherche annulée", "Sea Marks": "Balisage maritime", - "Ski Pistes": "Pistes de ski" + "Ski Pistes": "Pistes de ski", + "{days}d": "{days}j", + "All markers": "Tous les marqueurs", + "Show markers of every age": "Afficher les marqueurs de tout âge", + "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index cd0e85571..0cb73f516 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contato sul riquadro di delimitazione ({area}), non sul contorno", "Search cancelled": "Ricerca annullata", "Sea Marks": "Segnalamenti marittimi", - "Ski Pistes": "Piste da sci" + "Ski Pistes": "Piste da sci", + "{days}d": "{days}g", + "All markers": "Tutti gli indicatori", + "Show markers of every age": "Mostra gli indicatori di qualsiasi età", + "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 9a38c8549..ff79dafb8 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "外接矩形 ({area}) で数えた結果で、輪郭ではありません", "Search cancelled": "検索をキャンセルしました", "Sea Marks": "航路標識", - "Ski Pistes": "スキーゲレンデ" + "Ski Pistes": "スキーゲレンデ", + "{days}d": "{days}日", + "All markers": "すべてのマーカー", + "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", + "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数：{days}日。日付のないマーカーは保持", + "Event age: {days}d · undated kept": "経過日数：{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index 54225789d..9ed99b438 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3081,5 +3081,10 @@ "counted over the bounding box of {area}, not its outline": "contado na caixa delimitadora ({area}), não no contorno", "Search cancelled": "Busca cancelada", "Sea Marks": "Sinais marítimos", - "Ski Pistes": "Pistas de esqui" + "Ski Pistes": "Pistas de esqui", + "{days}d": "{days}d", + "All markers": "Todos os marcadores", + "Show markers of every age": "Mostrar marcadores de qualquer idade", + "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index eb4ca4dad..333a3518d 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3156,5 +3156,10 @@ "counted over the bounding box of {area}, not its outline": "подсчитано по ограничивающему прямоугольнику ({area}), а не по контуру", "Search cancelled": "Поиск отменён", "Sea Marks": "Морские знаки", - "Ski Pistes": "Горнолыжные трассы" + "Ski Pistes": "Горнолыжные трассы", + "{days}d": "{days}д", + "All markers": "Все метки", + "Show markers of every age": "Показать метки любого возраста", + "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index 903db6586..a989f456f 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3156,5 +3156,10 @@ "counted over the bounding box of {area}, not its outline": "підраховано за обмежувальним прямокутником ({area}), а не за контуром", "Search cancelled": "Пошук скасовано", "Sea Marks": "Морські знаки", - "Ski Pistes": "Гірськолижні траси" + "Ski Pistes": "Гірськолижні траси", + "{days}d": "{days}д", + "All markers": "Усі позначки", + "Show markers of every age": "Показати позначки будь-якого віку", + "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index 78a62c593..ddd567e92 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "按外接矩形（{area}）计数，而非其轮廓", "Search cancelled": "搜索已取消", "Sea Marks": "航标", - "Ski Pistes": "滑雪道" + "Ski Pistes": "滑雪道", + "{days}d": "{days}天", + "All markers": "所有标记", + "Show markers of every age": "显示所有时间的标记", + "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间：{days}天；保留无日期标记", + "Event age: {days}d · undated kept": "事件时间：{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index e2a5b9cc4..f1a52b067 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -2931,5 +2931,10 @@ "counted over the bounding box of {area}, not its outline": "按外接矩形（{area}）計數，而非其輪廓", "Search cancelled": "搜索已取消", "Sea Marks": "航標", - "Ski Pistes": "滑雪道" + "Ski Pistes": "滑雪道", + "{days}d": "{days}天", + "All markers": "所有標記", + "Show markers of every age": "顯示所有時間的標記", + "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間：{days}天；保留無日期標記", + "Event age: {days}d · undated kept": "事件時間：{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 01d564d05..7899c26b5 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -16,7 +16,15 @@ import { } from '../../i18n/index.js'; import { createHorizonCull } from '../streetLevel/view.js'; import { readGhostMapsKml } from './kml.js'; -import { fieldLabel, parseDescriptionFields, wrapCardText } from './records.js'; +import { + fieldLabel, + GHOSTMAPS_AGE_OPTIONS, + GHOSTMAPS_DEFAULT_AGE_DAYS, + ghostMapsEventDate, + isGhostMapsEventExpired, + parseDescriptionFields, + wrapCardText, +} from './records.js'; export * from './catalog.js'; export * from './records.js'; export { readGhostMapsKml } from './kml.js'; @@ -26,6 +34,7 @@ export const GHOSTMAPS_LAYER_ID = 's2-ghostmaps'; export const GHOSTMAPS_OVERLAY_SOURCE_ID = 's2-ghostmaps'; const PICK_PREFIX = 'ghostmaps:'; const STORAGE_KEY = 'gev.ghostmaps.products'; +const AGE_STORAGE_KEY = 'gev.ghostmaps.ageDays'; const ACCENT = '#c8ff00'; const MAX_ICON_BYTES = 512 * 1024; const CARD_HOST_OPTIONS = Object.freeze({ @@ -82,6 +91,19 @@ function writeStoredSelection(storage, selected) { } } +function readStoredAge(storage) { + try { + const raw = storage?.getItem(AGE_STORAGE_KEY); + if (raw !== null && raw !== undefined) { + const value = JSON.parse(raw); + if (GHOSTMAPS_AGE_OPTIONS.includes(value)) return value; + } + } catch { + /* unavailable or invalid storage uses the default */ + } + return GHOSTMAPS_DEFAULT_AGE_DAYS; +} + /** A round marker for placemarks whose icon is missing or remote. */ const markerCache = new Map(); function markerImage(color) { @@ -137,6 +159,7 @@ export function createGhostMapsLayer({ pointer = null, openExternal = null, storage = defaultStorage(), + now = () => Date.now(), } = {}) { if (typeof source?.getCatalog !== 'function') throw new TypeError('GhostMaps layer requires a catalog source'); @@ -154,7 +177,9 @@ export function createGhostMapsLayer({ let _selectedLink = null; let _lifetime = new AbortController(); let _selection = readStoredSelection(storage); - /** productId → {version, dataSource, urls, count, features} */ + let _ageDays = readStoredAge(storage); + let _expiryTimer = null; + /** productId → {version, dataSource, urls, features: Map, pins} */ const _loaded = new Map(); /** productId → in-flight version */ const _loading = new Map(); @@ -192,8 +217,31 @@ export function createGhostMapsLayer({ if (_selectedKey?.startsWith(`${id}:`)) clearSelection(); } + // Remove expired entities, rather than hiding them: horizon culling must + // never make an old pin visible again. Map keys keep surviving picks stable. + function expireEvents() { + const at = now(); + let changed = false; + for (const [productId, item] of _loaded) { + for (const [index, feature] of item.features) { + if (!isGhostMapsEventExpired(feature.dateMs, at, _ageDays)) continue; + for (const entity of feature.entities) + item.dataSource.entities.remove(entity); + item.features.delete(index); + if (_selectedKey === `${productId}:${index}`) clearSelection(); + changed = true; + } + item.pins = item.pins.filter((pin) => item.features.has(pin.index)); + } + if (changed) { + notifyRow(); + render(); + } + } + /** Build one product's data source off-scene, then swap it in. */ async function load(product, signal) { + const ageDays = _ageDays; const bytes = await source.getFile(product.id, product.version, { signal }); signal.throwIfAborted(); const entries = await readZipEntries(bytes); @@ -223,12 +271,16 @@ export function createGhostMapsLayer({ const dataSource = new Cesium.CustomDataSource( `${GHOSTMAPS_LAYER_ID}:${product.id}`, ); - const features = []; + const features = new Map(); const pins = []; + const at = now(); dataSource.entities.suspendEvents(); parsed.features.forEach((feature, index) => { + const dateMs = ghostMapsEventDate(feature.description); + if (isGhostMapsEventExpired(dateMs, at, ageDays)) return; const key = `${product.id}:${index}`; const { style } = feature; + const entities = []; feature.geometries.forEach((geometry, part) => { const id = `${PICK_PREFIX}${key}:${part}`; if (geometry.type === 'point') { @@ -255,7 +307,9 @@ export function createGhostMapsLayer({ disableDepthTestDistance: Number.POSITIVE_INFINITY, }, }); + entities.push(entity); pins.push({ + index, position, get show() { return entity.show; @@ -265,61 +319,76 @@ export function createGhostMapsLayer({ }, }); } else if (geometry.type === 'line') { - dataSource.entities.add({ - id, - polyline: { - positions: Cesium.Cartesian3.fromDegreesArray( - geometry.positions.flat(), - ), - width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), - material: cesiumColor(style.lineColor, color), - clampToGround: true, - }, - }); - } else { - const ring = (points) => - Cesium.Cartesian3.fromDegreesArray(points.flat()); - dataSource.entities.add({ - id, - polygon: { - hierarchy: new Cesium.PolygonHierarchy( - ring(geometry.outer), - geometry.holes.map( - (hole) => new Cesium.PolygonHierarchy(ring(hole)), - ), - ), - material: - style.fill === false - ? Cesium.Color.TRANSPARENT - : cesiumColor( - style.polyColor, - color, - style.polyColor ? undefined : 0.25, - ), - classificationType: Cesium.ClassificationType.BOTH, - }, - }); - if (style.outline !== false) + entities.push( dataSource.entities.add({ - id: `${id}:outline`, + id, polyline: { - positions: ring([...geometry.outer, geometry.outer[0]]), - width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + positions: Cesium.Cartesian3.fromDegreesArray( + geometry.positions.flat(), + ), + width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), material: cesiumColor(style.lineColor, color), clampToGround: true, }, - }); + }), + ); + } else { + const ring = (points) => + Cesium.Cartesian3.fromDegreesArray(points.flat()); + entities.push( + dataSource.entities.add({ + id, + polygon: { + hierarchy: new Cesium.PolygonHierarchy( + ring(geometry.outer), + geometry.holes.map( + (hole) => new Cesium.PolygonHierarchy(ring(hole)), + ), + ), + material: + style.fill === false + ? Cesium.Color.TRANSPARENT + : cesiumColor( + style.polyColor, + color, + style.polyColor ? undefined : 0.25, + ), + classificationType: Cesium.ClassificationType.BOTH, + }, + }), + ); + if (style.outline !== false) + entities.push( + dataSource.entities.add({ + id: `${id}:outline`, + polyline: { + positions: ring([...geometry.outer, geometry.outer[0]]), + width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + material: cesiumColor(style.lineColor, color), + clampToGround: true, + }, + }), + ); } }); - features.push({ + features.set(index, { name: feature.name, folder: feature.folder, description: feature.description, + dateMs, + entities, anchor: anchorOf(feature.geometries[0]), }); }); dataSource.entities.resumeEvents(); - return { version: product.version, dataSource, urls, features, pins }; + return { + version: product.version, + ageDays, + dataSource, + urls, + features, + pins, + }; } catch (error) { urls.forEach((url) => URL.revokeObjectURL(url)); throw error; @@ -330,6 +399,7 @@ export function createGhostMapsLayer({ function sync() { const run = _queue.then(async () => { if (!_enabled || !_viewer || !_catalog) return; + expireEvents(); const signal = _lifetime.signal; const products = new Map(_catalog.products.map((p) => [p.id, p])); for (const id of [..._loaded.keys()]) { @@ -338,13 +408,15 @@ export function createGhostMapsLayer({ } for (const product of _catalog.products) { if (!wanted(product) || signal.aborted) continue; - if (_loaded.get(product.id)?.version === product.version) continue; + const loaded = _loaded.get(product.id); + if (loaded?.version === product.version && loaded.ageDays === _ageDays) + continue; _loading.set(product.id, product.version); _failed.delete(product.id); notifyRow(); try { const next = await load(product, signal); - if (signal.aborted || !_enabled) { + if (signal.aborted || !_enabled || next.ageDays !== _ageDays) { next.urls.forEach((url) => URL.revokeObjectURL(url)); return; } @@ -391,12 +463,28 @@ export function createGhostMapsLayer({ void sync(); } + function setAgeDays(ageDays) { + if (!GHOSTMAPS_AGE_OPTIONS.includes(ageDays) || ageDays === _ageDays) + return; + _ageDays = ageDays; + try { + storage?.setItem(AGE_STORAGE_KEY, JSON.stringify(ageDays)); + } catch { + /* private window or blocked storage: keep the session choice */ + } + expireEvents(); + notifyRow(); + // A wider window must restore previously omitted placemarks from the + // cached KMZ, even though its upstream version has not changed. + return sync(); + } + // ---- selection card ---------------------------------------------------- function featureFor(key) { const [productId, index] = key.split(':'); return { product: _catalog?.products.find((p) => p.id === productId) || null, - feature: _loaded.get(productId)?.features[Number(index)] || null, + feature: _loaded.get(productId)?.features.get(Number(index)) || null, }; } @@ -490,7 +578,7 @@ export function createGhostMapsLayer({ if (typeof pickId !== 'string' || !pickId.startsWith(PICK_PREFIX)) return null; const [productId, index] = pickId.slice(PICK_PREFIX.length).split(':'); - return _loaded.get(productId)?.features[Number(index)] + return _loaded.get(productId)?.features.has(Number(index)) ? `${productId}:${index}` : null; } @@ -542,7 +630,7 @@ export function createGhostMapsLayer({ function featureCount() { let count = 0; - for (const item of _loaded.values()) count += item.features.length; + for (const item of _loaded.values()) count += item.features.size; return count; } @@ -563,6 +651,8 @@ export function createGhostMapsLayer({ enable() { _enabled = true; _lifetime = new AbortController(); + if (_expiryTimer === null) + _expiryTimer = setInterval(expireEvents, 60_000); for (const item of _loaded.values()) item.dataSource.show = true; installClickHandler(); if (!_unsubscribe && typeof source.subscribe === 'function') @@ -580,6 +670,8 @@ export function createGhostMapsLayer({ disable() { _enabled = false; + clearInterval(_expiryTimer); + _expiryTimer = null; _lifetime.abort(); _unsubscribe?.(); _unsubscribe = null; @@ -593,6 +685,7 @@ export function createGhostMapsLayer({ async update(_viewerArg, { signal } = {}) { if (!_enabled || signal?.aborted) return false; + expireEvents(); try { const catalog = await source.getCatalog({ signal }); if (!_enabled) return false; @@ -637,37 +730,54 @@ export function createGhostMapsLayer({ }) : t('Connecting to the S2 GhostMaps repository…'); return { - chips: products.map((product) => { - const loaded = _loaded.get(product.id); - const failed = _failed.get(product.id); - return { - id: product.id, - label: t(PRODUCT_LABELS[product.id] || product.title), - active: wanted(product), - busy: _loading.has(product.id), - state: failed ? 'error' : undefined, - color: PRODUCT_COLORS[product.id], - title: [ - product.fileTitle, - loaded - ? tn( - loaded.features.length, - '{count} placemark', - '{count} placemarks', - { - count: formatNumber(loaded.features.length), - }, - ) - : null, - failed ? t(failed) : null, - ] - .filter(Boolean) - .join(' · '), - onClick: () => toggleProduct(product.id), - }; - }), + chips: [ + ...GHOSTMAPS_AGE_OPTIONS.map((days) => ({ + id: `age-${days ?? 'all'}`, + label: + days === null + ? t('All markers') + : t('{days}d', { days: formatNumber(days) }), + title: + days === null + ? t('Show markers of every age') + : t('Maximum event age: {days}d; undated markers are kept', { + days: formatNumber(days), + }), + active: days === _ageDays, + onClick: () => setAgeDays(days), + })), + ...products.map((product) => { + const loaded = _loaded.get(product.id); + const failed = _failed.get(product.id); + return { + id: product.id, + label: t(PRODUCT_LABELS[product.id] || product.title), + active: wanted(product), + busy: _loading.has(product.id), + state: failed ? 'error' : undefined, + color: PRODUCT_COLORS[product.id], + title: [ + product.fileTitle, + loaded + ? tn( + loaded.features.size, + '{count} placemark', + '{count} placemarks', + { + count: formatNumber(loaded.features.size), + }, + ) + : null, + failed ? t(failed) : null, + ] + .filter(Boolean) + .join(' · '), + onClick: () => toggleProduct(product.id), + }; + }), + ], legend: [], - info, + info: `${_ageDays === null ? t('All markers') : t('Event age: {days}d · undated kept', { days: formatNumber(_ageDays) })} · ${info}`, infoTitle: t( 'S2 Underground publishes these KMZ products to GitHub. New uploads are detected within seconds and replace the previous edition automatically. Click a marker for its report.', ), @@ -677,6 +787,7 @@ export function createGhostMapsLayer({ getStats() { return { count: featureCount(), + ageDays: _ageDays, products: [..._loaded.keys()], commit: _catalog?.commit || null, lastUpdate: _catalog?.committedAt || _catalog?.changedAt || null, @@ -693,7 +804,7 @@ export function createGhostMapsLayer({ if (!_enabled) return []; const out = []; for (const [productId, item] of _loaded) { - for (const feature of item.features) { + for (const feature of item.features.values()) { if (out.length >= maxCount) return out; out.push({ product: productId, diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 51d546017..0b3a580e6 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import * as Cesium from 'cesium'; import { createGhostMapsLayer } from './index.js'; +import { GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP } from './catalog.js'; function element(localName, children = [], value = '', attributes = {}) { const node = { @@ -22,7 +23,7 @@ function element(localName, children = [], value = '', attributes = {}) { return node; } -function documentFixture() { +function documentFixture(reports = [{ name: 'Test report' }]) { return element('document', [ element('kml', [ element('Document', [ @@ -36,11 +37,14 @@ function documentFixture() { '', { id: 'pin' }, ), - element('Placemark', [ - element('name', [], 'Test report'), - element('styleUrl', [], '#pin'), - element('Point', [element('coordinates', [], '12,34')]), - ]), + ...reports.map(({ name, description = '', geometry }) => + element('Placemark', [ + element('name', [], name), + element('description', [], description), + element('styleUrl', [], '#pin'), + geometry || element('Point', [element('coordinates', [], '12,34')]), + ]), + ), ]), ]), ]); @@ -83,11 +87,21 @@ function fixtureKmz() { return new Uint8Array(Buffer.concat([...local, directory, end])); } -function fixture(t, { beforeFileReturns, onCreateUrl } = {}) { +function fixture( + t, + { + beforeFileReturns, + onCreateUrl, + reports, + now, + products, + controls = {}, + } = {}, +) { const previous = Object.getOwnPropertyDescriptor(globalThis, 'DOMParser'); globalThis.DOMParser = class { parseFromString() { - return documentFixture(); + return documentFixture(reports); } }; t.after(() => { @@ -119,9 +133,13 @@ function fixture(t, { beforeFileReturns, onCreateUrl } = {}) { }; const layer = createGhostMapsLayer({ storage: null, + now, + ...controls, source: { getCatalog: async () => ({ - products: [{ id: 'cip', version: 'a'.repeat(40), defaultOn: true }], + products: products ?? [ + { id: 'cip', version: 'a'.repeat(40), defaultOn: true }, + ], }), getFile: async () => { beforeFileReturns?.(layer); @@ -178,3 +196,284 @@ test('successful load keeps icons until the product is disabled', async (t) => { assert.deepEqual(revoked, created); assert.equal(mounted.size, 0); }); + +const dated = (name, value, label = 'Date') => ({ + name, + description: `
${label}${value}
`, +}); + +const ageChip = (layer, days) => + layer.getRowControls().chips.find(({ id }) => id === `age-${days ?? 'all'}`); + +test('age controls default to seven days and widening restores filtered and expired reports', async (t) => { + let at = Date.UTC(2026, 10, 3, 12); + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + dated('Old', '10/1/2026'), + dated('Ten days', '10/24/2026'), + dated('Recent', '11/2/2026'), + { name: 'Undated' }, + ], + }); + await layer.update(viewer); + assert.equal(layer.getStats().ageDays, 7); + assert.equal(ageChip(layer, 7).active, true); + assert.equal(layer.getStats().count, 2); + await ageChip(layer, 14).onClick(); + assert.equal(layer.getStats().count, 3); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().count, 4); + at += 100 * 86400_000; + await layer.update(viewer); + assert.equal(layer.getStats().count, 4, 'all markers never ages out'); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); + assert.equal(layer.getAnalystRecords()[0].name, 'Undated'); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().count, 4); + assert.equal(mounted.size, 1, 'filter changes never duplicate data sources'); + const ageChips = layer + .getRowControls() + .chips.filter(({ id }) => id.startsWith('age-')); + assert.equal(ageChips.filter(({ active }) => active).length, 1); + assert.equal(ageChip(layer, null).active, true); +}); + +test('age preferences persist independently of product choices and reject invalid storage', async (t) => { + const values = new Map([['gev.ghostmaps.products', '["cip"]']]); + const storage = { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => values.set(key, value), + }; + await t.test('save All markers without changing products', async (t) => { + const { layer, viewer } = fixture(t, { controls: { storage } }); + await layer.update(viewer); + await ageChip(layer, null).onClick(); + assert.equal(values.get('gev.ghostmaps.ageDays'), 'null'); + assert.equal(values.get('gev.ghostmaps.products'), '["cip"]'); + }); + await t.test('restore All markers and save thirty days', async (t) => { + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, null); + await ageChip(layer, 30).onClick(); + assert.equal(values.get('gev.ghostmaps.ageDays'), '30'); + }); + await t.test('restore thirty days', (t) => { + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, 30); + }); + for (const value of ['garbage', '"7"', '-1', '0', '2', '{}']) { + await t.test(`reject ${value}`, (t) => { + values.set('gev.ghostmaps.ageDays', value); + const { layer } = fixture(t, { controls: { storage } }); + assert.equal(layer.getStats().ageDays, 7); + }); + } +}); + +test('blocked storage still permits changing the age within the session', async (t) => { + const storage = { + getItem() { + throw new Error('blocked'); + }, + setItem() { + throw new Error('blocked'); + }, + }; + const { layer, viewer } = fixture(t, { controls: { storage } }); + await layer.update(viewer); + assert.equal(layer.getStats().ageDays, 7); + await ageChip(layer, null).onClick(); + assert.equal(layer.getStats().ageDays, null); +}); + +test('a filter changed during a pending load never installs the superseded filter', async (t) => { + let changed = false; + const { layer, viewer, mounted, revoked, created } = fixture(t, { + now: () => Date.UTC(2026, 10, 3), + reports: [dated('Old', '10/1/2026')], + beforeFileReturns: (layer) => { + if (changed) return; + changed = true; + ageChip(layer, null).onClick(); + }, + }); + await layer.update(viewer); + await layer.update(viewer); + assert.equal(layer.getStats().count, 1); + assert.equal(layer.getStats().ageDays, null); + assert.equal(mounted.size, 1); + assert.equal( + revoked.length, + created.length - 1, + 'discarded load releases icons', + ); + + test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { + for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { + await t.test(product.id, async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + products: [{ ...product, version: 'a'.repeat(40) }], + reports: [ + dated( + 'Old event', + '10/1/2026', + product.id === 'cip' ? 'Date of Event' : 'Date', + ), + dated('Recent event', '11/2/2026'), + { name: 'Undated reference feature' }, + ], + }); + await layer.update(viewer); + if (!product.defaultOn) { + layer + .getRowControls() + .chips.find(({ id }) => id === product.id) + .onClick(); + await layer.update(viewer); + } + assert.deepEqual(layer.getStats().products, [product.id]); + assert.equal(layer.getStats().count, 2); + assert.deepEqual( + layer.getAnalystRecords().map(({ name }) => name), + ['Recent event', 'Undated reference feature'], + ); + await ageChip(layer, null).onClick(); + assert.equal( + layer.getStats().count, + 3, + "All markers restores this product's old events", + ); + await ageChip(layer, 7).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal(mounted.size, 1); + }); + } + }); +}); + +test('default products filter old event dates while keeping recent, future and undated markers', async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [ + dated('Old event', '10/20/2026'), + dated('Recent event', '11/2/2026'), + dated('At cutoff', '10/27/2026 12:00:00 PM'), + dated('Before cutoff', '10/27/2026 11:59:59 AM'), + dated('Future event', '11/4/2026'), + { name: 'Undated site' }, + dated('Blank date', '<Null>'), + dated('Invalid date', '2/30/2026'), + dated('Only a report date', '10/1/2026', 'Date Of Report'), + { + name: 'Old event reported recently', + description: + '
Date of Event10/1/2026
Date Of Report11/2/2026
', + }, + ], + }); + await layer.update(viewer); + assert.deepEqual(layer.getStats().products, ['cip']); + assert.deepEqual( + layer.getAnalystRecords().map((r) => r.name), + [ + 'Recent event', + 'At cutoff', + 'Future event', + 'Undated site', + 'Blank date', + 'Invalid date', + 'Only a report date', + ], + ); + assert.equal(layer.getStats().count, 7); + assert.equal([...mounted][0].entities.values.length, 7); + // Filtering cannot shift pick ids onto a different report. + assert.ok([...mounted][0].entities.getById('ghostmaps:cip:1:0')); + assert.equal( + [...mounted][0].entities.getById('ghostmaps:cip:0:0'), + undefined, + ); +}); + +test('unchanged cached products age out on the timer, clear selected cards and keep surviving picks', async (t) => { + t.mock.timers.enable({ apis: ['setInterval'] }); + let at = Date.UTC(2026, 10, 3, 12); + let click; + const cards = []; + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + dated('Expiring', '10/27/2026 12:00:00 PM'), + dated('Surviving', '11/2/2026'), + ], + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + const ds = [...mounted][0]; + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + at += 60_000; + t.mock.timers.tick(60_000); + assert.equal(layer.getStats().count, 1); + assert.equal(ds.entities.values.length, 1); + assert.equal(ds.entities.getById('ghostmaps:cip:0:0'), undefined); + assert.ok(ds.entities.getById('ghostmaps:cip:1:0')); + assert.deepEqual(cards.at(-1), []); + await layer.update(viewer); + assert.equal([...mounted][0], ds, 'same-version KMZ is not reloaded'); + viewer.scene.pick = () => 'ghostmaps:cip:1:0'; + click({ position: { x: 0, y: 0 } }); + assert.match(cards.at(-1)[0].title, /Surviving/); + layer.disable(); + at += 14 * 86400_000; + t.mock.timers.tick(60_000); + assert.equal(mounted.size, 0); + layer.enable(); + await layer.update(viewer); + assert.equal(layer.getStats().count, 0); +}); + +test('expiry removes every geometry and polygon outline belonging to an old report', async (t) => { + t.mock.timers.enable({ apis: ['setInterval'] }); + let at = Date.UTC(2026, 10, 3); + const polygon = element('Polygon', [ + element('outerBoundaryIs', [ + element('LinearRing', [ + element('coordinates', [], '12,34 13,34 13,35 12,34'), + ]), + ]), + ]); + const { layer, viewer, mounted } = fixture(t, { + now: () => at, + reports: [ + { + ...dated('Multi-part', '10/27/2026'), + geometry: element('MultiGeometry', [ + element('Point', [element('coordinates', [], '12,34')]), + element('LineString', [element('coordinates', [], '12,34 13,35')]), + polygon, + ]), + }, + ], + }); + await layer.update(viewer); + const ds = [...mounted][0]; + assert.equal(ds.entities.values.length, 4); + at += 60_000; + t.mock.timers.tick(60_000); + assert.equal(ds.entities.values.length, 0); + assert.equal(layer.getStats().count, 0); +}); diff --git a/src/layers/ghostMaps/records.js b/src/layers/ghostMaps/records.js index 7c4eb58bf..3c307fa63 100644 --- a/src/layers/ghostMaps/records.js +++ b/src/layers/ghostMaps/records.js @@ -1,4 +1,4 @@ -// Pure helpers for GhostMaps placemarks: description tables to plain fields, +// Pure helpers for GhostMaps placemarks: report fields and event dates, // KML colors, and card text wrapping. Strings in, data out — description // markup is only ever read as text, never inserted into the page. @@ -51,6 +51,104 @@ const MAX_FIELDS = 24; const MAX_VALUE = 600; const MAX_LINKS = 6; +// Use the same leaf rows for report cards and event dates, without the card's +// field-count/value-length limits affecting the age filter. +function* descriptionRows(source) { + for (const row of source.matchAll( + /]*>((?:(?!/gi, + )) { + const cells = [ + ...row[1].matchAll(/]*>([\s\S]*?)<\/t[dh]\s*>/gi), + ].map((cell) => cell[1]); + if (cells.length === 2) + yield { + label: htmlToText(cells[0]), + value: htmlToText(cells[1]), + markup: cells[1], + }; + } +} + +export const GHOSTMAPS_DEFAULT_AGE_DAYS = 7; +export const GHOSTMAPS_AGE_OPTIONS = Object.freeze([1, 3, 7, 14, 30, null]); + +/** ArcGIS US dates (optionally with a time) and ISO dates, interpreted in UTC + * when no zone is supplied. Reject calendar rollovers instead of guessing. */ +export function parseGhostMapsEventDate(value) { + const input = String(value ?? '').trim(); + const us = + /^(\d{1,2})\/(\d{1,2})\/(\d{4})(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM))?$/i.exec( + input, + ); + const iso = + /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,3})?)?(?:Z|[+-]\d{2}:\d{2})?)?$/.exec( + input, + ); + if (!us && !iso) return null; + const year = Number(us ? us[3] : iso[1]); + const month = Number(us ? us[1] : iso[2]); + const day = Number(us ? us[2] : iso[3]); + let hour = Number(us?.[4] ?? iso?.[4] ?? 0); + const minute = Number(us?.[5] ?? iso?.[5] ?? 0); + const second = Number(us?.[6] ?? iso?.[6] ?? 0); + if (us?.[7]) { + if (hour < 1 || hour > 12) return null; + hour = (hour % 12) + (us[7].toUpperCase() === 'PM' ? 12 : 0); + } + const at = Date.UTC(year, month - 1, day, hour, minute, second); + const date = new Date(at); + if ( + year < 1900 || + month < 1 || + month > 12 || + day < 1 || + date.getUTCDate() !== day || + hour > 23 || + minute > 59 || + second > 59 + ) + return null; + if (iso?.[4]) { + const zoned = /(?:Z|[+-]\d{2}:\d{2})$/.test(input) ? input : `${input}Z`; + const parsed = Date.parse(zoned); + return Number.isFinite(parsed) ? parsed : null; + } + return at; +} + +/** The event's date, never its report/upload date or dates mentioned in prose. */ +export function ghostMapsEventDate(description) { + const rows = [ + ...descriptionRows( + String(description ?? '').replace( + /<(script|style|head)\b[\s\S]*?<\/\1\s*>/gi, + ' ', + ), + ), + ]; + for (const name of ['dateofevent', 'eventdate', 'date']) { + for (const { label, value } of rows) { + if (label.toLowerCase().replace(/[\s_]+/g, '') !== name) continue; + const at = parseGhostMapsEventDate(value); + if (at !== null) return at; + } + } + return null; +} + +/** Unknown and future dates stay visible; only a confirmed old event expires. */ +export function isGhostMapsEventExpired( + dateMs, + now = Date.now(), + ageDays = GHOSTMAPS_DEFAULT_AGE_DAYS, +) { + return ( + ageDays !== null && + Number.isFinite(dateMs) && + dateMs < now - ageDays * 86400_000 + ); +} + /** * Fields and links from a placemark description. ArcGIS "popup" exports are a * two-column table (label, value); anything else becomes one unlabelled field. @@ -73,18 +171,10 @@ export function parseDescriptionFields(html) { }; // Innermost rows only: ArcGIS nests the field table inside an outer row, // and only leaf rows with exactly two cells are label/value pairs. - for (const row of source.matchAll( - /]*>((?:(?!/gi, - )) { - const cells = [ - ...row[1].matchAll(/]*>([\s\S]*?)<\/t[dh]\s*>/gi), - ].map((cell) => cell[1]); - if (cells.length !== 2) continue; - const label = htmlToText(cells[0]); - const value = htmlToText(cells[1]); + for (const { label, value, markup } of descriptionRows(source)) { if (!label || SKIPPED_LABELS.has(label.toLowerCase())) continue; if (EMPTY_VALUES.has(value.toLowerCase())) continue; - addLinks(cells[1]); + addLinks(markup); // A bare research link is carried by `links`; the card does not need // the URL twice. if (/^https?:\/\/\S+$/i.test(value)) continue; diff --git a/src/layers/ghostMaps/records.test.mjs b/src/layers/ghostMaps/records.test.mjs index 86fecb52b..a28d1f36d 100644 --- a/src/layers/ghostMaps/records.test.mjs +++ b/src/layers/ghostMaps/records.test.mjs @@ -7,8 +7,83 @@ import { kmlColor, parseDescriptionFields, wrapCardText, + ghostMapsEventDate, + isGhostMapsEventExpired, + parseGhostMapsEventDate, } from './records.js'; +test('event dates parse ArcGIS calendar dates and timestamps consistently in UTC', () => { + for (const [value, iso] of [ + ['10/28/2026', '2026-10-28T00:00:00Z'], + ['10/28/2026 6:12:00 PM', '2026-10-28T18:12:00Z'], + ['1/17/2026 9:46:41 AM', '2026-01-17T09:46:41Z'], + ['10/28/2026 12:00 AM', '2026-10-28T00:00:00Z'], + ['10/28/2026 12:00 PM', '2026-10-28T12:00:00Z'], + ['2026-10-28', '2026-10-28T00:00:00Z'], + ['2026-10-28T12:34:56', '2026-10-28T12:34:56Z'], + ['2026-10-28T12:34:56.123-04:00', '2026-10-28T16:34:56.123Z'], + ['2/29/2024', '2024-02-29T00:00:00Z'], + ]) + assert.equal(parseGhostMapsEventDate(value), Date.parse(iso), value); + for (const value of [ + '', + '', + 'N/A', + 'unknown', + '2/29/2026', + '9/31/2026', + '2026-02-30', + '13/1/2026', + '1/0/2026', + '10/28/2026 13:00 PM', + '2026-10-28T24:00:00Z', + '2026-10-28T12:60:00Z', + ]) { + assert.equal(parseGhostMapsEventDate(value), null, value); + } +}); + +test('event date fields take precedence over report dates and prose, even after long report tables', () => { + const row = (label, value) => `${label}${value}`; + const html = `
${ + row('Date Of Report', '11/2/2026') + + row('Description', 'Incident mentioned on 11/1/2026') + + Array.from({ length: 30 }, (_, i) => row(`Field${i}`, 'value')).join('') + + row('Date', '10/30/2026') + + row('Date of Event', '10/28/2026') + }
`; + assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 28)); + assert.equal( + ghostMapsEventDate( + `${row('Date', '<Null>')}${row('Date Of Report', '11/2/2026')}
`, + ), + null, + ); + assert.equal( + ghostMapsEventDate(`${row('Date_Of_Event', '10/28/2026')}
`), + Date.UTC(2026, 9, 28), + ); + assert.equal(ghostMapsEventDate('Occurred 10/28/2026'), null); + assert.equal( + ghostMapsEventDate( + ``, + ), + null, + ); +}); + +test('seven-day expiry keeps the exact boundary, future dates and unknown dates', () => { + const at = Date.UTC(2026, 10, 3, 12); + assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12), at), false); + assert.equal( + isGhostMapsEventExpired(Date.UTC(2026, 9, 27, 12) - 1, at), + true, + ); + assert.equal(isGhostMapsEventExpired(Date.UTC(2026, 10, 4), at), false); + for (const value of [null, undefined, NaN]) + assert.equal(isGhostMapsEventExpired(value, at), false); +}); + // The shape ArcGIS Pro writes into every S2 placemark description. const ARCGIS = ` @@ -27,6 +102,14 @@ const ARCGIS = ` function changeImage(a, b) { document.getElementById('x').innerHTML = b; } `; +test('custom ages change the cutoff and all markers disables expiry', () => { + const at = Date.UTC(2026, 10, 3, 12); + const event = at - 10 * 86400_000; + assert.equal(isGhostMapsEventExpired(event, at, 7), true); + assert.equal(isGhostMapsEventExpired(event, at, 14), false); + assert.equal(isGhostMapsEventExpired(event, at, null), false); +}); + test('ArcGIS popup tables become plain label/value fields and http links', () => { const { fields, links } = parseDescriptionFields(ARCGIS); assert.deepEqual(fields, [ From 134597c3cf53aab6252fce260e1fa8cdc0f8c6ad Mon Sep 17 00:00:00 2001 From: Dylan Maniatakes Date: Tue, 3 Nov 2026 13:23:40 -0500 Subject: [PATCH 2/4] docs: add S2 age filter changelog for PR 1528 --- changelog.d/1528-s2-ghostmaps-age-filter.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1528-s2-ghostmaps-age-filter.md diff --git a/changelog.d/1528-s2-ghostmaps-age-filter.md b/changelog.d/1528-s2-ghostmaps-age-filter.md new file mode 100644 index 000000000..2ba019286 --- /dev/null +++ b/changelog.d/1528-s2-ghostmaps-age-filter.md @@ -0,0 +1,4 @@ +- Add an adjustable event-age filter across S2 GhostMaps products, defaulting + to seven days with 1-, 3-, 14- and 30-day choices and an All markers option. + Preserve undated features, save the selected window and expire old events + while the map stays open. Thanks @dylanmaniatakes (#1528). From c4298c44501a0e5c21216648f91e6bf9a2eeafd7 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Fri, 6 Nov 2026 17:25:01 +0100 Subject: [PATCH 3/4] fix(ghostmaps): keep age-window changes consistent and make the age choice a switch Review fixes on top of #1528: - Narrowing the age window only removes markers; a product loaded with a narrower window is reloaded when the window widens again. Fixes markers missing after 7d -> 1d -> 7d and avoids reparsing every KMZ on narrowing. - A wider window over the same edition keeps the open report card. - Date-only event dates cover their whole UTC day, so 1d still shows yesterday's reports. - The age options are one labelled "Event age" radio switch (row `segmented` control, as FIRMS confidence) instead of toggle chips mixed with the product chips. - Un-nest the per-product age test; QA compares data-source counts with the live defaults instead of a hard-coded 2. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 61 ++++----- scripts/qa-ghostmaps.mjs | 33 +++-- src/i18n/locales/ar.json | 1 + src/i18n/locales/de.json | 1 + src/i18n/locales/es.json | 1 + src/i18n/locales/fr.json | 1 + src/i18n/locales/it.json | 1 + src/i18n/locales/ja.json | 1 + src/i18n/locales/pt.json | 1 + src/i18n/locales/ru.json | 1 + src/i18n/locales/uk.json | 1 + src/i18n/locales/zh-CN.json | 1 + src/i18n/locales/zh-Hant.json | 1 + src/layers/ghostMaps/index.js | 86 +++++++----- src/layers/ghostMaps/lifecycle.test.mjs | 175 +++++++++++++++++------- src/layers/ghostMaps/records.js | 6 +- src/layers/ghostMaps/records.test.mjs | 10 +- 17 files changed, 247 insertions(+), 135 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1a7205096..fe837e421 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -250,7 +250,6 @@ style listeners. Synthetic fixtures cover privacy, acquisition, rendering, selection and card placement. Deterministic operation checks run in `npm test`; wall-clock budgets run in `npm run test:perf`. All 11 locales are supported. - ## Prohibited & Restricted Airspace — FAA special-use airspace The Infrastructure group includes **Prohibited & Restricted Airspace** (`drone-airspace`, @@ -4342,35 +4341,35 @@ nearby place—always use station selection. Unqualified “turn on/start the ra requests use Play; a qualified Play-shaped tool call is normalized to Select so its criteria cannot be silently ignored. -| Layer | Source | File | Proxy | Update Interval | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | -| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/flights` (OAuth + fallback) | 30s | -| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/military` | 15s | -| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/vessels` | 60s (+800ms visibility pass) | -| Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | -| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | -| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | -| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | -| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | -| Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Hong Kong Transport Department + FL511 (FL, opt-in) + Freeway Bureau, MOTC (Taiwan) + Houston TranStar (opt-in) + Autostrada del Brennero (IT, A22 stills) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | -| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | -| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | -| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | -| Datacenters ▣ | OSM extract (bundled) | `src/data/infrastructure.js` | — | static | -| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/infrastructure.js` | — | static | -| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | -| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | -| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | -| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | -| GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | -| S2 GhostMaps 👻 (`s2-ghostmaps`) | S2 Underground's public GhostMaps repository on GitHub (KMZ intel products, keyless, CC BY-NC-SA 4.0) | `src/layers/ghostMaps/` via `src/app/layers/ghostMaps.js` | `/api/ghostmaps/catalog`, `/api/ghostmaps/events` (server-sent events), `/api/ghostmaps/file/{product}` | branch head checked every 15 s; new commits pushed to the page; 5 min catalog backstop | -| Sea Marks ⚓ (`sea-marks`) | OpenSeaMap seamark raster tiles (keyless; CC BY-SA 2.0 tiles, OSM data ODbL) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | -| Ski Pistes ⛷ (`ski-pistes`) | OpenSnowMap piste raster tiles (keyless; CC BY-SA tiles, OSM data ODbL; no bulk download) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | -| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | -| Prohibited & Restricted Airspace 🚫 | FAA Special Use Airspace prohibited (P) and restricted (R) zones (keyless FeatureServer, simplified geometry) | `src/layers/droneAirspace/` via `src/app/layers/droneAirspace.js` | `/api/drone-airspace` | 1 h (server cache 1 h; stale-on-error) | -| Wildlife Sightings 🦉 | iNaturalist wildlife observed within 48 h (whales 7 d), per view; Acartia Salish Sea marine mammal coverage (keyless) | `src/layers/wildlifeSightings/` via `src/app/layers/wildlifeSightings.js` | `/api/inaturalist/observations`, `/api/inaturalist/grid`, `/api/acartia/sightings` | camera moveEnd (600 ms debounce; enable settles 300 ms) + 5 min (observations 5 / 10 min; tiles 15 min; stale ≤ 1 h / 24 h) | +| Layer | Source | File | Proxy | Update Interval | +| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/flights` (OAuth + fallback) | 30s | +| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/military` | 15s | +| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/vessels` | 60s (+800ms visibility pass) | +| Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | +| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | +| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | +| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | +| Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Hong Kong Transport Department + FL511 (FL, opt-in) + Freeway Bureau, MOTC (Taiwan) + Houston TranStar (opt-in) + Autostrada del Brennero (IT, A22 stills) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | +| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | +| Datacenters ▣ | OSM extract (bundled) | `src/data/infrastructure.js` | — | static | +| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/infrastructure.js` | — | static | +| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | +| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | +| Street Level 📷 | Mapillary street-level imagery (vector tiles, Graph API, MapillaryJS) | `src/layers/streetLevel/` via `src/app/layers/streetLevel.js` | `/api/mapillary/status`, `/api/mapillary/tiles/coverage/{z}/{x}/{y}` (`MAPILLARY_CLIENT_TOKEN`) | camera-driven (320 ms debounce, ≤9 tiles, server memory cache 24 h) | +| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | +| GNSS Integrity 📡 (`gnss-integrity`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5° cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | +| S2 GhostMaps 👻 (`s2-ghostmaps`) | S2 Underground's public GhostMaps repository on GitHub (KMZ intel products, keyless, CC BY-NC-SA 4.0) | `src/layers/ghostMaps/` via `src/app/layers/ghostMaps.js` | `/api/ghostmaps/catalog`, `/api/ghostmaps/events` (server-sent events), `/api/ghostmaps/file/{product}` | branch head checked every 15 s; new commits pushed to the page; 5 min catalog backstop | +| Sea Marks ⚓ (`sea-marks`) | OpenSeaMap seamark raster tiles (keyless; CC BY-SA 2.0 tiles, OSM data ODbL) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | +| Ski Pistes ⛷ (`ski-pistes`) | OpenSnowMap piste raster tiles (keyless; CC BY-SA tiles, OSM data ODbL; no bulk download) | `src/data/osmOverlays.js` | — (browser-direct tiles) | on view (Cesium tile requests; no polling) | +| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | +| Prohibited & Restricted Airspace 🚫 | FAA Special Use Airspace prohibited (P) and restricted (R) zones (keyless FeatureServer, simplified geometry) | `src/layers/droneAirspace/` via `src/app/layers/droneAirspace.js` | `/api/drone-airspace` | 1 h (server cache 1 h; stale-on-error) | +| Wildlife Sightings 🦉 | iNaturalist wildlife observed within 48 h (whales 7 d), per view; Acartia Salish Sea marine mammal coverage (keyless) | `src/layers/wildlifeSightings/` via `src/app/layers/wildlifeSightings.js` | `/api/inaturalist/observations`, `/api/inaturalist/grid`, `/api/acartia/sightings` | camera moveEnd (600 ms debounce; enable settles 300 ms) + 5 min (observations 5 / 10 min; tiles 15 min; stale ≤ 1 h / 24 h) | GNSS Integrity samples a 250 nm circle around the Earth point under the screen centre. Camera reads debounce for 600 ms with a monotonic 2.5 s minimum gap; @@ -6008,7 +6007,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. -- **Event age filter.** The same filter applies to every selected S2 product, including Border Crisis, Iran Picture, Iran Kinetic, Yemen FLOT and Latest Drop. S2 products default to seven days, with row buttons for 1, 3, 7, 14 or 30 days and **All markers**. The choice persists independently of product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage uses seven days. Only the placemark's `Date of Event` (or `EventDate`/`Date`) determines its age, never report dates, dates in prose or KMZ edition/upload dates. Undated, blank or unparseable event dates and future dates stay visible. ArcGIS `M/D/YYYY` dates and optional AM/PM times, plus ISO dates/timestamps, are supported; values without a zone use UTC and date-only values use midnight UTC. The exact cutoff is included. Events age out every minute and during catalog updates, even without a new file or a successful catalog fetch. Widening the window or choosing All markers restores omitted events from the cached KMZ. Removed entities and selected cards disappear together; surviving pick ids stay stable. Product defaults and the update stream are unchanged. +- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All markers**. The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/scripts/qa-ghostmaps.mjs b/scripts/qa-ghostmaps.mjs index 24edcc60b..3c3f0b880 100644 --- a/scripts/qa-ghostmaps.mjs +++ b/scripts/qa-ghostmaps.mjs @@ -203,15 +203,12 @@ async function main() { ); record( 'LIVE: one chip per product, defaults pressed', - chips.filter((chip) => !chip.id.startsWith('age-')).length === - catalog.products.length && - chips - .filter((chip) => !chip.id.startsWith('age-')) - .every( - (chip) => - chip.active === - catalog.products.find((p) => p.id === chip.id)?.defaultOn, - ), + chips.length === catalog.products.length && + chips.every( + (chip) => + chip.active === + catalog.products.find((p) => p.id === chip.id)?.defaultOn, + ), chips.map((c) => `${c.id}${c.active ? '*' : ''}`).join(' '), ); const stream = await page @@ -229,7 +226,18 @@ async function main() { record('LIVE: the /api/ghostmaps/events stream is open', stream); await page.screenshot({ path: path.join(SHOTS_DIR, 'ghostmaps-live.png') }); - const ageChips = chips.filter((chip) => chip.id.startsWith('age-')); + const ageChips = await page.evaluate( + (id) => + [ + ...document.querySelectorAll( + `[data-layer-id="${id}"] [data-segmented-id="age"] [role="radio"]`, + ), + ].map((option) => ({ + id: option.dataset.chipId, + active: option.getAttribute('aria-checked') === 'true', + })), + LAYER_ID, + ); record( 'AGE: seven days is the default with an All markers option', live.ageDays === 7 && @@ -262,7 +270,8 @@ async function main() { const all = await selectAge(null); record( 'AGE: All markers restores old events without duplicate data sources', - all.count > live.count && (await sources(page)).length === 2, + all.count > live.count && + (await sources(page)).length === live.products.length, `${live.count} → ${all.count}`, ); await page.reload({ waitUntil: 'domcontentloaded' }); @@ -279,7 +288,7 @@ async function main() { const one = await selectAge(1); const seven = await selectAge(7); record( - 'AGE: shorter and longer windows change the count and seven days restores the default', + 'AGE: counts follow the window order and seven days restores the default', one.count <= live.count && thirty.count >= live.count && thirty.count <= all.count && diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index 456917068..99826e32c 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3471,5 +3471,6 @@ "All markers": "كل العلامات", "Show markers of every age": "عرض العلامات من جميع الأعمار", "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "Event age": "عمر الحدث", "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index d38cc685f..826091ee8 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3171,5 +3171,6 @@ "All markers": "Alle Markierungen", "Show markers of every age": "Markierungen jeden Alters anzeigen", "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "Event age": "Ereignisalter", "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index 45fd1961c..2f1cc02f1 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3246,5 +3246,6 @@ "All markers": "Todos los marcadores", "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "Event age": "Antigüedad", "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index 528d01d9c..ff0b02c01 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3246,5 +3246,6 @@ "All markers": "Tous les marqueurs", "Show markers of every age": "Afficher les marqueurs de tout âge", "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "Event age": "Ancienneté", "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index b8204a980..4a26811fa 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3246,5 +3246,6 @@ "All markers": "Tutti gli indicatori", "Show markers of every age": "Mostra gli indicatori di qualsiasi età", "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "Event age": "Età degli eventi", "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 6f88bd36d..9f111d91d 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -3096,5 +3096,6 @@ "All markers": "すべてのマーカー", "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数：{days}日。日付のないマーカーは保持", + "Event age": "経過日数", "Event age: {days}d · undated kept": "経過日数：{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index 0f328d23b..ed83191d9 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3246,5 +3246,6 @@ "All markers": "Todos os marcadores", "Show markers of every age": "Mostrar marcadores de qualquer idade", "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "Event age": "Idade do evento", "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index f3d050aba..82aa91f06 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3321,5 +3321,6 @@ "All markers": "Все метки", "Show markers of every age": "Показать метки любого возраста", "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "Event age": "Возраст события", "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index e340f2e12..4d8ff9711 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3321,5 +3321,6 @@ "All markers": "Усі позначки", "Show markers of every age": "Показати позначки будь-якого віку", "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "Event age": "Вік події", "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index 9e088ac3e..a7cb60910 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -3096,5 +3096,6 @@ "All markers": "所有标记", "Show markers of every age": "显示所有时间的标记", "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间：{days}天；保留无日期标记", + "Event age": "事件时间", "Event age: {days}d · undated kept": "事件时间：{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index 578fbcf40..ebbf0680c 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -3096,5 +3096,6 @@ "All markers": "所有標記", "Show markers of every age": "顯示所有時間的標記", "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間：{days}天；保留無日期標記", + "Event age": "事件時間", "Event age: {days}d · undated kept": "事件時間：{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 7899c26b5..7d8716483 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -208,13 +208,13 @@ export function createGhostMapsLayer({ return (_catalog?.products || []).filter(wanted).map(({ id }) => id); } - function unload(id) { + function unload(id, keepSelection = false) { const item = _loaded.get(id); if (!item) return; _loaded.delete(id); _viewer?.dataSources?.remove(item.dataSource, true); item.urls.forEach((url) => URL.revokeObjectURL(url)); - if (_selectedKey?.startsWith(`${id}:`)) clearSelection(); + if (!keepSelection && _selectedKey?.startsWith(`${id}:`)) clearSelection(); } // Remove expired entities, rather than hiding them: horizon culling must @@ -423,8 +423,9 @@ export function createGhostMapsLayer({ // Cull before the first frame so far-side pins never flash. _horizon.update(next.pins); await _viewer.dataSources.add(next.dataSource); - // The new edition is on the globe before the old one leaves. - unload(product.id); + // The new edition is on the globe before the old one leaves. A wider + // window over the same edition keeps indices, so the card stays. + unload(product.id, loaded?.version === next.version); _loaded.set(product.id, next); } catch (error) { if (signal.aborted) return; @@ -473,9 +474,16 @@ export function createGhostMapsLayer({ /* private window or blocked storage: keep the session choice */ } expireEvents(); + // Narrowing only removes markers. A product loaded with a narrower window + // keeps its old ageDays, so sync() restores omitted placemarks from the + // cached KMZ even though its upstream version has not changed. + for (const item of _loaded.values()) + if ( + item.ageDays === null || + (ageDays !== null && item.ageDays >= ageDays) + ) + item.ageDays = ageDays; notifyRow(); - // A wider window must restore previously omitted placemarks from the - // cached KMZ, even though its upstream version has not changed. return sync(); } @@ -730,8 +738,10 @@ export function createGhostMapsLayer({ }) : t('Connecting to the S2 GhostMaps repository…'); return { - chips: [ - ...GHOSTMAPS_AGE_OPTIONS.map((days) => ({ + segmented: { + id: 'age', + label: t('Event age'), + options: GHOSTMAPS_AGE_OPTIONS.map((days) => ({ id: `age-${days ?? 'all'}`, label: days === null @@ -746,36 +756,36 @@ export function createGhostMapsLayer({ active: days === _ageDays, onClick: () => setAgeDays(days), })), - ...products.map((product) => { - const loaded = _loaded.get(product.id); - const failed = _failed.get(product.id); - return { - id: product.id, - label: t(PRODUCT_LABELS[product.id] || product.title), - active: wanted(product), - busy: _loading.has(product.id), - state: failed ? 'error' : undefined, - color: PRODUCT_COLORS[product.id], - title: [ - product.fileTitle, - loaded - ? tn( - loaded.features.size, - '{count} placemark', - '{count} placemarks', - { - count: formatNumber(loaded.features.size), - }, - ) - : null, - failed ? t(failed) : null, - ] - .filter(Boolean) - .join(' · '), - onClick: () => toggleProduct(product.id), - }; - }), - ], + }, + chips: products.map((product) => { + const loaded = _loaded.get(product.id); + const failed = _failed.get(product.id); + return { + id: product.id, + label: t(PRODUCT_LABELS[product.id] || product.title), + active: wanted(product), + busy: _loading.has(product.id), + state: failed ? 'error' : undefined, + color: PRODUCT_COLORS[product.id], + title: [ + product.fileTitle, + loaded + ? tn( + loaded.features.size, + '{count} placemark', + '{count} placemarks', + { + count: formatNumber(loaded.features.size), + }, + ) + : null, + failed ? t(failed) : null, + ] + .filter(Boolean) + .join(' · '), + onClick: () => toggleProduct(product.id), + }; + }), legend: [], info: `${_ageDays === null ? t('All markers') : t('Event age: {days}d · undated kept', { days: formatNumber(_ageDays) })} · ${info}`, infoTitle: t( diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 0b3a580e6..7d1a067e6 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -203,7 +203,9 @@ const dated = (name, value, label = 'Date') => ({ }); const ageChip = (layer, days) => - layer.getRowControls().chips.find(({ id }) => id === `age-${days ?? 'all'}`); + layer + .getRowControls() + .segmented.options.find(({ id }) => id === `age-${days ?? 'all'}`); test('age controls default to seven days and widening restores filtered and expired reports', async (t) => { let at = Date.UTC(2026, 10, 3, 12); @@ -233,10 +235,16 @@ test('age controls default to seven days and widening restores filtered and expi await ageChip(layer, null).onClick(); assert.equal(layer.getStats().count, 4); assert.equal(mounted.size, 1, 'filter changes never duplicate data sources'); - const ageChips = layer - .getRowControls() - .chips.filter(({ id }) => id.startsWith('age-')); - assert.equal(ageChips.filter(({ active }) => active).length, 1); + const controls = layer.getRowControls(); + assert.equal( + controls.segmented.options.filter(({ active }) => active).length, + 1, + ); + assert.equal( + controls.chips.some(({ id }) => id.startsWith('age-')), + false, + 'age options are one switch, not product chips', + ); assert.equal(ageChip(layer, null).active, true); }); @@ -309,49 +317,49 @@ test('a filter changed during a pending load never installs the superseded filte created.length - 1, 'discarded load releases icons', ); +}); - test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { - for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { - await t.test(product.id, async (t) => { - const { layer, viewer, mounted } = fixture(t, { - now: () => Date.UTC(2026, 10, 3, 12), - products: [{ ...product, version: 'a'.repeat(40) }], - reports: [ - dated( - 'Old event', - '10/1/2026', - product.id === 'cip' ? 'Date of Event' : 'Date', - ), - dated('Recent event', '11/2/2026'), - { name: 'Undated reference feature' }, - ], - }); - await layer.update(viewer); - if (!product.defaultOn) { - layer - .getRowControls() - .chips.find(({ id }) => id === product.id) - .onClick(); - await layer.update(viewer); - } - assert.deepEqual(layer.getStats().products, [product.id]); - assert.equal(layer.getStats().count, 2); - assert.deepEqual( - layer.getAnalystRecords().map(({ name }) => name), - ['Recent event', 'Undated reference feature'], - ); - await ageChip(layer, null).onClick(); - assert.equal( - layer.getStats().count, - 3, - "All markers restores this product's old events", - ); - await ageChip(layer, 7).onClick(); - assert.equal(layer.getStats().count, 2); - assert.equal(mounted.size, 1); +test('age filtering and All markers apply to every conflict-map product, including optional maps', async (t) => { + for (const product of [...GHOSTMAPS_PRODUCTS, GHOSTMAPS_LATEST_DROP]) { + await t.test(product.id, async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + products: [{ ...product, version: 'a'.repeat(40) }], + reports: [ + dated( + 'Old event', + '10/1/2026', + product.id === 'cip' ? 'Date of Event' : 'Date', + ), + dated('Recent event', '11/2/2026'), + { name: 'Undated reference feature' }, + ], }); - } - }); + await layer.update(viewer); + if (!product.defaultOn) { + layer + .getRowControls() + .chips.find(({ id }) => id === product.id) + .onClick(); + await layer.update(viewer); + } + assert.deepEqual(layer.getStats().products, [product.id]); + assert.equal(layer.getStats().count, 2); + assert.deepEqual( + layer.getAnalystRecords().map(({ name }) => name), + ['Recent event', 'Undated reference feature'], + ); + await ageChip(layer, null).onClick(); + assert.equal( + layer.getStats().count, + 3, + "All markers restores this product's old events", + ); + await ageChip(layer, 7).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal(mounted.size, 1); + }); + } }); test('default products filter old event dates while keeping recent, future and undated markers', async (t) => { @@ -448,7 +456,8 @@ test('unchanged cached products age out on the timer, clear selected cards and k test('expiry removes every geometry and polygon outline belonging to an old report', async (t) => { t.mock.timers.enable({ apis: ['setInterval'] }); - let at = Date.UTC(2026, 10, 3); + // 10/27 is a whole UTC day: at 7d it expires when 11/3 ends. + let at = Date.UTC(2026, 10, 4) - 30_000; const polygon = element('Polygon', [ element('outerBoundaryIs', [ element('LinearRing', [ @@ -477,3 +486,75 @@ test('expiry removes every geometry and polygon outline belonging to an old repo assert.equal(ds.entities.values.length, 0); assert.equal(layer.getStats().count, 0); }); + +test('narrowing then returning to the previous window restores its reports without a duplicate source', async (t) => { + const { layer, viewer, mounted } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Today', '11/3/2026'), dated('Three days', '10/31/2026')], + }); + await layer.update(viewer); + assert.equal(layer.getStats().count, 2); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); + await ageChip(layer, 7).onClick(); + await layer.update(viewer); + assert.equal(layer.getStats().count, 2, '7d must show both reports again'); + assert.equal(mounted.size, 1); +}); + +test('two quick clicks, 1d then 7d, restore the report', async (t) => { + const at = Date.UTC(2026, 10, 3, 12); + const { layer, viewer } = fixture(t, { + now: () => at, + reports: [dated('Today', '11/3/2026'), dated('Three days', '10/31/2026')], + }); + await layer.update(viewer); + const a = ageChip(layer, 1).onClick(); + const b = ageChip(layer, 7).onClick(); + await a; + await b; + await layer.update(viewer); + assert.equal(layer.getStats().count, 2); +}); + +test('narrowing the window keeps the open card of a surviving marker', async (t) => { + let click; + const cards = []; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')], + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + await ageChip(layer, 30).onClick(); + assert.equal(layer.getStats().count, 2); + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + await ageChip(layer, 3).onClick(); + assert.equal(layer.getStats().count, 1); + assert.equal( + cards.at(-1).length, + 1, + 'card of the surviving marker stays open', + ); +}); + +test('a report dated yesterday is shown by 1d one hour into today (UTC)', async (t) => { + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 6, 1), + reports: [dated('Yesterday', '11/5/2026')], + }); + await layer.update(viewer); + await ageChip(layer, 1).onClick(); + assert.equal(layer.getStats().count, 1); +}); diff --git a/src/layers/ghostMaps/records.js b/src/layers/ghostMaps/records.js index 3c307fa63..62e48d3b3 100644 --- a/src/layers/ghostMaps/records.js +++ b/src/layers/ghostMaps/records.js @@ -73,7 +73,9 @@ export const GHOSTMAPS_DEFAULT_AGE_DAYS = 7; export const GHOSTMAPS_AGE_OPTIONS = Object.freeze([1, 3, 7, 14, 30, null]); /** ArcGIS US dates (optionally with a time) and ISO dates, interpreted in UTC - * when no zone is supplied. Reject calendar rollovers instead of guessing. */ + * when no zone is supplied. A date without a time means the whole UTC day, so + * it resolves to the day's last millisecond: "1d" still shows yesterday's + * reports. Reject calendar rollovers instead of guessing. */ export function parseGhostMapsEventDate(value) { const input = String(value ?? '').trim(); const us = @@ -113,7 +115,7 @@ export function parseGhostMapsEventDate(value) { const parsed = Date.parse(zoned); return Number.isFinite(parsed) ? parsed : null; } - return at; + return us?.[4] ? at : at + 86400_000 - 1; } /** The event's date, never its report/upload date or dates mentioned in prose. */ diff --git a/src/layers/ghostMaps/records.test.mjs b/src/layers/ghostMaps/records.test.mjs index a28d1f36d..f674fb1cb 100644 --- a/src/layers/ghostMaps/records.test.mjs +++ b/src/layers/ghostMaps/records.test.mjs @@ -14,15 +14,15 @@ import { test('event dates parse ArcGIS calendar dates and timestamps consistently in UTC', () => { for (const [value, iso] of [ - ['10/28/2026', '2026-10-28T00:00:00Z'], + ['10/28/2026', '2026-10-28T23:59:59.999Z'], ['10/28/2026 6:12:00 PM', '2026-10-28T18:12:00Z'], ['1/17/2026 9:46:41 AM', '2026-01-17T09:46:41Z'], ['10/28/2026 12:00 AM', '2026-10-28T00:00:00Z'], ['10/28/2026 12:00 PM', '2026-10-28T12:00:00Z'], - ['2026-10-28', '2026-10-28T00:00:00Z'], + ['2026-10-28', '2026-10-28T23:59:59.999Z'], ['2026-10-28T12:34:56', '2026-10-28T12:34:56Z'], ['2026-10-28T12:34:56.123-04:00', '2026-10-28T16:34:56.123Z'], - ['2/29/2024', '2024-02-29T00:00:00Z'], + ['2/29/2024', '2024-02-29T23:59:59.999Z'], ]) assert.equal(parseGhostMapsEventDate(value), Date.parse(iso), value); for (const value of [ @@ -52,7 +52,7 @@ test('event date fields take precedence over report dates and prose, even after row('Date', '10/30/2026') + row('Date of Event', '10/28/2026') }`; - assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 28)); + assert.equal(ghostMapsEventDate(html), Date.UTC(2026, 9, 29) - 1); assert.equal( ghostMapsEventDate( `${row('Date', '<Null>')}${row('Date Of Report', '11/2/2026')}
`, @@ -61,7 +61,7 @@ test('event date fields take precedence over report dates and prose, even after ); assert.equal( ghostMapsEventDate(`${row('Date_Of_Event', '10/28/2026')}
`), - Date.UTC(2026, 9, 28), + Date.UTC(2026, 9, 29) - 1, ); assert.equal(ghostMapsEventDate('Occurred 10/28/2026'), null); assert.equal( From 91ab9b5811c93cb66fbd292fff66f051e1854ed7 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Fri, 6 Nov 2026 17:37:29 +0100 Subject: [PATCH 4/4] fix(ghostmaps): follow-up review fixes for the age switch - Keep the open card across a widened reload only when its index still holds the same report; the server may answer a pinned version with newer bytes. - Narrowing to a window the loaded data already covers clears a failed wider reload, so the row no longer stays in error. - Label the every-age option "All" (contextual) so the six-option switch fits the row in every locale; tooltip and info line keep "All markers". Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 2 +- src/i18n/locales/ar.json | 1 + src/i18n/locales/de.json | 1 + src/i18n/locales/es.json | 1 + src/i18n/locales/fr.json | 1 + src/i18n/locales/it.json | 1 + src/i18n/locales/ja.json | 1 + src/i18n/locales/pt.json | 1 + src/i18n/locales/ru.json | 1 + src/i18n/locales/uk.json | 1 + src/i18n/locales/zh-CN.json | 1 + src/i18n/locales/zh-Hant.json | 1 + src/layers/ghostMaps/index.js | 24 +++++++++--- src/layers/ghostMaps/lifecycle.test.mjs | 51 +++++++++++++++++++++++++ 14 files changed, 81 insertions(+), 7 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fe837e421..0e1cd7889 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -6007,7 +6007,7 @@ request viewbox and continues searching without that geographic preference. - **What it shows.** Events ▸ **S2 GhostMaps** draws the current edition of S2 Underground's KMZ intel products from `github.com/s2underground/GhostMaps`. The products are the Common Intelligence Picture and the 30-day kinetic pull (both on by default), the Border Crisis Map, the Iran Common Intelligence Picture, Iran Kinetic Events, Yemen FLOT, and **Latest Drop**: the newest dated incident file outside those series, `0_Archive`, `Basedata` and `Reference Material`. Each product has a chip in the row. The chip choice is kept in `localStorage` (`gev.ghostmaps.products`) and falls back to the defaults when storage is blocked. - **Editions.** `src/layers/ghostMaps/catalog.js` (pure) matches whole repository paths per product. The newest edition wins by the date written in the file name, then by `Export_N`/`vN`. Undated names, such as the fixed 30-day file, are identified by blob sha. Files over 50 MiB are left out (the 62 MB census KMZ, for example). A renamed folder drops the product rather than matching elsewhere. -- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All markers**. The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. +- **Event age filter.** One **Event age** switch in the S2 GhostMaps row applies to every selected product: 1, 3, 7 (default), 14 or 30 days, or **All** (every marker). The choice is saved separately from product selection in `localStorage` (`gev.ghostmaps.ageDays`); missing, invalid or blocked storage falls back to seven days. Age comes only from the placemark's `Date of Event` (or `EventDate`/`Date`), never from report, prose or KMZ edition dates. Undated, unparseable and future dates stay visible. ArcGIS `M/D/YYYY` dates with optional AM/PM times and ISO dates are read as UTC when no zone is given; a date without a time covers its whole UTC day, so **1d** still shows yesterday's reports. Events age out every minute while the layer is on. Narrowing the window only removes markers; widening restores them from the cached KMZ. An open card stays while its marker is shown. - **Updates.** The repository is not ours, so there is no webhook. `server/providers/ghostMaps.js` starts with the first `/api/ghostmaps` request, so an install that never enables the layer never contacts GitHub. - It reads the branch head from git's smart-HTTP ref advertisement (`/info/refs?service=git-upload-pack`, about 0.5 KB, outside the REST rate limit) every `GHOSTMAPS_POLL_SECONDS` (15 s by default, 5 s minimum). Failures back off from 15 s to 5 min. - Only a new head costs REST calls: one recursive tree and one commit. An optional `GITHUB_TOKEN` raises the REST limit. diff --git a/src/i18n/locales/ar.json b/src/i18n/locales/ar.json index 99826e32c..f1d9903ed 100644 --- a/src/i18n/locales/ar.json +++ b/src/i18n/locales/ar.json @@ -3471,6 +3471,7 @@ "All markers": "كل العلامات", "Show markers of every age": "عرض العلامات من جميع الأعمار", "Maximum event age: {days}d; undated markers are kept": "الحد الأقصى لعمر الحدث: {days} يوم؛ تُحفظ العلامات غير المؤرخة", + "event age filter|All": "الكل", "Event age": "عمر الحدث", "Event age: {days}d · undated kept": "عمر الحدث: {days} يوم · الاحتفاظ بغير المؤرخ" } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index 826091ee8..50715c99f 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -3171,6 +3171,7 @@ "All markers": "Alle Markierungen", "Show markers of every age": "Markierungen jeden Alters anzeigen", "Maximum event age: {days}d; undated markers are kept": "Maximales Ereignisalter: {days} Tage; undatierte Markierungen bleiben", + "event age filter|All": "Alle", "Event age": "Ereignisalter", "Event age: {days}d · undated kept": "Ereignisalter: {days} Tage · undatierte bleiben" } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index 2f1cc02f1..6adb40da9 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -3246,6 +3246,7 @@ "All markers": "Todos los marcadores", "Show markers of every age": "Mostrar marcadores de cualquier antigüedad", "Maximum event age: {days}d; undated markers are kept": "Antigüedad máxima: {days} días; se conservan los marcadores sin fecha", + "event age filter|All": "Todos", "Event age": "Antigüedad", "Event age: {days}d · undated kept": "Antigüedad: {days} días · se conservan los marcadores sin fecha" } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index ff0b02c01..788210f89 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -3246,6 +3246,7 @@ "All markers": "Tous les marqueurs", "Show markers of every age": "Afficher les marqueurs de tout âge", "Maximum event age: {days}d; undated markers are kept": "Ancienneté maximale : {days} j ; marqueurs sans date conservés", + "event age filter|All": "Tous", "Event age": "Ancienneté", "Event age: {days}d · undated kept": "Ancienneté : {days} j · marqueurs sans date conservés" } diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index 4a26811fa..567c3c6ed 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -3246,6 +3246,7 @@ "All markers": "Tutti gli indicatori", "Show markers of every age": "Mostra gli indicatori di qualsiasi età", "Maximum event age: {days}d; undated markers are kept": "Età massima degli eventi: {days} giorni; indicatori senza data mantenuti", + "event age filter|All": "Tutti", "Event age": "Età degli eventi", "Event age: {days}d · undated kept": "Età degli eventi: {days} giorni · senza data mantenuti" } diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 9f111d91d..5b96cb3a8 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -3096,6 +3096,7 @@ "All markers": "すべてのマーカー", "Show markers of every age": "日付にかかわらずすべてのマーカーを表示", "Maximum event age: {days}d; undated markers are kept": "イベントの最大経過日数：{days}日。日付のないマーカーは保持", + "event age filter|All": "すべて", "Event age": "経過日数", "Event age: {days}d · undated kept": "経過日数：{days}日以内 · 日付なしは保持" } diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index ed83191d9..20bbea5aa 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -3246,6 +3246,7 @@ "All markers": "Todos os marcadores", "Show markers of every age": "Mostrar marcadores de qualquer idade", "Maximum event age: {days}d; undated markers are kept": "Idade máxima do evento: {days} dias; marcadores sem data mantidos", + "event age filter|All": "Todos", "Event age": "Idade do evento", "Event age: {days}d · undated kept": "Idade do evento: {days} dias · sem data mantidos" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index 82aa91f06..ca4813a5a 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -3321,6 +3321,7 @@ "All markers": "Все метки", "Show markers of every age": "Показать метки любого возраста", "Maximum event age: {days}d; undated markers are kept": "Максимальный возраст события: {days} дн.; метки без даты сохранены", + "event age filter|All": "Все", "Event age": "Возраст события", "Event age: {days}d · undated kept": "Возраст события: {days} дн. · метки без даты сохранены" } diff --git a/src/i18n/locales/uk.json b/src/i18n/locales/uk.json index 4d8ff9711..9b7ae1f61 100644 --- a/src/i18n/locales/uk.json +++ b/src/i18n/locales/uk.json @@ -3321,6 +3321,7 @@ "All markers": "Усі позначки", "Show markers of every age": "Показати позначки будь-якого віку", "Maximum event age: {days}d; undated markers are kept": "Максимальний вік події: {days} дн.; позначки без дати збережено", + "event age filter|All": "Усі", "Event age": "Вік події", "Event age: {days}d · undated kept": "Вік події: {days} дн. · позначки без дати збережено" } diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index a7cb60910..9b17dc66f 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -3096,6 +3096,7 @@ "All markers": "所有标记", "Show markers of every age": "显示所有时间的标记", "Maximum event age: {days}d; undated markers are kept": "事件最长保留时间：{days}天；保留无日期标记", + "event age filter|All": "全部", "Event age": "事件时间", "Event age: {days}d · undated kept": "事件时间：{days}天内 · 保留无日期标记" } diff --git a/src/i18n/locales/zh-Hant.json b/src/i18n/locales/zh-Hant.json index ebbf0680c..e3d9b716c 100644 --- a/src/i18n/locales/zh-Hant.json +++ b/src/i18n/locales/zh-Hant.json @@ -3096,6 +3096,7 @@ "All markers": "所有標記", "Show markers of every age": "顯示所有時間的標記", "Maximum event age: {days}d; undated markers are kept": "事件最長保留時間：{days}天；保留無日期標記", + "event age filter|All": "全部", "Event age": "事件時間", "Event age: {days}d · undated kept": "事件時間：{days}天內 · 保留無日期標記" } diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index 7d8716483..79971c715 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -423,9 +423,18 @@ export function createGhostMapsLayer({ // Cull before the first frame so far-side pins never flash. _horizon.update(next.pins); await _viewer.dataSources.add(next.dataSource); - // The new edition is on the globe before the old one leaves. A wider - // window over the same edition keeps indices, so the card stays. - unload(product.id, loaded?.version === next.version); + // The new edition is on the globe before the old one leaves. The card + // stays only if its index still holds the same report: the server + // may answer a pinned version with newer bytes. + const index = Number(_selectedKey?.split(':')[1]); + const before = loaded?.features.get(index); + const after = next.features.get(index); + unload( + product.id, + Boolean(before && after) && + before.name === after.name && + before.description === after.description, + ); _loaded.set(product.id, next); } catch (error) { if (signal.aborted) return; @@ -477,12 +486,15 @@ export function createGhostMapsLayer({ // Narrowing only removes markers. A product loaded with a narrower window // keeps its old ageDays, so sync() restores omitted placemarks from the // cached KMZ even though its upstream version has not changed. - for (const item of _loaded.values()) + for (const [id, item] of _loaded) if ( item.ageDays === null || (ageDays !== null && item.ageDays >= ageDays) - ) + ) { item.ageDays = ageDays; + // A failed wider reload no longer matters; a stale edition retries. + _failed.delete(id); + } notifyRow(); return sync(); } @@ -745,7 +757,7 @@ export function createGhostMapsLayer({ id: `age-${days ?? 'all'}`, label: days === null - ? t('All markers') + ? t('All', { context: 'event age filter' }) : t('{days}d', { days: formatNumber(days) }), title: days === null diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs index 7d1a067e6..ae9f63911 100644 --- a/src/layers/ghostMaps/lifecycle.test.mjs +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -558,3 +558,54 @@ test('a report dated yesterday is shown by 1d one hour into today (UTC)', async await ageChip(layer, 1).onClick(); assert.equal(layer.getStats().count, 1); }); + +test('a widened reload that returns a different report at the selected index closes its card', async (t) => { + let click; + const cards = []; + const reports = [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')]; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports, + controls: { + overlayHost: { setEntries: (_id, entries) => cards.push(entries) }, + screenSpaceEventHandlerFactory: () => ({ + setInputAction: (fn) => { + click = fn; + }, + destroy() {}, + }), + picking: { resolvePickId: (picked) => picked }, + }, + }); + await layer.update(viewer); + viewer.scene.pick = () => 'ghostmaps:cip:0:0'; + click({ position: { x: 0, y: 0 } }); + assert.equal(cards.at(-1).length, 1); + // The server answers the pinned version with a newer edition's bytes. + reports[0] = dated('Replacement', '11/2/2026'); + await ageChip(layer, 30).onClick(); + assert.equal(layer.getStats().count, 2); + assert.equal( + cards.at(-1).length, + 0, + 'a different report must not keep the old card', + ); +}); + +test('narrowing to a window the loaded data covers clears a failed wider reload', async (t) => { + let fail = false; + const { layer, viewer } = fixture(t, { + now: () => Date.UTC(2026, 10, 3, 12), + reports: [dated('Recent', '11/2/2026'), dated('Older', '10/20/2026')], + beforeFileReturns: () => { + if (fail) throw new Error('network down'); + }, + }); + await layer.update(viewer); + fail = true; + await ageChip(layer, 30).onClick(); + assert.ok(layer.getStats().error, 'the wider reload failed'); + await ageChip(layer, 3).onClick(); + assert.equal(layer.getStats().error, null); + assert.equal(layer.getStats().count, 1); +});