Merge main and preserve voice panel and GhostMaps schema coverage

This commit is contained in:
Sameh Khamis committed 2026-10-26 08:52:03 -07:00
commit c533b1994c
40 files changed
+3601 -52

No files matched your search

+437
View File
@@ -0,0 +1,437 @@
import fsp from 'node:fs/promises';
import path from 'node:path';
import {
GHOSTMAPS_MAX_FILE_BYTES,
GHOSTMAPS_REPO,
buildGhostMapsCatalog,
ghostMapsRawUrl,
parseGitRefAdvertisement,
} from '../../src/layers/ghostMaps/catalog.js';
import {
readResponseBytesCapped,
readResponseJsonCapped,
readResponseTextCapped,
} from './common/http.js';
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
// The repository is not ours, so there is no webhook: the watcher reads the
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
// REST API rate limit) every few seconds, and only a new commit costs REST
// calls (tree + commit). Browsers hear about a new commit over server-sent
// events and fetch only the products whose blob changed. Files are fetched by
// commit-pinned raw URLs and kept on disk by blob sha.
const DEFAULT_POLL_MS = 15_000;
const MIN_POLL_MS = 5_000;
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
const TIMEOUT_MS = 15_000;
const FILE_TIMEOUT_MS = 120_000;
const HEARTBEAT_MS = 25_000;
const MAX_SUBSCRIBERS = 64;
const MAX_REF_BYTES = 64 * 1024;
const MAX_TREE_BYTES = 16 * 1024 * 1024;
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
const seconds = Number(raw);
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
}
/**
* The repository watcher and file cache, independent of HTTP routing.
* @returns {{start: Function, stop: Function, catalog: Function,
* checkNow: Function, file: Function, subscribe: Function}}
*/
export function createGhostMapsWatcher({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
pollMs = ghostMapsPollMs(),
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
log = (...args) => console.log(...args),
warn = (...args) => console.warn(...args),
} = {}) {
let state = {
commit: null,
committedAt: null,
message: null,
products: [],
checkedAt: null,
changedAt: null,
error: null,
};
let timer = null;
let running = false;
let checking = null;
let failures = 0;
let diskLoaded = false;
const listeners = new Set();
const downloads = new Map();
const apiHeaders = () => ({
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
});
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
const response = await fetchImpl(url, {
headers,
redirect: 'follow',
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
status: response.status,
});
}
return response;
}
async function readHead() {
const response = await get(
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
{ 'User-Agent': USER_AGENT },
);
const sha = parseGitRefAdvertisement(
await readResponseTextCapped(response, MAX_REF_BYTES),
);
if (!sha) throw new Error('GhostMaps branch head not advertised');
return sha;
}
async function readCommit(sha) {
const api = `https://api.github.com/repos/${REPO_PATH}`;
const tree = await readResponseJsonCapped(
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
MAX_TREE_BYTES,
);
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
let committedAt = null;
let message = null;
try {
const commit = await readResponseJsonCapped(
await get(`${api}/commits/${sha}`, apiHeaders()),
MAX_COMMIT_BYTES,
);
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
message = String(commit?.commit?.message || '')
.split('\n')[0]
.slice(0, 200);
} catch {
// The commit date is decoration; the tree is what matters.
}
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
}
async function loadDisk() {
if (diskLoaded) return;
diskLoaded = true;
try {
const saved = JSON.parse(
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
);
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
state = { ...state, ...saved, error: null };
} catch {
/* first run */
}
}
async function saveDisk() {
try {
await fsp.mkdir(cacheDir, { recursive: true });
const { commit, committedAt, message, products, changedAt } = state;
await fsp.writeFile(
path.join(cacheDir, 'catalog.json'),
JSON.stringify({ commit, committedAt, message, products, changedAt }),
);
} catch (error) {
warn('[ghostmaps] could not save catalog:', error.message);
}
}
function emit() {
const event = publicCatalog();
for (const listener of listeners) {
try {
listener(event);
} catch {
/* a broken subscriber must not stop the rest */
}
}
}
async function check() {
await loadDisk();
const head = await readHead();
state.checkedAt = now();
if (head === state.commit) {
if (state.error !== null) {
state.error = null;
emit();
}
return false;
}
const next = await readCommit(head);
const previous = new Map(state.products.map((p) => [p.id, p.version]));
const changed = next.products
.filter((p) => previous.get(p.id) !== p.version)
.map((p) => p.id);
state = {
...state,
...next,
commit: head,
changedAt: now(),
error: null,
};
log(
`[ghostmaps] commit ${head.slice(0, 8)}${
next.message ? ` "${next.message}"` : ''
}; updated: ${changed.join(', ') || 'none'}`,
);
await saveDisk();
emit();
// Warm the default products so the first browser load is local.
for (const product of state.products)
if (product.defaultOn) file(product.id).catch(() => {});
return true;
}
/** Run one check now, coalescing with one already in flight. */
function checkNow() {
if (!checking)
checking = check()
.then(
(changed) => {
failures = 0;
return changed;
},
(error) => {
failures += 1;
state.error = error.message;
state.checkedAt = now();
warn('[ghostmaps] check failed:', error.message);
emit();
return false;
},
)
.finally(() => {
checking = null;
});
return checking;
}
function schedule() {
clearTimeout(timer);
if (!running) return;
const delay = failures
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
: pollMs;
timer = setTimeout(async () => {
await checkNow();
schedule();
}, delay);
timer.unref?.();
}
/** Start watching (idempotent); the first check runs immediately. */
function start() {
if (running) return checking || Promise.resolve(false);
running = true;
const first = checkNow();
first.then(schedule);
return first;
}
function stop() {
running = false;
clearTimeout(timer);
timer = null;
}
function publicCatalog() {
return {
repo: GHOSTMAPS_REPO.url,
branch: GHOSTMAPS_REPO.branch,
commit: state.commit,
committedAt: state.committedAt,
message: state.message,
checkedAt: state.checkedAt,
changedAt: state.changedAt,
pollMs,
error: state.error,
products: state.products.map(
({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}) => ({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}),
),
};
}
/**
* The bytes of one product's current edition, from disk or a commit-pinned
* download. Only catalog paths are ever fetched.
*/
async function file(id) {
const product = state.products.find((p) => p.id === id);
if (!product) return null;
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
try {
return { product, bytes: await fsp.readFile(target) };
} catch {
/* not cached yet */
}
const commit = state.commit;
let pending = downloads.get(product.version);
if (!pending) {
pending = (async () => {
const response = await get(
ghostMapsRawUrl(commit, product.path),
{ 'User-Agent': USER_AGENT },
FILE_TIMEOUT_MS,
);
const bytes = Buffer.from(
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
);
await fsp.mkdir(path.dirname(target), { recursive: true });
const temp = `${target}.${process.pid}.tmp`;
await fsp.writeFile(temp, bytes);
await fsp.rename(temp, target);
await pruneFiles();
return bytes;
})().finally(() => downloads.delete(product.version));
downloads.set(product.version, pending);
}
return { product, bytes: await pending };
}
/** Keep only files that a current product still points to. */
async function pruneFiles() {
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
const dir = path.join(cacheDir, 'files');
for (const name of await fsp.readdir(dir).catch(() => [])) {
if (!keep.has(name) && !name.endsWith('.tmp'))
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
}
}
function subscribe(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
}
return {
start,
stop,
checkNow,
catalog: publicCatalog,
file,
subscribe,
subscribers: () => listeners.size,
};
}
/** Same-origin GhostMaps routes for dev and preview. */
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
const json = (res, status, value) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
});
res.end(JSON.stringify(value));
};
function events(req, res) {
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
return json(res, 429, { error: 'too_many_subscribers' });
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-store',
Connection: 'keep-alive',
'X-Accel-Buffering': 'no',
});
const send = (catalog) =>
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
res.write('retry: 5000\n\n');
send(watcher.catalog());
const unsubscribe = watcher.subscribe(send);
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
heartbeat.unref?.();
const close = () => {
clearInterval(heartbeat);
unsubscribe();
};
req.on('close', close);
res.on('error', close);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const url = new URL(req.url || '/', 'http://local');
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
if (!['/catalog', '/events'].includes(url.pathname) && !match)
return json(res, 404, { error: 'unknown_route' });
// Watching starts with the first request, so an install that never turns
// the layer on never contacts GitHub.
const first = watcher.start();
if (url.pathname === '/catalog') {
if (!watcher.catalog().commit) await first;
return json(res, 200, watcher.catalog());
}
if (url.pathname === '/events') return events(req, res);
try {
if (!watcher.catalog().commit) await first;
const found = await watcher.file(match[1]);
if (!found) return json(res, 404, { error: 'unknown_product' });
const pinned = url.searchParams.get('v') === found.product.version;
res.writeHead(200, {
'Content-Type': 'application/vnd.google-earth.kmz',
'Content-Length': found.bytes.length,
'X-GhostMaps-Version': found.product.version,
// A versioned URL names one immutable blob.
'Cache-Control': pinned
? 'private, max-age=31536000, immutable'
: 'no-store',
});
return res.end(found.bytes);
} catch {
return json(res, 502, { error: 'ghostmaps_unavailable' });
}
}
return {
name: 'ghostmaps',
configureServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
};
}
+294
View File
@@ -0,0 +1,294 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdtemp, readdir, rm } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
createGhostMapsWatcher,
ghostMapsPollMs,
ghostMapsProxy,
} from './ghostMaps.js';
const MASTER =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
const KINETIC =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
const sha = (c) => c.repeat(40);
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
function fakeGitHub() {
const repo = { head: sha('1'), trees: new Map(), calls: [] };
repo.trees.set(sha('1'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('a'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
const body = (text, status = 200) =>
new Response(text, {
status,
headers: { 'content-length': String(Buffer.byteLength(text)) },
});
const fetchImpl = async (url) => {
repo.calls.push(url);
if (url.endsWith('/info/refs?service=git-upload-pack'))
return body(
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
);
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
if (/\/commits\//.test(url))
return body(
JSON.stringify({
commit: {
message: 'Oct 18 Update\nmore',
committer: { date: '2026-10-18T00:00:00Z' },
},
}),
);
if (url.startsWith('https://raw.githubusercontent.com/'))
return body('KMZ!');
return body('nope', 404);
};
return { repo, fetchImpl };
}
async function withCache(fn) {
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
try {
await fn(dir);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
// checkNow announces the catalog before background warm-up finishes. Join the
// same file reads before changing editions or removing the temporary cache.
async function warmDefaults(watcher) {
await Promise.all(
watcher
.catalog()
.products.filter((product) => product.defaultOn)
.map((product) => watcher.file(product.id)),
);
}
test('a new commit is announced once, naming only the products that changed', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
const first = watcher.catalog();
assert.equal(first.commit, sha('1'));
assert.equal(first.message, 'Oct 18 Update');
assert.deepEqual(
first.products.map((p) => p.id),
['cip', 'kinetic-30d'],
);
assert.equal(events.length, 1);
// Same head: one ref read, no REST calls, no event.
repo.calls.length = 0;
assert.equal(await watcher.checkNow(), false);
assert.equal(repo.calls.length, 1);
assert.equal(events.length, 1);
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
repo.head = sha('2');
repo.trees.set(sha('2'), [
...repo.trees.get(sha('1')),
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
]);
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
assert.equal(events.length, 2);
const cip = events[1].products.find((p) => p.id === 'cip');
assert.equal(cip.version, sha('c'));
assert.match(cip.fileName, /October_18/);
assert.equal(
events[1].products.find((p) => p.id === 'kinetic-30d').version,
sha('b'),
);
// Repository paths stay on the server.
assert.equal('path' in cip, false);
watcher.stop();
}));
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
await watcher.checkNow();
const found = await watcher.file('kinetic-30d');
assert.equal(found.bytes.toString(), 'KMZ!');
const raw = repo.calls.filter((url) =>
url.includes('raw.githubusercontent'),
);
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
assert.ok(
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
);
const before = repo.calls.length;
await watcher.file('kinetic-30d');
assert.equal(repo.calls.length, before, 'second read comes from disk');
assert.equal(await watcher.file('not-a-product'), null);
// When the edition changes, the old blob leaves the disk cache.
await watcher.file('cip');
repo.head = sha('2');
repo.trees.set(sha('2'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
await watcher.checkNow();
await watcher.file('cip');
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
watcher.stop();
}));
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
const quiet = { log: () => {}, warn: () => {} };
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
await online.checkNow();
await warmDefaults(online);
const offline = createGhostMapsWatcher({
fetchImpl: async () => new Response('down', { status: 503 }),
cacheDir,
...quiet,
});
assert.equal(await offline.checkNow(), false);
const catalog = offline.catalog();
assert.equal(catalog.commit, sha('1'));
assert.equal(catalog.products.length, 2);
assert.match(catalog.error, /HTTP 503/);
}));
test('an unchanged head clears an upstream outage and announces recovery once', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
let unavailable = false;
let checkedAt = 1000;
const requests = [];
const watcher = createGhostMapsWatcher({
cacheDir,
now: () => checkedAt,
fetchImpl: async (url, options) => {
requests.push(url);
if (unavailable) return new Response('down', { status: 503 });
return fetchImpl(url, options);
},
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
await watcher.checkNow();
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
const healthy = watcher.catalog();
assert.equal(events.length, 1);
unavailable = true;
checkedAt = 2000;
assert.equal(await watcher.checkNow(), false);
assert.match(watcher.catalog().error, /HTTP 503/);
assert.equal(events.length, 2);
unavailable = false;
checkedAt = 3000;
requests.length = 0;
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
const recovered = watcher.catalog();
assert.equal(recovered.error, null);
assert.equal(recovered.checkedAt, checkedAt);
assert.equal(recovered.commit, healthy.commit);
assert.equal(recovered.changedAt, healthy.changedAt);
assert.deepEqual(recovered.products, healthy.products);
assert.equal(requests.length, 1, 'recovery needs only the ref read');
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
assert.equal(events[2].error, null);
assert.equal(events[2].checkedAt, checkedAt);
checkedAt = 4000;
assert.equal(await watcher.checkNow(), false);
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
watcher.stop();
}));
test('poll interval is configurable with a 5 s floor', () => {
assert.equal(ghostMapsPollMs(undefined), 15000);
assert.equal(ghostMapsPollMs('30'), 30000);
assert.equal(ghostMapsPollMs('1'), 5000);
assert.equal(ghostMapsPollMs('soon'), 15000);
});
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
let started = 0;
const watcher = {
start: () => {
started += 1;
return Promise.resolve(false);
},
catalog: () => ({ commit: sha('1'), products: [] }),
file: async () => null,
subscribe: () => () => {},
subscribers: () => 0,
};
let handler;
ghostMapsProxy({ watcher }).configureServer({
middlewares: { use: (_route, fn) => (handler = fn) },
});
const call = async (method, url) => {
const res = {
destroyed: false,
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end(body) {
this.body = String(body ?? '');
},
};
await handler({ method, url }, res);
return res;
};
assert.equal((await call('GET', '/nope')).status, 404);
assert.equal((await call('POST', '/catalog')).status, 405);
assert.equal((await call('GET', '/file/../../etc')).status, 404);
assert.equal(started, 0);
assert.equal((await call('GET', '/file/cip')).status, 404);
const catalog = await call('GET', '/catalog');
assert.equal(catalog.status, 200);
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
assert.equal(started, 2);
});
+2
View File
@@ -26,6 +26,7 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { ghostMapsProxy } from './ghostMaps.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
@@ -64,6 +65,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
ghostMapsProxy(),
keySetupEndpoint(),
];
}