mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 02:14:01 +00:00
Merge main and preserve voice panel and GhostMaps schema coverage
This commit is contained in:
commit
c533b1994c
40 files changed
+3601
-52
No files matched your search
@@ -0,0 +1,437 @@
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
GHOSTMAPS_MAX_FILE_BYTES,
|
||||
GHOSTMAPS_REPO,
|
||||
buildGhostMapsCatalog,
|
||||
ghostMapsRawUrl,
|
||||
parseGitRefAdvertisement,
|
||||
} from '../../src/layers/ghostMaps/catalog.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from './common/http.js';
|
||||
|
||||
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
|
||||
// The repository is not ours, so there is no webhook: the watcher reads the
|
||||
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
|
||||
// REST API rate limit) every few seconds, and only a new commit costs REST
|
||||
// calls (tree + commit). Browsers hear about a new commit over server-sent
|
||||
// events and fetch only the products whose blob changed. Files are fetched by
|
||||
// commit-pinned raw URLs and kept on disk by blob sha.
|
||||
|
||||
const DEFAULT_POLL_MS = 15_000;
|
||||
const MIN_POLL_MS = 5_000;
|
||||
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const FILE_TIMEOUT_MS = 120_000;
|
||||
const HEARTBEAT_MS = 25_000;
|
||||
const MAX_SUBSCRIBERS = 64;
|
||||
const MAX_REF_BYTES = 64 * 1024;
|
||||
const MAX_TREE_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
|
||||
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
|
||||
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
|
||||
|
||||
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
|
||||
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
|
||||
const seconds = Number(raw);
|
||||
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
|
||||
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* The repository watcher and file cache, independent of HTTP routing.
|
||||
* @returns {{start: Function, stop: Function, catalog: Function,
|
||||
* checkNow: Function, file: Function, subscribe: Function}}
|
||||
*/
|
||||
export function createGhostMapsWatcher({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
pollMs = ghostMapsPollMs(),
|
||||
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
|
||||
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
|
||||
log = (...args) => console.log(...args),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
let state = {
|
||||
commit: null,
|
||||
committedAt: null,
|
||||
message: null,
|
||||
products: [],
|
||||
checkedAt: null,
|
||||
changedAt: null,
|
||||
error: null,
|
||||
};
|
||||
let timer = null;
|
||||
let running = false;
|
||||
let checking = null;
|
||||
let failures = 0;
|
||||
let diskLoaded = false;
|
||||
const listeners = new Set();
|
||||
const downloads = new Map();
|
||||
|
||||
const apiHeaders = () => ({
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': USER_AGENT,
|
||||
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
|
||||
});
|
||||
|
||||
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
|
||||
const response = await fetchImpl(url, {
|
||||
headers,
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function readHead() {
|
||||
const response = await get(
|
||||
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
);
|
||||
const sha = parseGitRefAdvertisement(
|
||||
await readResponseTextCapped(response, MAX_REF_BYTES),
|
||||
);
|
||||
if (!sha) throw new Error('GhostMaps branch head not advertised');
|
||||
return sha;
|
||||
}
|
||||
|
||||
async function readCommit(sha) {
|
||||
const api = `https://api.github.com/repos/${REPO_PATH}`;
|
||||
const tree = await readResponseJsonCapped(
|
||||
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
|
||||
MAX_TREE_BYTES,
|
||||
);
|
||||
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
|
||||
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
|
||||
let committedAt = null;
|
||||
let message = null;
|
||||
try {
|
||||
const commit = await readResponseJsonCapped(
|
||||
await get(`${api}/commits/${sha}`, apiHeaders()),
|
||||
MAX_COMMIT_BYTES,
|
||||
);
|
||||
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
|
||||
message = String(commit?.commit?.message || '')
|
||||
.split('\n')[0]
|
||||
.slice(0, 200);
|
||||
} catch {
|
||||
// The commit date is decoration; the tree is what matters.
|
||||
}
|
||||
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
|
||||
}
|
||||
|
||||
async function loadDisk() {
|
||||
if (diskLoaded) return;
|
||||
diskLoaded = true;
|
||||
try {
|
||||
const saved = JSON.parse(
|
||||
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
|
||||
);
|
||||
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
|
||||
state = { ...state, ...saved, error: null };
|
||||
} catch {
|
||||
/* first run */
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDisk() {
|
||||
try {
|
||||
await fsp.mkdir(cacheDir, { recursive: true });
|
||||
const { commit, committedAt, message, products, changedAt } = state;
|
||||
await fsp.writeFile(
|
||||
path.join(cacheDir, 'catalog.json'),
|
||||
JSON.stringify({ commit, committedAt, message, products, changedAt }),
|
||||
);
|
||||
} catch (error) {
|
||||
warn('[ghostmaps] could not save catalog:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function emit() {
|
||||
const event = publicCatalog();
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
listener(event);
|
||||
} catch {
|
||||
/* a broken subscriber must not stop the rest */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function check() {
|
||||
await loadDisk();
|
||||
const head = await readHead();
|
||||
state.checkedAt = now();
|
||||
if (head === state.commit) {
|
||||
if (state.error !== null) {
|
||||
state.error = null;
|
||||
emit();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const next = await readCommit(head);
|
||||
const previous = new Map(state.products.map((p) => [p.id, p.version]));
|
||||
const changed = next.products
|
||||
.filter((p) => previous.get(p.id) !== p.version)
|
||||
.map((p) => p.id);
|
||||
state = {
|
||||
...state,
|
||||
...next,
|
||||
commit: head,
|
||||
changedAt: now(),
|
||||
error: null,
|
||||
};
|
||||
log(
|
||||
`[ghostmaps] commit ${head.slice(0, 8)}${
|
||||
next.message ? ` "${next.message}"` : ''
|
||||
}; updated: ${changed.join(', ') || 'none'}`,
|
||||
);
|
||||
await saveDisk();
|
||||
emit();
|
||||
// Warm the default products so the first browser load is local.
|
||||
for (const product of state.products)
|
||||
if (product.defaultOn) file(product.id).catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Run one check now, coalescing with one already in flight. */
|
||||
function checkNow() {
|
||||
if (!checking)
|
||||
checking = check()
|
||||
.then(
|
||||
(changed) => {
|
||||
failures = 0;
|
||||
return changed;
|
||||
},
|
||||
(error) => {
|
||||
failures += 1;
|
||||
state.error = error.message;
|
||||
state.checkedAt = now();
|
||||
warn('[ghostmaps] check failed:', error.message);
|
||||
emit();
|
||||
return false;
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
checking = null;
|
||||
});
|
||||
return checking;
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
clearTimeout(timer);
|
||||
if (!running) return;
|
||||
const delay = failures
|
||||
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
|
||||
: pollMs;
|
||||
timer = setTimeout(async () => {
|
||||
await checkNow();
|
||||
schedule();
|
||||
}, delay);
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
/** Start watching (idempotent); the first check runs immediately. */
|
||||
function start() {
|
||||
if (running) return checking || Promise.resolve(false);
|
||||
running = true;
|
||||
const first = checkNow();
|
||||
first.then(schedule);
|
||||
return first;
|
||||
}
|
||||
|
||||
function stop() {
|
||||
running = false;
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
|
||||
function publicCatalog() {
|
||||
return {
|
||||
repo: GHOSTMAPS_REPO.url,
|
||||
branch: GHOSTMAPS_REPO.branch,
|
||||
commit: state.commit,
|
||||
committedAt: state.committedAt,
|
||||
message: state.message,
|
||||
checkedAt: state.checkedAt,
|
||||
changedAt: state.changedAt,
|
||||
pollMs,
|
||||
error: state.error,
|
||||
products: state.products.map(
|
||||
({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}) => ({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes of one product's current edition, from disk or a commit-pinned
|
||||
* download. Only catalog paths are ever fetched.
|
||||
*/
|
||||
async function file(id) {
|
||||
const product = state.products.find((p) => p.id === id);
|
||||
if (!product) return null;
|
||||
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
|
||||
try {
|
||||
return { product, bytes: await fsp.readFile(target) };
|
||||
} catch {
|
||||
/* not cached yet */
|
||||
}
|
||||
const commit = state.commit;
|
||||
let pending = downloads.get(product.version);
|
||||
if (!pending) {
|
||||
pending = (async () => {
|
||||
const response = await get(
|
||||
ghostMapsRawUrl(commit, product.path),
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
FILE_TIMEOUT_MS,
|
||||
);
|
||||
const bytes = Buffer.from(
|
||||
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
|
||||
);
|
||||
await fsp.mkdir(path.dirname(target), { recursive: true });
|
||||
const temp = `${target}.${process.pid}.tmp`;
|
||||
await fsp.writeFile(temp, bytes);
|
||||
await fsp.rename(temp, target);
|
||||
await pruneFiles();
|
||||
return bytes;
|
||||
})().finally(() => downloads.delete(product.version));
|
||||
downloads.set(product.version, pending);
|
||||
}
|
||||
return { product, bytes: await pending };
|
||||
}
|
||||
|
||||
/** Keep only files that a current product still points to. */
|
||||
async function pruneFiles() {
|
||||
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
|
||||
const dir = path.join(cacheDir, 'files');
|
||||
for (const name of await fsp.readdir(dir).catch(() => [])) {
|
||||
if (!keep.has(name) && !name.endsWith('.tmp'))
|
||||
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
checkNow,
|
||||
catalog: publicCatalog,
|
||||
file,
|
||||
subscribe,
|
||||
subscribers: () => listeners.size,
|
||||
};
|
||||
}
|
||||
|
||||
/** Same-origin GhostMaps routes for dev and preview. */
|
||||
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
|
||||
const json = (res, status, value) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
|
||||
function events(req, res) {
|
||||
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
|
||||
return json(res, 429, { error: 'too_many_subscribers' });
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-store',
|
||||
Connection: 'keep-alive',
|
||||
'X-Accel-Buffering': 'no',
|
||||
});
|
||||
const send = (catalog) =>
|
||||
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
|
||||
res.write('retry: 5000\n\n');
|
||||
send(watcher.catalog());
|
||||
const unsubscribe = watcher.subscribe(send);
|
||||
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
|
||||
heartbeat.unref?.();
|
||||
const close = () => {
|
||||
clearInterval(heartbeat);
|
||||
unsubscribe();
|
||||
};
|
||||
req.on('close', close);
|
||||
res.on('error', close);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
const url = new URL(req.url || '/', 'http://local');
|
||||
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
|
||||
if (!['/catalog', '/events'].includes(url.pathname) && !match)
|
||||
return json(res, 404, { error: 'unknown_route' });
|
||||
// Watching starts with the first request, so an install that never turns
|
||||
// the layer on never contacts GitHub.
|
||||
const first = watcher.start();
|
||||
if (url.pathname === '/catalog') {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
return json(res, 200, watcher.catalog());
|
||||
}
|
||||
if (url.pathname === '/events') return events(req, res);
|
||||
try {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
const found = await watcher.file(match[1]);
|
||||
if (!found) return json(res, 404, { error: 'unknown_product' });
|
||||
const pinned = url.searchParams.get('v') === found.product.version;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/vnd.google-earth.kmz',
|
||||
'Content-Length': found.bytes.length,
|
||||
'X-GhostMaps-Version': found.product.version,
|
||||
// A versioned URL names one immutable blob.
|
||||
'Cache-Control': pinned
|
||||
? 'private, max-age=31536000, immutable'
|
||||
: 'no-store',
|
||||
});
|
||||
return res.end(found.bytes);
|
||||
} catch {
|
||||
return json(res, 502, { error: 'ghostmaps_unavailable' });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'ghostmaps',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readdir, rm } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
createGhostMapsWatcher,
|
||||
ghostMapsPollMs,
|
||||
ghostMapsProxy,
|
||||
} from './ghostMaps.js';
|
||||
|
||||
const MASTER =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
|
||||
const KINETIC =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
|
||||
const sha = (c) => c.repeat(40);
|
||||
|
||||
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
|
||||
function fakeGitHub() {
|
||||
const repo = { head: sha('1'), trees: new Map(), calls: [] };
|
||||
repo.trees.set(sha('1'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('a'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
const body = (text, status = 200) =>
|
||||
new Response(text, {
|
||||
status,
|
||||
headers: { 'content-length': String(Buffer.byteLength(text)) },
|
||||
});
|
||||
const fetchImpl = async (url) => {
|
||||
repo.calls.push(url);
|
||||
if (url.endsWith('/info/refs?service=git-upload-pack'))
|
||||
return body(
|
||||
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
|
||||
);
|
||||
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
|
||||
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
|
||||
if (/\/commits\//.test(url))
|
||||
return body(
|
||||
JSON.stringify({
|
||||
commit: {
|
||||
message: 'Oct 18 Update\nmore',
|
||||
committer: { date: '2026-10-18T00:00:00Z' },
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (url.startsWith('https://raw.githubusercontent.com/'))
|
||||
return body('KMZ!');
|
||||
return body('nope', 404);
|
||||
};
|
||||
return { repo, fetchImpl };
|
||||
}
|
||||
|
||||
async function withCache(fn) {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
|
||||
try {
|
||||
await fn(dir);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// checkNow announces the catalog before background warm-up finishes. Join the
|
||||
// same file reads before changing editions or removing the temporary cache.
|
||||
async function warmDefaults(watcher) {
|
||||
await Promise.all(
|
||||
watcher
|
||||
.catalog()
|
||||
.products.filter((product) => product.defaultOn)
|
||||
.map((product) => watcher.file(product.id)),
|
||||
);
|
||||
}
|
||||
|
||||
test('a new commit is announced once, naming only the products that changed', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
const first = watcher.catalog();
|
||||
assert.equal(first.commit, sha('1'));
|
||||
assert.equal(first.message, 'Oct 18 Update');
|
||||
assert.deepEqual(
|
||||
first.products.map((p) => p.id),
|
||||
['cip', 'kinetic-30d'],
|
||||
);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// Same head: one ref read, no REST calls, no event.
|
||||
repo.calls.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(repo.calls.length, 1);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
...repo.trees.get(sha('1')),
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
]);
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
assert.equal(events.length, 2);
|
||||
const cip = events[1].products.find((p) => p.id === 'cip');
|
||||
assert.equal(cip.version, sha('c'));
|
||||
assert.match(cip.fileName, /October_18/);
|
||||
assert.equal(
|
||||
events[1].products.find((p) => p.id === 'kinetic-30d').version,
|
||||
sha('b'),
|
||||
);
|
||||
// Repository paths stay on the server.
|
||||
assert.equal('path' in cip, false);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
await watcher.checkNow();
|
||||
const found = await watcher.file('kinetic-30d');
|
||||
assert.equal(found.bytes.toString(), 'KMZ!');
|
||||
const raw = repo.calls.filter((url) =>
|
||||
url.includes('raw.githubusercontent'),
|
||||
);
|
||||
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
|
||||
assert.ok(
|
||||
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
|
||||
);
|
||||
const before = repo.calls.length;
|
||||
await watcher.file('kinetic-30d');
|
||||
assert.equal(repo.calls.length, before, 'second read comes from disk');
|
||||
assert.equal(await watcher.file('not-a-product'), null);
|
||||
|
||||
// When the edition changes, the old blob leaves the disk cache.
|
||||
await watcher.file('cip');
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
await watcher.checkNow();
|
||||
await watcher.file('cip');
|
||||
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
|
||||
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
const quiet = { log: () => {}, warn: () => {} };
|
||||
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
|
||||
await online.checkNow();
|
||||
await warmDefaults(online);
|
||||
const offline = createGhostMapsWatcher({
|
||||
fetchImpl: async () => new Response('down', { status: 503 }),
|
||||
cacheDir,
|
||||
...quiet,
|
||||
});
|
||||
assert.equal(await offline.checkNow(), false);
|
||||
const catalog = offline.catalog();
|
||||
assert.equal(catalog.commit, sha('1'));
|
||||
assert.equal(catalog.products.length, 2);
|
||||
assert.match(catalog.error, /HTTP 503/);
|
||||
}));
|
||||
|
||||
test('an unchanged head clears an upstream outage and announces recovery once', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
let unavailable = false;
|
||||
let checkedAt = 1000;
|
||||
const requests = [];
|
||||
const watcher = createGhostMapsWatcher({
|
||||
cacheDir,
|
||||
now: () => checkedAt,
|
||||
fetchImpl: async (url, options) => {
|
||||
requests.push(url);
|
||||
if (unavailable) return new Response('down', { status: 503 });
|
||||
return fetchImpl(url, options);
|
||||
},
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
await watcher.checkNow();
|
||||
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
|
||||
const healthy = watcher.catalog();
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
unavailable = true;
|
||||
checkedAt = 2000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.match(watcher.catalog().error, /HTTP 503/);
|
||||
assert.equal(events.length, 2);
|
||||
|
||||
unavailable = false;
|
||||
checkedAt = 3000;
|
||||
requests.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
|
||||
const recovered = watcher.catalog();
|
||||
assert.equal(recovered.error, null);
|
||||
assert.equal(recovered.checkedAt, checkedAt);
|
||||
assert.equal(recovered.commit, healthy.commit);
|
||||
assert.equal(recovered.changedAt, healthy.changedAt);
|
||||
assert.deepEqual(recovered.products, healthy.products);
|
||||
assert.equal(requests.length, 1, 'recovery needs only the ref read');
|
||||
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
|
||||
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
|
||||
assert.equal(events[2].error, null);
|
||||
assert.equal(events[2].checkedAt, checkedAt);
|
||||
|
||||
checkedAt = 4000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('poll interval is configurable with a 5 s floor', () => {
|
||||
assert.equal(ghostMapsPollMs(undefined), 15000);
|
||||
assert.equal(ghostMapsPollMs('30'), 30000);
|
||||
assert.equal(ghostMapsPollMs('1'), 5000);
|
||||
assert.equal(ghostMapsPollMs('soon'), 15000);
|
||||
});
|
||||
|
||||
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
|
||||
let started = 0;
|
||||
const watcher = {
|
||||
start: () => {
|
||||
started += 1;
|
||||
return Promise.resolve(false);
|
||||
},
|
||||
catalog: () => ({ commit: sha('1'), products: [] }),
|
||||
file: async () => null,
|
||||
subscribe: () => () => {},
|
||||
subscribers: () => 0,
|
||||
};
|
||||
let handler;
|
||||
ghostMapsProxy({ watcher }).configureServer({
|
||||
middlewares: { use: (_route, fn) => (handler = fn) },
|
||||
});
|
||||
const call = async (method, url) => {
|
||||
const res = {
|
||||
destroyed: false,
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end(body) {
|
||||
this.body = String(body ?? '');
|
||||
},
|
||||
};
|
||||
await handler({ method, url }, res);
|
||||
return res;
|
||||
};
|
||||
assert.equal((await call('GET', '/nope')).status, 404);
|
||||
assert.equal((await call('POST', '/catalog')).status, 405);
|
||||
assert.equal((await call('GET', '/file/../../etc')).status, 404);
|
||||
assert.equal(started, 0);
|
||||
assert.equal((await call('GET', '/file/cip')).status, 404);
|
||||
const catalog = await call('GET', '/catalog');
|
||||
assert.equal(catalog.status, 200);
|
||||
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
|
||||
assert.equal(started, 2);
|
||||
});
|
||||
@@ -26,6 +26,7 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { ghostMapsProxy } from './ghostMaps.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
@@ -64,6 +65,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
ghostMapsProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user