mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 18:34:02 +00:00
438 lines
13 KiB
JavaScript
438 lines
13 KiB
JavaScript
import fsp from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import {
|
|
GHOSTMAPS_MAX_FILE_BYTES,
|
|
GHOSTMAPS_REPO,
|
|
buildGhostMapsCatalog,
|
|
ghostMapsRawUrl,
|
|
parseGitRefAdvertisement,
|
|
} from '../../src/layers/ghostMaps/catalog.js';
|
|
import {
|
|
readResponseBytesCapped,
|
|
readResponseJsonCapped,
|
|
readResponseTextCapped,
|
|
} from './common/http.js';
|
|
|
|
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
|
|
// The repository is not ours, so there is no webhook: the watcher reads the
|
|
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
|
|
// REST API rate limit) every few seconds, and only a new commit costs REST
|
|
// calls (tree + commit). Browsers hear about a new commit over server-sent
|
|
// events and fetch only the products whose blob changed. Files are fetched by
|
|
// commit-pinned raw URLs and kept on disk by blob sha.
|
|
|
|
const DEFAULT_POLL_MS = 15_000;
|
|
const MIN_POLL_MS = 5_000;
|
|
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
|
|
const TIMEOUT_MS = 15_000;
|
|
const FILE_TIMEOUT_MS = 120_000;
|
|
const HEARTBEAT_MS = 25_000;
|
|
const MAX_SUBSCRIBERS = 64;
|
|
const MAX_REF_BYTES = 64 * 1024;
|
|
const MAX_TREE_BYTES = 16 * 1024 * 1024;
|
|
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
|
|
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
|
|
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
|
|
|
|
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
|
|
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
|
|
const seconds = Number(raw);
|
|
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
|
|
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
|
|
}
|
|
|
|
/**
|
|
* The repository watcher and file cache, independent of HTTP routing.
|
|
* @returns {{start: Function, stop: Function, catalog: Function,
|
|
* checkNow: Function, file: Function, subscribe: Function}}
|
|
*/
|
|
export function createGhostMapsWatcher({
|
|
fetchImpl = (...args) => globalThis.fetch(...args),
|
|
now = () => Date.now(),
|
|
pollMs = ghostMapsPollMs(),
|
|
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
|
|
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
|
|
log = (...args) => console.log(...args),
|
|
warn = (...args) => console.warn(...args),
|
|
} = {}) {
|
|
let state = {
|
|
commit: null,
|
|
committedAt: null,
|
|
message: null,
|
|
products: [],
|
|
checkedAt: null,
|
|
changedAt: null,
|
|
error: null,
|
|
};
|
|
let timer = null;
|
|
let running = false;
|
|
let checking = null;
|
|
let failures = 0;
|
|
let diskLoaded = false;
|
|
const listeners = new Set();
|
|
const downloads = new Map();
|
|
|
|
const apiHeaders = () => ({
|
|
Accept: 'application/vnd.github+json',
|
|
'User-Agent': USER_AGENT,
|
|
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
|
|
});
|
|
|
|
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
|
|
const response = await fetchImpl(url, {
|
|
headers,
|
|
redirect: 'follow',
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
});
|
|
if (!response.ok) {
|
|
await response.body?.cancel().catch(() => {});
|
|
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
|
|
status: response.status,
|
|
});
|
|
}
|
|
return response;
|
|
}
|
|
|
|
async function readHead() {
|
|
const response = await get(
|
|
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
|
|
{ 'User-Agent': USER_AGENT },
|
|
);
|
|
const sha = parseGitRefAdvertisement(
|
|
await readResponseTextCapped(response, MAX_REF_BYTES),
|
|
);
|
|
if (!sha) throw new Error('GhostMaps branch head not advertised');
|
|
return sha;
|
|
}
|
|
|
|
async function readCommit(sha) {
|
|
const api = `https://api.github.com/repos/${REPO_PATH}`;
|
|
const tree = await readResponseJsonCapped(
|
|
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
|
|
MAX_TREE_BYTES,
|
|
);
|
|
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
|
|
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
|
|
let committedAt = null;
|
|
let message = null;
|
|
try {
|
|
const commit = await readResponseJsonCapped(
|
|
await get(`${api}/commits/${sha}`, apiHeaders()),
|
|
MAX_COMMIT_BYTES,
|
|
);
|
|
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
|
|
message = String(commit?.commit?.message || '')
|
|
.split('\n')[0]
|
|
.slice(0, 200);
|
|
} catch {
|
|
// The commit date is decoration; the tree is what matters.
|
|
}
|
|
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
|
|
}
|
|
|
|
async function loadDisk() {
|
|
if (diskLoaded) return;
|
|
diskLoaded = true;
|
|
try {
|
|
const saved = JSON.parse(
|
|
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
|
|
);
|
|
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
|
|
state = { ...state, ...saved, error: null };
|
|
} catch {
|
|
/* first run */
|
|
}
|
|
}
|
|
|
|
async function saveDisk() {
|
|
try {
|
|
await fsp.mkdir(cacheDir, { recursive: true });
|
|
const { commit, committedAt, message, products, changedAt } = state;
|
|
await fsp.writeFile(
|
|
path.join(cacheDir, 'catalog.json'),
|
|
JSON.stringify({ commit, committedAt, message, products, changedAt }),
|
|
);
|
|
} catch (error) {
|
|
warn('[ghostmaps] could not save catalog:', error.message);
|
|
}
|
|
}
|
|
|
|
function emit() {
|
|
const event = publicCatalog();
|
|
for (const listener of listeners) {
|
|
try {
|
|
listener(event);
|
|
} catch {
|
|
/* a broken subscriber must not stop the rest */
|
|
}
|
|
}
|
|
}
|
|
|
|
async function check() {
|
|
await loadDisk();
|
|
const head = await readHead();
|
|
state.checkedAt = now();
|
|
if (head === state.commit) {
|
|
if (state.error !== null) {
|
|
state.error = null;
|
|
emit();
|
|
}
|
|
return false;
|
|
}
|
|
const next = await readCommit(head);
|
|
const previous = new Map(state.products.map((p) => [p.id, p.version]));
|
|
const changed = next.products
|
|
.filter((p) => previous.get(p.id) !== p.version)
|
|
.map((p) => p.id);
|
|
state = {
|
|
...state,
|
|
...next,
|
|
commit: head,
|
|
changedAt: now(),
|
|
error: null,
|
|
};
|
|
log(
|
|
`[ghostmaps] commit ${head.slice(0, 8)}${
|
|
next.message ? ` "${next.message}"` : ''
|
|
}; updated: ${changed.join(', ') || 'none'}`,
|
|
);
|
|
await saveDisk();
|
|
emit();
|
|
// Warm the default products so the first browser load is local.
|
|
for (const product of state.products)
|
|
if (product.defaultOn) file(product.id).catch(() => {});
|
|
return true;
|
|
}
|
|
|
|
/** Run one check now, coalescing with one already in flight. */
|
|
function checkNow() {
|
|
if (!checking)
|
|
checking = check()
|
|
.then(
|
|
(changed) => {
|
|
failures = 0;
|
|
return changed;
|
|
},
|
|
(error) => {
|
|
failures += 1;
|
|
state.error = error.message;
|
|
state.checkedAt = now();
|
|
warn('[ghostmaps] check failed:', error.message);
|
|
emit();
|
|
return false;
|
|
},
|
|
)
|
|
.finally(() => {
|
|
checking = null;
|
|
});
|
|
return checking;
|
|
}
|
|
|
|
function schedule() {
|
|
clearTimeout(timer);
|
|
if (!running) return;
|
|
const delay = failures
|
|
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
|
|
: pollMs;
|
|
timer = setTimeout(async () => {
|
|
await checkNow();
|
|
schedule();
|
|
}, delay);
|
|
timer.unref?.();
|
|
}
|
|
|
|
/** Start watching (idempotent); the first check runs immediately. */
|
|
function start() {
|
|
if (running) return checking || Promise.resolve(false);
|
|
running = true;
|
|
const first = checkNow();
|
|
first.then(schedule);
|
|
return first;
|
|
}
|
|
|
|
function stop() {
|
|
running = false;
|
|
clearTimeout(timer);
|
|
timer = null;
|
|
}
|
|
|
|
function publicCatalog() {
|
|
return {
|
|
repo: GHOSTMAPS_REPO.url,
|
|
branch: GHOSTMAPS_REPO.branch,
|
|
commit: state.commit,
|
|
committedAt: state.committedAt,
|
|
message: state.message,
|
|
checkedAt: state.checkedAt,
|
|
changedAt: state.changedAt,
|
|
pollMs,
|
|
error: state.error,
|
|
products: state.products.map(
|
|
({
|
|
id,
|
|
title,
|
|
defaultOn,
|
|
fileName,
|
|
fileTitle,
|
|
version,
|
|
size,
|
|
dateMs,
|
|
}) => ({
|
|
id,
|
|
title,
|
|
defaultOn,
|
|
fileName,
|
|
fileTitle,
|
|
version,
|
|
size,
|
|
dateMs,
|
|
}),
|
|
),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* The bytes of one product's current edition, from disk or a commit-pinned
|
|
* download. Only catalog paths are ever fetched.
|
|
*/
|
|
async function file(id) {
|
|
const product = state.products.find((p) => p.id === id);
|
|
if (!product) return null;
|
|
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
|
|
try {
|
|
return { product, bytes: await fsp.readFile(target) };
|
|
} catch {
|
|
/* not cached yet */
|
|
}
|
|
const commit = state.commit;
|
|
let pending = downloads.get(product.version);
|
|
if (!pending) {
|
|
pending = (async () => {
|
|
const response = await get(
|
|
ghostMapsRawUrl(commit, product.path),
|
|
{ 'User-Agent': USER_AGENT },
|
|
FILE_TIMEOUT_MS,
|
|
);
|
|
const bytes = Buffer.from(
|
|
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
|
|
);
|
|
await fsp.mkdir(path.dirname(target), { recursive: true });
|
|
const temp = `${target}.${process.pid}.tmp`;
|
|
await fsp.writeFile(temp, bytes);
|
|
await fsp.rename(temp, target);
|
|
await pruneFiles();
|
|
return bytes;
|
|
})().finally(() => downloads.delete(product.version));
|
|
downloads.set(product.version, pending);
|
|
}
|
|
return { product, bytes: await pending };
|
|
}
|
|
|
|
/** Keep only files that a current product still points to. */
|
|
async function pruneFiles() {
|
|
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
|
|
const dir = path.join(cacheDir, 'files');
|
|
for (const name of await fsp.readdir(dir).catch(() => [])) {
|
|
if (!keep.has(name) && !name.endsWith('.tmp'))
|
|
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
|
|
}
|
|
}
|
|
|
|
function subscribe(listener) {
|
|
listeners.add(listener);
|
|
return () => listeners.delete(listener);
|
|
}
|
|
|
|
return {
|
|
start,
|
|
stop,
|
|
checkNow,
|
|
catalog: publicCatalog,
|
|
file,
|
|
subscribe,
|
|
subscribers: () => listeners.size,
|
|
};
|
|
}
|
|
|
|
/** Same-origin GhostMaps routes for dev and preview. */
|
|
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
|
|
const json = (res, status, value) => {
|
|
if (res.destroyed) return;
|
|
res.writeHead(status, {
|
|
'Content-Type': 'application/json',
|
|
'Cache-Control': 'no-store',
|
|
...(status === 405 ? { Allow: 'GET' } : {}),
|
|
});
|
|
res.end(JSON.stringify(value));
|
|
};
|
|
|
|
function events(req, res) {
|
|
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
|
|
return json(res, 429, { error: 'too_many_subscribers' });
|
|
res.writeHead(200, {
|
|
'Content-Type': 'text/event-stream',
|
|
'Cache-Control': 'no-store',
|
|
Connection: 'keep-alive',
|
|
'X-Accel-Buffering': 'no',
|
|
});
|
|
const send = (catalog) =>
|
|
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
|
|
res.write('retry: 5000\n\n');
|
|
send(watcher.catalog());
|
|
const unsubscribe = watcher.subscribe(send);
|
|
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
|
|
heartbeat.unref?.();
|
|
const close = () => {
|
|
clearInterval(heartbeat);
|
|
unsubscribe();
|
|
};
|
|
req.on('close', close);
|
|
res.on('error', close);
|
|
}
|
|
|
|
async function handler(req, res) {
|
|
if (req.method !== 'GET')
|
|
return json(res, 405, { error: 'method_not_allowed' });
|
|
const url = new URL(req.url || '/', 'http://local');
|
|
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
|
|
if (!['/catalog', '/events'].includes(url.pathname) && !match)
|
|
return json(res, 404, { error: 'unknown_route' });
|
|
// Watching starts with the first request, so an install that never turns
|
|
// the layer on never contacts GitHub.
|
|
const first = watcher.start();
|
|
if (url.pathname === '/catalog') {
|
|
if (!watcher.catalog().commit) await first;
|
|
return json(res, 200, watcher.catalog());
|
|
}
|
|
if (url.pathname === '/events') return events(req, res);
|
|
try {
|
|
if (!watcher.catalog().commit) await first;
|
|
const found = await watcher.file(match[1]);
|
|
if (!found) return json(res, 404, { error: 'unknown_product' });
|
|
const pinned = url.searchParams.get('v') === found.product.version;
|
|
res.writeHead(200, {
|
|
'Content-Type': 'application/vnd.google-earth.kmz',
|
|
'Content-Length': found.bytes.length,
|
|
'X-GhostMaps-Version': found.product.version,
|
|
// A versioned URL names one immutable blob.
|
|
'Cache-Control': pinned
|
|
? 'private, max-age=31536000, immutable'
|
|
: 'no-store',
|
|
});
|
|
return res.end(found.bytes);
|
|
} catch {
|
|
return json(res, 502, { error: 'ghostmaps_unavailable' });
|
|
}
|
|
}
|
|
|
|
return {
|
|
name: 'ghostmaps',
|
|
configureServer({ middlewares }) {
|
|
middlewares.use('/api/ghostmaps', handler);
|
|
},
|
|
configurePreviewServer({ middlewares }) {
|
|
middlewares.use('/api/ghostmaps', handler);
|
|
},
|
|
};
|
|
}
|