Merge branch 'main' into upstream/perimeters-stale

server/providers/ghostMaps.js

@@ -0,0 +1,437 @@
+import fsp from 'node:fs/promises';
+import path from 'node:path';
+import {
+  GHOSTMAPS_MAX_FILE_BYTES,
+  GHOSTMAPS_REPO,
+  buildGhostMapsCatalog,
+  ghostMapsRawUrl,
+  parseGitRefAdvertisement,
+} from '../../src/layers/ghostMaps/catalog.js';
+import {
+  readResponseBytesCapped,
+  readResponseJsonCapped,
+  readResponseTextCapped,
+} from './common/http.js';
+
+// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
+// The repository is not ours, so there is no webhook: the watcher reads the
+// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
+// REST API rate limit) every few seconds, and only a new commit costs REST
+// calls (tree + commit). Browsers hear about a new commit over server-sent
+// events and fetch only the products whose blob changed. Files are fetched by
+// commit-pinned raw URLs and kept on disk by blob sha.
+
+const DEFAULT_POLL_MS = 15_000;
+const MIN_POLL_MS = 5_000;
+const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
+const TIMEOUT_MS = 15_000;
+const FILE_TIMEOUT_MS = 120_000;
+const HEARTBEAT_MS = 25_000;
+const MAX_SUBSCRIBERS = 64;
+const MAX_REF_BYTES = 64 * 1024;
+const MAX_TREE_BYTES = 16 * 1024 * 1024;
+const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
+const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
+const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
+
+/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
+export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
+  const seconds = Number(raw);
+  if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
+  return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
+}
+
+/**
+ * The repository watcher and file cache, independent of HTTP routing.
+ * @returns {{start: Function, stop: Function, catalog: Function,
+ *   checkNow: Function, file: Function, subscribe: Function}}
+ */
+export function createGhostMapsWatcher({
+  fetchImpl = (...args) => globalThis.fetch(...args),
+  now = () => Date.now(),
+  pollMs = ghostMapsPollMs(),
+  cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
+  token = () => String(process.env.GITHUB_TOKEN || '').trim(),
+  log = (...args) => console.log(...args),
+  warn = (...args) => console.warn(...args),
+} = {}) {
+  let state = {
+    commit: null,
+    committedAt: null,
+    message: null,
+    products: [],
+    checkedAt: null,
+    changedAt: null,
+    error: null,
+  };
+  let timer = null;
+  let running = false;
+  let checking = null;
+  let failures = 0;
+  let diskLoaded = false;
+  const listeners = new Set();
+  const downloads = new Map();
+
+  const apiHeaders = () => ({
+    Accept: 'application/vnd.github+json',
+    'User-Agent': USER_AGENT,
+    ...(token() ? { Authorization: `Bearer ${token()}` } : {}),
+  });
+
+  async function get(url, headers, timeoutMs = TIMEOUT_MS) {
+    const response = await fetchImpl(url, {
+      headers,
+      redirect: 'follow',
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => {});
+      throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
+        status: response.status,
+      });
+    }
+    return response;
+  }
+
+  async function readHead() {
+    const response = await get(
+      `${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
+      { 'User-Agent': USER_AGENT },
+    );
+    const sha = parseGitRefAdvertisement(
+      await readResponseTextCapped(response, MAX_REF_BYTES),
+    );
+    if (!sha) throw new Error('GhostMaps branch head not advertised');
+    return sha;
+  }
+
+  async function readCommit(sha) {
+    const api = `https://api.github.com/repos/${REPO_PATH}`;
+    const tree = await readResponseJsonCapped(
+      await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
+      MAX_TREE_BYTES,
+    );
+    if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
+    if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
+    let committedAt = null;
+    let message = null;
+    try {
+      const commit = await readResponseJsonCapped(
+        await get(`${api}/commits/${sha}`, apiHeaders()),
+        MAX_COMMIT_BYTES,
+      );
+      committedAt = Date.parse(commit?.commit?.committer?.date) || null;
+      message = String(commit?.commit?.message || '')
+        .split('\n')[0]
+        .slice(0, 200);
+    } catch {
+      // The commit date is decoration; the tree is what matters.
+    }
+    return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
+  }
+
+  async function loadDisk() {
+    if (diskLoaded) return;
+    diskLoaded = true;
+    try {
+      const saved = JSON.parse(
+        await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
+      );
+      if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
+        state = { ...state, ...saved, error: null };
+    } catch {
+      /* first run */
+    }
+  }
+
+  async function saveDisk() {
+    try {
+      await fsp.mkdir(cacheDir, { recursive: true });
+      const { commit, committedAt, message, products, changedAt } = state;
+      await fsp.writeFile(
+        path.join(cacheDir, 'catalog.json'),
+        JSON.stringify({ commit, committedAt, message, products, changedAt }),
+      );
+    } catch (error) {
+      warn('[ghostmaps] could not save catalog:', error.message);
+    }
+  }
+
+  function emit() {
+    const event = publicCatalog();
+    for (const listener of listeners) {
+      try {
+        listener(event);
+      } catch {
+        /* a broken subscriber must not stop the rest */
+      }
+    }
+  }
+
+  async function check() {
+    await loadDisk();
+    const head = await readHead();
+    state.checkedAt = now();
+    if (head === state.commit) {
+      if (state.error !== null) {
+        state.error = null;
+        emit();
+      }
+      return false;
+    }
+    const next = await readCommit(head);
+    const previous = new Map(state.products.map((p) => [p.id, p.version]));
+    const changed = next.products
+      .filter((p) => previous.get(p.id) !== p.version)
+      .map((p) => p.id);
+    state = {
+      ...state,
+      ...next,
+      commit: head,
+      changedAt: now(),
+      error: null,
+    };
+    log(
+      `[ghostmaps] commit ${head.slice(0, 8)}${
+        next.message ? ` "${next.message}"` : ''
+      }; updated: ${changed.join(', ') || 'none'}`,
+    );
+    await saveDisk();
+    emit();
+    // Warm the default products so the first browser load is local.
+    for (const product of state.products)
+      if (product.defaultOn) file(product.id).catch(() => {});
+    return true;
+  }
+
+  /** Run one check now, coalescing with one already in flight. */
+  function checkNow() {
+    if (!checking)
+      checking = check()
+        .then(
+          (changed) => {
+            failures = 0;
+            return changed;
+          },
+          (error) => {
+            failures += 1;
+            state.error = error.message;
+            state.checkedAt = now();
+            warn('[ghostmaps] check failed:', error.message);
+            emit();
+            return false;
+          },
+        )
+        .finally(() => {
+          checking = null;
+        });
+    return checking;
+  }
+
+  function schedule() {
+    clearTimeout(timer);
+    if (!running) return;
+    const delay = failures
+      ? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
+      : pollMs;
+    timer = setTimeout(async () => {
+      await checkNow();
+      schedule();
+    }, delay);
+    timer.unref?.();
+  }
+
+  /** Start watching (idempotent); the first check runs immediately. */
+  function start() {
+    if (running) return checking || Promise.resolve(false);
+    running = true;
+    const first = checkNow();
+    first.then(schedule);
+    return first;
+  }
+
+  function stop() {
+    running = false;
+    clearTimeout(timer);
+    timer = null;
+  }
+
+  function publicCatalog() {
+    return {
+      repo: GHOSTMAPS_REPO.url,
+      branch: GHOSTMAPS_REPO.branch,
+      commit: state.commit,
+      committedAt: state.committedAt,
+      message: state.message,
+      checkedAt: state.checkedAt,
+      changedAt: state.changedAt,
+      pollMs,
+      error: state.error,
+      products: state.products.map(
+        ({
+          id,
+          title,
+          defaultOn,
+          fileName,
+          fileTitle,
+          version,
+          size,
+          dateMs,
+        }) => ({
+          id,
+          title,
+          defaultOn,
+          fileName,
+          fileTitle,
+          version,
+          size,
+          dateMs,
+        }),
+      ),
+    };
+  }
+
+  /**
+   * The bytes of one product's current edition, from disk or a commit-pinned
+   * download. Only catalog paths are ever fetched.
+   */
+  async function file(id) {
+    const product = state.products.find((p) => p.id === id);
+    if (!product) return null;
+    const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
+    try {
+      return { product, bytes: await fsp.readFile(target) };
+    } catch {
+      /* not cached yet */
+    }
+    const commit = state.commit;
+    let pending = downloads.get(product.version);
+    if (!pending) {
+      pending = (async () => {
+        const response = await get(
+          ghostMapsRawUrl(commit, product.path),
+          { 'User-Agent': USER_AGENT },
+          FILE_TIMEOUT_MS,
+        );
+        const bytes = Buffer.from(
+          await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
+        );
+        await fsp.mkdir(path.dirname(target), { recursive: true });
+        const temp = `${target}.${process.pid}.tmp`;
+        await fsp.writeFile(temp, bytes);
+        await fsp.rename(temp, target);
+        await pruneFiles();
+        return bytes;
+      })().finally(() => downloads.delete(product.version));
+      downloads.set(product.version, pending);
+    }
+    return { product, bytes: await pending };
+  }
+
+  /** Keep only files that a current product still points to. */
+  async function pruneFiles() {
+    const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
+    const dir = path.join(cacheDir, 'files');
+    for (const name of await fsp.readdir(dir).catch(() => [])) {
+      if (!keep.has(name) && !name.endsWith('.tmp'))
+        await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
+    }
+  }
+
+  function subscribe(listener) {
+    listeners.add(listener);
+    return () => listeners.delete(listener);
+  }
+
+  return {
+    start,
+    stop,
+    checkNow,
+    catalog: publicCatalog,
+    file,
+    subscribe,
+    subscribers: () => listeners.size,
+  };
+}
+
+/** Same-origin GhostMaps routes for dev and preview. */
+export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
+  const json = (res, status, value) => {
+    if (res.destroyed) return;
+    res.writeHead(status, {
+      'Content-Type': 'application/json',
+      'Cache-Control': 'no-store',
+      ...(status === 405 ? { Allow: 'GET' } : {}),
+    });
+    res.end(JSON.stringify(value));
+  };
+
+  function events(req, res) {
+    if (watcher.subscribers() >= MAX_SUBSCRIBERS)
+      return json(res, 429, { error: 'too_many_subscribers' });
+    res.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-store',
+      Connection: 'keep-alive',
+      'X-Accel-Buffering': 'no',
+    });
+    const send = (catalog) =>
+      res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
+    res.write('retry: 5000\n\n');
+    send(watcher.catalog());
+    const unsubscribe = watcher.subscribe(send);
+    const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
+    heartbeat.unref?.();
+    const close = () => {
+      clearInterval(heartbeat);
+      unsubscribe();
+    };
+    req.on('close', close);
+    res.on('error', close);
+  }
+
+  async function handler(req, res) {
+    if (req.method !== 'GET')
+      return json(res, 405, { error: 'method_not_allowed' });
+    const url = new URL(req.url || '/', 'http://local');
+    const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
+    if (!['/catalog', '/events'].includes(url.pathname) && !match)
+      return json(res, 404, { error: 'unknown_route' });
+    // Watching starts with the first request, so an install that never turns
+    // the layer on never contacts GitHub.
+    const first = watcher.start();
+    if (url.pathname === '/catalog') {
+      if (!watcher.catalog().commit) await first;
+      return json(res, 200, watcher.catalog());
+    }
+    if (url.pathname === '/events') return events(req, res);
+    try {
+      if (!watcher.catalog().commit) await first;
+      const found = await watcher.file(match[1]);
+      if (!found) return json(res, 404, { error: 'unknown_product' });
+      const pinned = url.searchParams.get('v') === found.product.version;
+      res.writeHead(200, {
+        'Content-Type': 'application/vnd.google-earth.kmz',
+        'Content-Length': found.bytes.length,
+        'X-GhostMaps-Version': found.product.version,
+        // A versioned URL names one immutable blob.
+        'Cache-Control': pinned
+          ? 'private, max-age=31536000, immutable'
+          : 'no-store',
+      });
+      return res.end(found.bytes);
+    } catch {
+      return json(res, 502, { error: 'ghostmaps_unavailable' });
+    }
+  }
+
+  return {
+    name: 'ghostmaps',
+    configureServer({ middlewares }) {
+      middlewares.use('/api/ghostmaps', handler);
+    },
+    configurePreviewServer({ middlewares }) {
+      middlewares.use('/api/ghostmaps', handler);
+    },
+  };
+}

server/providers/ghostMaps.test.mjs

@@ -0,0 +1,294 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, readdir, rm } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import {
+  createGhostMapsWatcher,
+  ghostMapsPollMs,
+  ghostMapsProxy,
+} from './ghostMaps.js';
+
+const MASTER =
+  'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
+const KINETIC =
+  'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
+const sha = (c) => c.repeat(40);
+
+/** A fake GitHub: ref advertisement, tree, commit and raw files. */
+function fakeGitHub() {
+  const repo = { head: sha('1'), trees: new Map(), calls: [] };
+  repo.trees.set(sha('1'), [
+    {
+      path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
+      type: 'blob',
+      size: 4,
+      sha: sha('a'),
+    },
+    { path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
+  ]);
+  const body = (text, status = 200) =>
+    new Response(text, {
+      status,
+      headers: { 'content-length': String(Buffer.byteLength(text)) },
+    });
+  const fetchImpl = async (url) => {
+    repo.calls.push(url);
+    if (url.endsWith('/info/refs?service=git-upload-pack'))
+      return body(
+        `001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
+      );
+    const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
+    if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
+    if (/\/commits\//.test(url))
+      return body(
+        JSON.stringify({
+          commit: {
+            message: 'Oct 18 Update\nmore',
+            committer: { date: '2026-10-18T00:00:00Z' },
+          },
+        }),
+      );
+    if (url.startsWith('https://raw.githubusercontent.com/'))
+      return body('KMZ!');
+    return body('nope', 404);
+  };
+  return { repo, fetchImpl };
+}
+
+async function withCache(fn) {
+  const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
+  try {
+    await fn(dir);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+// checkNow announces the catalog before background warm-up finishes. Join the
+// same file reads before changing editions or removing the temporary cache.
+async function warmDefaults(watcher) {
+  await Promise.all(
+    watcher
+      .catalog()
+      .products.filter((product) => product.defaultOn)
+      .map((product) => watcher.file(product.id)),
+  );
+}
+
+test('a new commit is announced once, naming only the products that changed', () =>
+  withCache(async (cacheDir) => {
+    const { repo, fetchImpl } = fakeGitHub();
+    const watcher = createGhostMapsWatcher({
+      fetchImpl,
+      cacheDir,
+      log: () => {},
+      warn: () => {},
+    });
+    const events = [];
+    watcher.subscribe((catalog) => events.push(catalog));
+
+    assert.equal(await watcher.checkNow(), true);
+    await warmDefaults(watcher);
+    const first = watcher.catalog();
+    assert.equal(first.commit, sha('1'));
+    assert.equal(first.message, 'Oct 18 Update');
+    assert.deepEqual(
+      first.products.map((p) => p.id),
+      ['cip', 'kinetic-30d'],
+    );
+    assert.equal(events.length, 1);
+
+    // Same head: one ref read, no REST calls, no event.
+    repo.calls.length = 0;
+    assert.equal(await watcher.checkNow(), false);
+    assert.equal(repo.calls.length, 1);
+    assert.equal(events.length, 1);
+
+    // A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
+    repo.head = sha('2');
+    repo.trees.set(sha('2'), [
+      ...repo.trees.get(sha('1')),
+      {
+        path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
+        type: 'blob',
+        size: 4,
+        sha: sha('c'),
+      },
+    ]);
+    assert.equal(await watcher.checkNow(), true);
+    await warmDefaults(watcher);
+    assert.equal(events.length, 2);
+    const cip = events[1].products.find((p) => p.id === 'cip');
+    assert.equal(cip.version, sha('c'));
+    assert.match(cip.fileName, /October_18/);
+    assert.equal(
+      events[1].products.find((p) => p.id === 'kinetic-30d').version,
+      sha('b'),
+    );
+    // Repository paths stay on the server.
+    assert.equal('path' in cip, false);
+    watcher.stop();
+  }));
+
+test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
+  withCache(async (cacheDir) => {
+    const { repo, fetchImpl } = fakeGitHub();
+    const watcher = createGhostMapsWatcher({
+      fetchImpl,
+      cacheDir,
+      log: () => {},
+      warn: () => {},
+    });
+    await watcher.checkNow();
+    const found = await watcher.file('kinetic-30d');
+    assert.equal(found.bytes.toString(), 'KMZ!');
+    const raw = repo.calls.filter((url) =>
+      url.includes('raw.githubusercontent'),
+    );
+    assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
+    assert.ok(
+      raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
+    );
+    const before = repo.calls.length;
+    await watcher.file('kinetic-30d');
+    assert.equal(repo.calls.length, before, 'second read comes from disk');
+    assert.equal(await watcher.file('not-a-product'), null);
+
+    // When the edition changes, the old blob leaves the disk cache.
+    await watcher.file('cip');
+    repo.head = sha('2');
+    repo.trees.set(sha('2'), [
+      {
+        path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
+        type: 'blob',
+        size: 4,
+        sha: sha('c'),
+      },
+      { path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
+    ]);
+    await watcher.checkNow();
+    await watcher.file('cip');
+    const files = (await readdir(path.join(cacheDir, 'files'))).sort();
+    assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
+    watcher.stop();
+  }));
+
+test('the catalog survives a restart from disk, and upstream failures are reported', () =>
+  withCache(async (cacheDir) => {
+    const { fetchImpl } = fakeGitHub();
+    const quiet = { log: () => {}, warn: () => {} };
+    const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
+    await online.checkNow();
+    await warmDefaults(online);
+    const offline = createGhostMapsWatcher({
+      fetchImpl: async () => new Response('down', { status: 503 }),
+      cacheDir,
+      ...quiet,
+    });
+    assert.equal(await offline.checkNow(), false);
+    const catalog = offline.catalog();
+    assert.equal(catalog.commit, sha('1'));
+    assert.equal(catalog.products.length, 2);
+    assert.match(catalog.error, /HTTP 503/);
+  }));
+
+test('an unchanged head clears an upstream outage and announces recovery once', () =>
+  withCache(async (cacheDir) => {
+    const { fetchImpl } = fakeGitHub();
+    let unavailable = false;
+    let checkedAt = 1000;
+    const requests = [];
+    const watcher = createGhostMapsWatcher({
+      cacheDir,
+      now: () => checkedAt,
+      fetchImpl: async (url, options) => {
+        requests.push(url);
+        if (unavailable) return new Response('down', { status: 503 });
+        return fetchImpl(url, options);
+      },
+      log: () => {},
+      warn: () => {},
+    });
+    const events = [];
+    watcher.subscribe((catalog) => events.push(catalog));
+    await watcher.checkNow();
+    await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
+    const healthy = watcher.catalog();
+    assert.equal(events.length, 1);
+
+    unavailable = true;
+    checkedAt = 2000;
+    assert.equal(await watcher.checkNow(), false);
+    assert.match(watcher.catalog().error, /HTTP 503/);
+    assert.equal(events.length, 2);
+
+    unavailable = false;
+    checkedAt = 3000;
+    requests.length = 0;
+    assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
+    const recovered = watcher.catalog();
+    assert.equal(recovered.error, null);
+    assert.equal(recovered.checkedAt, checkedAt);
+    assert.equal(recovered.commit, healthy.commit);
+    assert.equal(recovered.changedAt, healthy.changedAt);
+    assert.deepEqual(recovered.products, healthy.products);
+    assert.equal(requests.length, 1, 'recovery needs only the ref read');
+    assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
+    assert.equal(events.length, 3, 'subscribers see the recovered catalog');
+    assert.equal(events[2].error, null);
+    assert.equal(events[2].checkedAt, checkedAt);
+
+    checkedAt = 4000;
+    assert.equal(await watcher.checkNow(), false);
+    assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
+    watcher.stop();
+  }));
+
+test('poll interval is configurable with a 5 s floor', () => {
+  assert.equal(ghostMapsPollMs(undefined), 15000);
+  assert.equal(ghostMapsPollMs('30'), 30000);
+  assert.equal(ghostMapsPollMs('1'), 5000);
+  assert.equal(ghostMapsPollMs('soon'), 15000);
+});
+
+test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
+  let started = 0;
+  const watcher = {
+    start: () => {
+      started += 1;
+      return Promise.resolve(false);
+    },
+    catalog: () => ({ commit: sha('1'), products: [] }),
+    file: async () => null,
+    subscribe: () => () => {},
+    subscribers: () => 0,
+  };
+  let handler;
+  ghostMapsProxy({ watcher }).configureServer({
+    middlewares: { use: (_route, fn) => (handler = fn) },
+  });
+  const call = async (method, url) => {
+    const res = {
+      destroyed: false,
+      writeHead(status, headers) {
+        this.status = status;
+        this.headers = headers;
+      },
+      end(body) {
+        this.body = String(body ?? '');
+      },
+    };
+    await handler({ method, url }, res);
+    return res;
+  };
+  assert.equal((await call('GET', '/nope')).status, 404);
+  assert.equal((await call('POST', '/catalog')).status, 405);
+  assert.equal((await call('GET', '/file/../../etc')).status, 404);
+  assert.equal(started, 0);
+  assert.equal((await call('GET', '/file/cip')).status, 404);
+  const catalog = await call('GET', '/catalog');
+  assert.equal(catalog.status, 200);
+  assert.equal(JSON.parse(catalog.body).commit, sha('1'));
+  assert.equal(started, 2);
+});

server/providers/local.js

@@ -26,6 +26,7 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
 import { weatherProxy } from './weather.js';
 import { firePerimetersProxy } from './firePerimeters.js';
 import { gnssIntegrityProxy } from './gnssIntegrity.js';
+import { ghostMapsProxy } from './ghostMaps.js';
 import { cycloneProxy } from './cyclones.js';
 import { windProxy } from './wind.js';
 
@@ -64,6 +65,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
     mapillaryProxy(),
     firePerimetersProxy(),
     gnssIntegrityProxy(),
+    ghostMapsProxy(),
     keySetupEndpoint(),
   ];
 }

server/providers/openai/instructions.js

@@ -10,7 +10,7 @@ function realtimeInstructions(
     'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
     'Only control the app by calling the provided tools. Never invent tool names or arguments.',
     'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
-    'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
+    'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
     'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
     'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
     'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
@@ -18,13 +18,12 @@ function realtimeInstructions(
     'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
     'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
     'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
-    'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
+    'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
     'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
     'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
     'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
     'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
     'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
-    'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
     'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
     'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
     'Do not mention disabled layers or stale selections.',

server/providers/openai/toolDescriptions.js

@@ -44,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
   },
   select_nearest_aircraft: {
     description:
-      'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
+      'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
     $position: 1,
     parameters: {
       properties: {
@@ -111,12 +111,18 @@ export const ACTION_DESCRIPTIONS = {
     },
   },
   set_panel_open: {
-    description: 'Open or close a GEV UI panel/dropdown.',
+    description:
+      'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
+    $position: 1,
+  },
+  set_voice_inactivity_timeout: {
+    description:
+      'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
     $position: 1,
   },
   set_context_mode: {
     description:
-      'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
+      'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
     $position: 1,
     parameters: {
       properties: {
@@ -129,23 +135,23 @@ export const ACTION_DESCRIPTIONS = {
   },
   control_cockpit: {
     description:
-      'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
+      'Explicit Cockpit requests only: enter from a selected/tracked aircraft (establishes Contacts itself; no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
     $position: 1,
     parameters: {
       properties: {
         action: {
           description:
-            'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
+            'previous/next (or prev) cycles nearby Cockpit contacts.',
           $position: 2,
         },
         targetLayer: {
           description:
-            'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
+            'Optional next/previous layer filter (e.g. military-only).',
           $position: 2,
         },
         aircraftClass: {
           description:
-            'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
+            'Optional next/previous aircraft class filter (e.g. helicopter).',
           $position: 1,
         },
       },