gods-eye-view/scripts/precompute-cctv-heights.mjs

import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { chromeExecutablePath } from './qa-chrome.mjs';
import {
  planeSupportPoints,
  poseHash,
  SUPPORT_KEYS,
} from '../src/data/cctvFootprint.js';

const ROOT = fileURLToPath(new URL('../', import.meta.url));
const SAMPLE_KEYS = ['mount', ...SUPPORT_KEYS];
const BATCH_SIZES = [16, 4, 1];
const DEADLINE_MS = 90_000;

/** Extract the served nominal pose (the values the catalog ships). */
export function nominalPose(source) {
  const pose = Object.fromEntries(
    [
      'lat',
      'lon',
      'headingDeg',
      'pitchDeg',
      'fovDeg',
      'rangeM',
      'mountHeightM',
    ].map((key) => {
      if (typeof source[key] !== 'number' || !Number.isFinite(source[key])) {
        throw new Error(`Camera ${source.id}: invalid ${key}`);
      }
      return [key, source[key]];
    }),
  );
  if (Math.abs(pose.lat) > 90 || Math.abs(pose.lon) > 180) {
    throw new Error(`Camera ${source.id}: invalid coordinates`);
  }
  // Hashed exactly as served: the server join (groundHeights.js) and the
  // client (camera.servedPose) hash the same raw values, so no clamp here.
  return pose;
}

/** Return a roughly 2 km grid key, with longitude width fixed per row. */
export function cellKey({ lat, lon }) {
  if (
    !Number.isFinite(lat) ||
    !Number.isFinite(lon) ||
    Math.abs(lat) > 90 ||
    Math.abs(lon) > 180
  ) {
    throw new Error('Invalid cell coordinates');
  }
  const row = Math.min(8999, Math.floor((lat + 90) / 0.02));
  const rowLat = -90 + (row + 0.5) * 0.02;
  const lonWidth = 0.02 / Math.cos((rowLat * Math.PI) / 180);
  const col = Math.floor(((lon === 180 ? -180 : lon) + 180) / lonWidth);
  return `${row},${col}`;
}

/** Split an ordered array into bounded batches without mutating it. */
export function batches(items, size = 16) {
  if (!Number.isSafeInteger(size) || size < 1)
    throw new Error('Invalid batch size');
  const result = [];
  for (let i = 0; i < items.length; i += size)
    result.push(items.slice(i, i + size));
  return result;
}

/** Group cameras by cell and order cells numerically by row, then column. */
export function planCells(cameras) {
  const cells = new Map();
  for (const camera of cameras) {
    const key = cellKey(camera.pose);
    if (!cells.has(key)) cells.set(key, { key, cameras: [] });
    cells.get(key).cameras.push(camera);
  }
  return [...cells.values()].sort((a, b) => {
    const [ar, ac] = a.key.split(',').map(Number);
    const [br, bc] = b.key.split(',').map(Number);
    return ar - br || ac - bc;
  });
}

/** Preserve ellipsoidal metres; missing, non-finite and implausible values are null. */
export function plausibleHeight(height) {
  return Number.isFinite(height) && height >= -150 ? height : null;
}

/**
 * Only a complete, successful entry for exactly the current nominal pose is
 * reusable; an entry with missing supports is resampled (valid heights are
 * kept by mergeCameraSamples).
 */
export function needsSampling(entry, pose) {
  return (
    entry?.status !== 'ok' ||
    entry.poseHash !== poseHash(pose) ||
    (Array.isArray(entry.misses) && entry.misses.length > 0)
  );
}

/** Merge retry samples without losing valid heights; a missing mount stores no heights. */
export function mergeCameraSamples(
  pose,
  heights,
  previous,
  attempts,
  sampledAt,
) {
  const hash = poseHash(pose);
  const prior = previous?.poseHash === hash ? previous : null;
  const values = SAMPLE_KEYS.map(
    (key, i) =>
      plausibleHeight(heights[i]) ??
      plausibleHeight(
        key === 'mount' ? prior?.mountGroundM : prior?.supports?.[key],
      ),
  );
  const entry = {
    poseHash: hash,
    status: values[0] === null ? 'miss' : 'ok',
    misses: SAMPLE_KEYS.filter((_, i) => values[i] === null),
    sampledAt,
    attempts,
  };
  if (entry.status === 'ok') {
    entry.mountGroundM = values[0];
    entry.supports = Object.fromEntries(
      SUPPORT_KEYS.map((key, i) => [key, values[i + 1]]),
    );
  }
  return entry;
}

/** Build a sidecar while retaining entries outside this run's selection. */
export function mergeSidecar(existing, updates, generatedAt) {
  const cameras = { ...existing?.cameras, ...updates };
  return {
    schemaVersion: 1,
    provider: 'google-3d-tiles',
    heightReference: 'WGS84-ellipsoid',
    generatedAt,
    cameras: Object.fromEntries(
      Object.keys(cameras)
        .sort()
        .map((id) => [id, cameras[id]]),
    ),
  };
}

/** Parse pilot filters; the limit applies after kinds, before resume filtering. */
export function parseArgs(argv) {
  const options = { limit: Infinity, kinds: null, dryRun: false };
  for (let i = 0; i < argv.length; i += 1) {
    const arg = argv[i];
    if (arg === '--dry-run') options.dryRun = true;
    else if (arg === '--limit') {
      const value = argv[++i];
      if (!/^\d+$/.test(value ?? '') || !Number.isSafeInteger(Number(value))) {
        throw new Error('--limit requires a non-negative integer');
      }
      options.limit = Number(value);
    } else if (arg === '--kinds') {
      const value = argv[++i];
      if (
        !value ||
        value.startsWith('--') ||
        value.split(',').some((kind) => !kind.trim())
      ) {
        throw new Error('--kinds requires comma-separated sourceKinds');
      }
      options.kinds = value.split(',').map((kind) => kind.trim());
    } else throw new Error(`Unknown argument: ${arg}`);
  }
  return options;
}

/** Use the shared footprint's mount and nine supports in the sampling wire order. */
export function samplingPoints(pose) {
  const footprint = planeSupportPoints(pose);
  return SAMPLE_KEYS.map((key) => {
    const point = key === 'mount' ? footprint.mount : footprint.supports[key];
    if (!Number.isFinite(point?.lat) || !Number.isFinite(point?.lon)) {
      throw new Error(
        `Shared planeSupportPoints returned invalid coordinates for ${key}`,
      );
    }
    return { lat: point.lat, lon: point.lon };
  });
}

/** Sample one cell serially, retrying failed cameras at most three times (16/4/1). */
export async function sampleCell(
  cameras,
  sampler,
  now = () => new Date().toISOString(),
  existing = null,
) {
  // Seed with the prior entries so a resumed camera keeps every valid height
  // it already has (mergeCameraSamples only replaces misses).
  const entries = Object.create(null);
  for (const camera of cameras) {
    if (existing?.[camera.id]) entries[camera.id] = existing[camera.id];
  }
  let pending = cameras;
  for (const [index, size] of BATCH_SIZES.entries()) {
    const retry = [];
    for (const batch of batches(pending, size)) {
      let samples;
      try {
        samples = await sampler.sample(batch);
      } catch (error) {
        sampler.warn?.(
          `Attempt ${index + 1}, ${batch.length} cameras: ${error.message}`,
        );
        // A timed-out evaluate can still be sampling: destroy its page before retrying.
        await sampler.recycle();
        samples = batch.map(() => []);
      }
      const sampledAt = now();
      for (let i = 0; i < batch.length; i += 1) {
        const camera = batch[i];
        const entry = mergeCameraSamples(
          camera.pose,
          samples[i] ?? [],
          entries[camera.id],
          index + 1,
          sampledAt,
        );
        entries[camera.id] = entry;
        if (entry.misses.length) retry.push(camera);
      }
    }
    pending = retry;
    if (!pending.length) break;
  }
  return entries;
}

/** Bound a sampling operation; callers must recycle its page after rejection. */
export async function withDeadline(operation, timeoutMs = DEADLINE_MS) {
  let timer;
  try {
    return await Promise.race([
      operation,
      new Promise((_, reject) => {
        timer = setTimeout(
          () =>
            reject(
              new Error(
                `Height sampling timed out after ${timeoutMs / 1000} s`,
              ),
            ),
          timeoutMs,
        );
      }),
    ]);
  } finally {
    clearTimeout(timer);
  }
}

async function openPage(browser, base) {
  const page = await browser.newPage();
  try {
    await page.goto(
      `${base}/#v=2&lat=30&lon=-95&alt=17000000&heading=0&pitch=-90&roll=0`,
    );
    await page.waitForFunction(() => !!window.__godsEyeView?.viewer);
    return page;
  } catch (error) {
    await page.close();
    throw error;
  }
}

async function loadSidecar(output) {
  let existing;
  try {
    existing = JSON.parse(await readFile(output, 'utf8'));
  } catch (error) {
    if (error.code === 'ENOENT') return null;
    throw error;
  }
  if (
    existing?.schemaVersion !== 1 ||
    existing.provider !== 'google-3d-tiles' ||
    existing.heightReference !== 'WGS84-ellipsoid' ||
    !existing.cameras ||
    typeof existing.cameras !== 'object' ||
    Array.isArray(existing.cameras)
  ) {
    throw new Error(`Incompatible sidecar: ${output}`);
  }
  return existing;
}

async function writeSidecar(output, sidecar) {
  await mkdir(path.dirname(output), { recursive: true });
  const temporary = `${output}.${process.pid}.tmp`;
  try {
    await writeFile(temporary, `${JSON.stringify(sidecar, null, 2)}\n`);
    await rename(temporary, output);
  } finally {
    await rm(temporary, { force: true });
  }
}

function counts(cameras, entries) {
  const ok = cameras.filter(
    (camera) => entries[camera.id]?.status === 'ok',
  ).length;
  return { ok, miss: cameras.length - ok };
}

async function main() {
  const started = Date.now();
  const options = parseArgs(process.argv.slice(2));
  const base = (process.env.GEV_BASE || 'http://localhost:4173').replace(
    /\/+$/,
    '',
  );
  const output = path.resolve(
    ROOT,
    process.env.OUT ||
      'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json',
  );
  let sidecar = await loadSidecar(output);
  // A cold catalog waits for its slowest pack (Tarktee gets 30 s), so allow
  // more than that.
  const response = await fetch(`${base}/api/cctv/sources`, {
    signal: AbortSignal.timeout(60_000),
  });
  if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`);
  const catalog = await response.json();
  if (!Array.isArray(catalog.sources))
    throw new Error('Catalog must contain sources[]');
  const ids = new Set();
  const selected = catalog.sources
    .filter(
      (source) => !options.kinds || options.kinds.includes(source.sourceKind),
    )
    .slice(0, options.limit)
    .flatMap((source) => {
      if (typeof source.id !== 'string' || !source.id || ids.has(source.id)) {
        throw new Error('Catalog has an invalid or duplicate camera id');
      }
      ids.add(source.id);
      let pose;
      try {
        pose = nominalPose(source);
      } catch (error) {
        // One camera with a malformed pose must not abort the whole run.
        console.warn(`skip ${source.id}: ${error.message}`);
        return [];
      }
      return [
        {
          id: source.id,
          sourceKind: source.sourceKind || 'unknown',
          pose,
        },
      ];
    });
  const pending = selected.filter((camera) =>
    needsSampling(sidecar?.cameras?.[camera.id], camera.pose),
  );
  const cells = planCells(pending);
  console.log(
    `${selected.length} cameras selected, ${selected.length - pending.length} resumed, ${pending.length} to sample in ${cells.length} cells`,
  );
  if (options.dryRun) {
    for (const cell of cells) {
      const sizes = batches(cell.cameras).map(
        (batch) => `${batch.length} cameras/${batch.length * 10} points`,
      );
      console.log(
        `cell ${cell.key}: ${cell.cameras.length} cameras; batches: ${sizes.join(', ')}`,
      );
    }
    console.log(
      'Retries: failed cameras in batches of 4, then singly; 3 attempts, 90 s per call.',
    );
    return;
  }

  // Validate shared geometry before starting an expensive browser session.
  for (const camera of pending) camera.points = samplingPoints(camera.pose);
  let browser;
  try {
    if (pending.length) {
      const { default: puppeteer } = await import('puppeteer');
      browser = await puppeteer.launch({
        headless: false,
        executablePath: chromeExecutablePath(),
        protocolTimeout: 600000,
        args: [
          '--no-sandbox',
          '--disable-background-timer-throttling',
          '--disable-renderer-backgrounding',
          '--window-size=1200,800',
        ],
      });
      let page = await openPage(browser, base);
      const sampler = {
        warn: (message) => console.error(message),
        recycle: async () => {
          await page.close();
          page = await openPage(browser, base);
        },
        sample: (batch) =>
          withDeadline(
            page.evaluate(async (cameras) => {
              const gev = window.__godsEyeView;
              const scene = gev.viewer.scene;
              const Cartesian3 = gev.viewer.camera.position.constructor;
              const Cartographic =
                gev.viewer.camera.positionCartographic.constructor;
              const lat =
                cameras.reduce((sum, camera) => sum + camera.pose.lat, 0) /
                cameras.length;
              const lon =
                cameras.reduce((sum, camera) => sum + camera.pose.lon, 0) /
                cameras.length;
              gev.viewer.camera.setView({
                destination: Cartesian3.fromDegrees(lon, lat, 3000),
                orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 },
              });
              const cartographics = cameras.flatMap((camera) =>
                camera.points.map((point) =>
                  Cartographic.fromDegrees(point.lon, point.lat),
                ),
              );
              const sampled =
                await scene.sampleHeightMostDetailed(cartographics);
              return cameras.map((_, i) =>
                sampled
                  .slice(i * 10, (i + 1) * 10)
                  .map((point) =>
                    Number.isFinite(point.height) ? point.height : null,
                  ),
              );
            }, batch),
          ),
      };
      for (const cell of cells) {
        const cellStarted = Date.now();
        const entries = await sampleCell(
          cell.cameras,
          sampler,
          undefined,
          sidecar?.cameras,
        );
        sidecar = mergeSidecar(sidecar, entries, new Date().toISOString());
        await writeSidecar(output, sidecar);
        const tally = counts(cell.cameras, entries);
        console.log(
          `cell ${cell.key}: ${cell.cameras.length} cameras, ${tally.ok} ok/${tally.miss} miss, ${((Date.now() - cellStarted) / 1000).toFixed(1)} s`,
        );
      }
    }
  } finally {
    if (browser) await browser.close();
  }
  const entries = sidecar?.cameras || {};
  const tally = counts(selected, entries);
  console.log(
    `Total: ${selected.length} cameras, ${tally.ok} ok/${tally.miss} miss, ${((Date.now() - started) / 1000).toFixed(1)} s`,
  );
  for (const kind of [
    ...new Set(selected.map((camera) => camera.sourceKind)),
  ].sort()) {
    const tally = counts(
      selected.filter((camera) => camera.sourceKind === kind),
      entries,
    );
    console.log(`  ${kind}: ${tally.ok} ok/${tally.miss} miss`);
  }
}

if (
  process.argv[1] &&
  import.meta.url === pathToFileURL(process.argv[1]).href
) {
  main().catch((error) => {
    console.error(error.message);
    process.exitCode = 1;
  });
}