gods-eye-view/server/providers/terrain.js

import path from 'node:path';
import { promises as fsp } from 'node:fs';

import {
  fetchTerrainChunkWithRetry,
  parseTerrainPoints,
  resolveTerrainHeightRequest,
  terrainPointKey,
  terrainRetryAfterMs,
  validTerrainResult,
} from '../../src/data/terrainHeightsProxy.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSiteKeyless } from './common/same-site.js';

/**
 * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height
 * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json
 * (UPSTREAM_CHUNK points per call — sized against measured latency, see
 * below). Terrain doesn't move, so results are cached to
 * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk
 * cache and serve-stale shape. Cache entries and stale fallback are keyed per
 * 5dp point, so reordered and partially overlapping batches reuse prior work.
 * Only missing/stale points go upstream; the response is rebuilt in exact
 * request order. Each request is bounded to one 64-point upstream chunk;
 * larger client workloads are split by the browser terrain service.
 *
 * Bounds, because every distinct 5dp point is its own key: coordinates must be
 * in WGS-84 range (rejected at parse), the point cache has an entry ceiling
 * with oldest-first eviction, an over-large cache file is not read back, and
 * the route carries an always-on per-client limiter. The route also refuses
 * cross-site browser requests, so another web page cannot use the visitor's
 * browser to fill the cache or spend the visitor's own rate-limit quota.
 * Reverse-proxied same-origin requests are admitted: terrain is keyless and
 * needed for rendering, and a 403 would send every chunk to the geoid fallback.
 */
export function terrainHeightsProxy(options = {}) {
  // Destructured in the body, not the signature: `proxyErrorResponses.test.mjs`
  // extracts these proxies by slicing to the first closing brace in column
  // zero, so a multi-line parameter list would cut the function in half.
  // Overridable so the ceilings can be exercised as behaviour rather than
  // asserted as constants; production constructs this with no arguments.
  const {
    cacheMaxEntries = 50_000,
    cacheMaxDiskBytes = 24 * 1024 * 1024,
    cachePath,
  } = options;
  const TTL_MS = 30 * 24 * 3600_000;
  const CACHE_PATH =
    cachePath || path.join(process.cwd(), '.gev-cache', 'terrain-heights.json');
  const CACHE_DIR = path.dirname(CACHE_PATH);
  // Sized against measured upstream latency, not the documented page cap.
  // Re:Earth serves 87-186 ms/point depending on load (observed 2026-09-12,
  // a 2x swing within one hour). 256 points therefore costs 22-48 s and blows
  // the 30 s attempt timeout whenever the service is slow — the long-standing
  // cause of "[terrain-heights-proxy] refresh incomplete" (owner incident
  // 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
  // that range.
  const UPSTREAM_CHUNK = 64;
  // After a 429 the upstream is left alone for its Retry-After (at most
  // RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a
  // restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for
  // new points (about once a second while the camera moves) otherwise asks
  // again with its own retries, which keeps the limit in force and logs each
  // time.
  const RATE_LIMIT_PAUSE_MS = 60_000;
  const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000;
  let rateLimitedUntil = 0;
  const MAX_POINTS = UPSTREAM_CHUNK;
  /**
   * Entry ceiling for the point cache, and with it the file written from it.
   *
   * Every distinct 5dp point is a separate key, so without a ceiling the map
   * — and the JSON dumped from it verbatim — grow for as long as a caller
   * supplies new coordinates. Measured against real use rather than guessed:
   * this checkout's `.gev-cache/terrain-heights.json` held 6,189 entries in
   * 1.0 MB after ordinary sessions, i.e. ~168 bytes/entry. 50,000 entries is
   * eight times that working set and ~8 MB on disk, comfortably inside the
   * 24 MB ceiling `rocketLaunchesProxy` already applies to its own cache file.
   */
  const CACHE_MAX_ENTRIES = cacheMaxEntries;
  /** Refuse to load a cache file larger than this (mirrors rocketLaunchesProxy). */
  const CACHE_MAX_DISK_BYTES = cacheMaxDiskBytes;
  /**
   * Requests/min/client. Terrain is keyless and free, so this guards the point
   * cache and the upstream community service rather than a bill. Sized from
   * measurement: across a full `npm run test:track` run — the heaviest
   * exercise this app has — the route served 10 requests, peaking at 8/min.
   * 90/min is an order of magnitude above that, and the band `/api/overpass`
   * and `/api/military-installations` already use.
   */
  const RATE_PER_MIN = 90;
  const allow = makeRateLimiter({
    windowMs: 60_000,
    max: RATE_PER_MIN,
    globalMax: RATE_PER_MIN * 3,
  });

  /** @type {Map<string, {at:number, result:object}>} keyed by canonical 5dp lon/lat. */
  const mem = new Map();

  /**
   * Drop the oldest entries until the map is back inside its ceiling.
   * The map is kept in fetch-time order (sorted once on load, and the resolver
   * re-inserts every point it refetches), so the oldest are at the front and
   * no write sorts the map. Terrain does not move, so evicting an old point
   * costs one refetch, never a wrong answer.
   */
  function evictOverflow() {
    for (const key of mem.keys()) {
      if (mem.size <= CACHE_MAX_ENTRIES) return;
      mem.delete(key);
    }
  }

  /** Put the map in fetch-time order; a file is not guaranteed to be. */
  function orderByAge() {
    const byAge = [...mem.entries()].sort((a, b) => a[1].at - b[1].at);
    mem.clear();
    for (const [key, entry] of byAge) mem.set(key, entry);
  }
  /** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
  const inflight = new Map();
  let diskLoaded = false;
  let diskDirty = false;

  /** Load the on-disk cache into memory once, lazily (first request only). */
  async function loadDiskOnce() {
    if (diskLoaded) return;
    diskLoaded = true;
    try {
      // A cache file past the ceiling is not read back into memory: loading it
      // would reinstate exactly the unbounded map the ceiling exists to stop.
      // The next flush rewrites a bounded one.
      //
      // "Cannot measure it" is not "too large": if stat is unavailable, fall
      // through and let readFile decide, so an absent file behaves exactly as
      // it did before this guard existed.
      let sizeBytes = null;
      try {
        sizeBytes = (await fsp.stat(CACHE_PATH)).size;
      } catch {
        sizeBytes = null;
      }
      if (sizeBytes !== null && sizeBytes > CACHE_MAX_DISK_BYTES)
        throw new Error('cache file too large');
      const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
      const pointEntries =
        parsed?.version === 2 &&
        parsed.points &&
        typeof parsed.points === 'object'
          ? parsed.points
          : null;
      if (pointEntries) {
        for (const [key, entry] of Object.entries(pointEntries)) {
          if (
            entry &&
            Number.isFinite(entry.at) &&
            validTerrainResult(entry.result)
          ) {
            mem.set(key, entry);
          }
        }
      } else if (parsed && typeof parsed === 'object') {
        // One-time migration from the former raw-batch cache. Zip only real,
        // positionally present results; an absent value never becomes height 0.
        for (const [rawPoints, entry] of Object.entries(parsed)) {
          const points = parseTerrainPoints(rawPoints);
          if (
            !points ||
            !entry ||
            !Number.isFinite(entry.at) ||
            !Array.isArray(entry.results)
          )
            continue;
          for (let i = 0; i < points.length; i += 1) {
            const result = entry.results[i];
            if (!validTerrainResult(result)) continue;
            const key = terrainPointKey(points[i]);
            const existing = mem.get(key);
            if (!existing || entry.at > existing.at)
              mem.set(key, { at: entry.at, result });
          }
        }
        diskDirty = mem.size > 0;
      }
      orderByAge();
      evictOverflow();
    } catch {
      /* no disk cache yet */
    }
    // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of
    // hitting disk on every request.
    setInterval(async () => {
      if (!diskDirty) return;
      diskDirty = false;
      try {
        await fsp.mkdir(CACHE_DIR, { recursive: true });
        const obj = { version: 2, points: Object.fromEntries(mem.entries()) };
        await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8');
      } catch (err) {
        diskDirty = true; // retry next tick
        console.warn('[terrain-heights-proxy] cache write failed');
      }
    }, 15_000).unref?.();
  }

  /**
   * Fetch all missing chunks sequentially, UPSTREAM_CHUNK points per call.
   *
   * The first try keeps its empirically required 30s timeout; network errors,
   * 429, and 5xx receive up to three jittered retries sharing a 10s added-time
   * budget, with Retry-After honored within that bound.
   *
   * A chunk that still fails yields nulls for its own positions rather than
   * rejecting: with several chunks per request, throwing would discard every
   * chunk that did resolve and re-fetch it on the next poll. The resolver
   * caches the resolved positions and reports the null ones as omitted. Total
   * failure (nothing resolved) still rejects, so the real upstream error
   * reaches the caller instead of a generic omission count.
   * @param {Array<[number, number]>} points
   * @returns {Promise<Array<object>>}
   */
  async function fetchUpstreamAll(points) {
    if (Date.now() < rateLimitedUntil) {
      // Rate-limited: asking again only extends the limit. Cached points are
      // still served; the rest resolve on the page through its geoid.
      const error = new Error('HTTP 429');
      error.coolingDown = true;
      throw error;
    }
    const results = [];
    let firstError = null;
    for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
      const chunk = points.slice(i, i + UPSTREAM_CHUNK);
      let chunkResults = [];
      if (Date.now() >= rateLimitedUntil) {
        try {
          chunkResults = await fetchTerrainChunkWithRetry(chunk);
        } catch (error) {
          firstError = firstError || error;
          if (error?.message === 'HTTP 429')
            rateLimitedUntil =
              Date.now() +
              Math.min(
                terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS,
                RATE_LIMIT_PAUSE_MAX_MS,
              );
        }
      }
      // Keep later chunks aligned even if a malformed upstream response omits
      // trailing positions. The resolver will reject each null individually.
      for (let j = 0; j < chunk.length; j += 1)
        results.push(chunkResults[j] ?? null);
    }
    if (firstError && results.every((result) => result == null))
      throw firstError;
    return results;
  }

  /** Coalesce concurrent requests for the same canonical missing-point list. */
  function fetchMissingSingleFlight(points) {
    const key = points.map(terrainPointKey).join(';');
    if (!inflight.has(key)) {
      const request = fetchUpstreamAll(points).finally(() => {
        if (inflight.get(key) === request) inflight.delete(key);
      });
      inflight.set(key, request);
    }
    return inflight.get(key);
  }

  const installMiddleware = (server) => {
    server.middlewares.use('/api/terrain/heights', async (req, res) => {
      // Refuse cross-site browser requests (an <img> or fetch from another
      // site) before the limiter, cache or upstream see them. Same-origin app
      // requests, non-browser loopback callers (MCP panel_request) and the app
      // served behind a reverse proxy pass.
      if (admitSameSiteKeyless(req, res)) return;
      const send = (status, bodyObj) => {
        if (res.headersSent) return;
        res.writeHead(status, { 'Content-Type': 'application/json' });
        res.end(JSON.stringify(bodyObj));
      };
      try {
        if (!allow(clientKey(req))) {
          res.writeHead(429, {
            'Content-Type': 'application/json',
            'Retry-After': '10',
          });
          res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
          return;
        }
        const parsedUrl = new URL(req.url || '', 'http://internal');
        const rawPoints = parsedUrl.searchParams.get('points');
        const points = parseTerrainPoints(rawPoints);
        if (!points) {
          send(400, {
            error:
              'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers',
          });
          return;
        }
        if (points.length > MAX_POINTS) {
          send(413, {
            error: `too many points (${points.length}); max ${MAX_POINTS} per request`,
          });
          return;
        }

        await loadDiskOnce();
        const outcome = await resolveTerrainHeightRequest({
          points,
          cache: mem,
          fetchMissing: fetchMissingSingleFlight,
          ttlMs: TTL_MS,
        });
        if (outcome.cacheChanged) {
          evictOverflow();
          diskDirty = true;
        }
        if (outcome.upstreamError?.coolingDown) {
          // Logged once, when the pause began.
        } else if (outcome.upstreamError) {
          const pause =
            outcome.upstreamError.message === 'HTTP 429' &&
            Date.now() < rateLimitedUntil
              ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)`
              : '';
          console.warn(
            '[terrain-heights-proxy] refresh incomplete' +
              pause +
              ' — serving stale points when available',
          );
        } else if (outcome.absentPoints > 0) {
          // Not a refresh failure. The upstream answered every position and
          // returned a null height for a few of them (~0.16%, transient);
          // the next poll re-asks and the client meanwhile resolves those
          // through its bundled geoid. Informational, not actionable.
          console.info(
            `[terrain-heights-proxy] ${outcome.absentPoints}` +
              ` of ${outcome.requestedPoints} position(s) had no upstream` +
              ' height this poll — retrying next cycle',
          );
        }
        send(outcome.status, outcome.body);
      } catch (err) {
        console.error('[terrain-heights-proxy] request failed');
        send(500, { error: 'terrain heights proxy error' });
      }
    });
  };
  return {
    name: 'terrain-heights-proxy',
    configureServer: installMiddleware,
    configurePreviewServer: installMiddleware,
  };
}