import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { fetchTerrainChunkWithRetry, parseTerrainPoints, resolveTerrainHeightRequest, terrainPointKey, terrainRetryAfterMs, validTerrainResult, } from '../../src/data/terrainHeightsProxy.js'; import { makeRateLimiter, clientKey } from './common/rate-limit.js'; import { admitSameSiteKeyless } from './common/same-site.js'; /** * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json * (UPSTREAM_CHUNK points per call — sized against measured latency, see * below). Terrain doesn't move, so results are cached to * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk * cache and serve-stale shape. Cache entries and stale fallback are keyed per * 5dp point, so reordered and partially overlapping batches reuse prior work. * Only missing/stale points go upstream; the response is rebuilt in exact * request order. Each request is bounded to one 64-point upstream chunk; * larger client workloads are split by the browser terrain service. * * Bounds, because every distinct 5dp point is its own key: coordinates must be * in WGS-84 range (rejected at parse), the point cache has an entry ceiling * with oldest-first eviction, an over-large cache file is not read back, and * the route carries an always-on per-client limiter. The route also refuses * cross-site browser requests, so another web page cannot use the visitor's * browser to fill the cache or spend the visitor's own rate-limit quota. * Reverse-proxied same-origin requests are admitted: terrain is keyless and * needed for rendering, and a 403 would send every chunk to the geoid fallback. */ export function terrainHeightsProxy(options = {}) { // Destructured in the body, not the signature: `proxyErrorResponses.test.mjs` // extracts these proxies by slicing to the first closing brace in column // zero, so a multi-line parameter list would cut the function in half. // Overridable so the ceilings can be exercised as behaviour rather than // asserted as constants; production constructs this with no arguments. const { cacheMaxEntries = 50_000, cacheMaxDiskBytes = 24 * 1024 * 1024, cachePath, } = options; const TTL_MS = 30 * 24 * 3600_000; const CACHE_PATH = cachePath || path.join(process.cwd(), '.gev-cache', 'terrain-heights.json'); const CACHE_DIR = path.dirname(CACHE_PATH); // Sized against measured upstream latency, not the documented page cap. // Re:Earth serves 87-186 ms/point depending on load (observed 2026-09-12, // a 2x swing within one hour). 256 points therefore costs 22-48 s and blows // the 30 s attempt timeout whenever the service is slow — the long-standing // cause of "[terrain-heights-proxy] refresh incomplete" (owner incident // 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of // that range. const UPSTREAM_CHUNK = 64; // After a 429 the upstream is left alone for its Retry-After (at most // RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a // restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for // new points (about once a second while the camera moves) otherwise asks // again with its own retries, which keeps the limit in force and logs each // time. const RATE_LIMIT_PAUSE_MS = 60_000; const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000; let rateLimitedUntil = 0; const MAX_POINTS = UPSTREAM_CHUNK; /** * Entry ceiling for the point cache, and with it the file written from it. * * Every distinct 5dp point is a separate key, so without a ceiling the map * — and the JSON dumped from it verbatim — grow for as long as a caller * supplies new coordinates. Measured against real use rather than guessed: * this checkout's `.gev-cache/terrain-heights.json` held 6,189 entries in * 1.0 MB after ordinary sessions, i.e. ~168 bytes/entry. 50,000 entries is * eight times that working set and ~8 MB on disk, comfortably inside the * 24 MB ceiling `rocketLaunchesProxy` already applies to its own cache file. */ const CACHE_MAX_ENTRIES = cacheMaxEntries; /** Refuse to load a cache file larger than this (mirrors rocketLaunchesProxy). */ const CACHE_MAX_DISK_BYTES = cacheMaxDiskBytes; /** * Requests/min/client. Terrain is keyless and free, so this guards the point * cache and the upstream community service rather than a bill. Sized from * measurement: across a full `npm run test:track` run — the heaviest * exercise this app has — the route served 10 requests, peaking at 8/min. * 90/min is an order of magnitude above that, and the band `/api/overpass` * and `/api/military-installations` already use. */ const RATE_PER_MIN = 90; const allow = makeRateLimiter({ windowMs: 60_000, max: RATE_PER_MIN, globalMax: RATE_PER_MIN * 3, }); /** @type {Map} keyed by canonical 5dp lon/lat. */ const mem = new Map(); /** * Drop the oldest entries until the map is back inside its ceiling. * The map is kept in fetch-time order (sorted once on load, and the resolver * re-inserts every point it refetches), so the oldest are at the front and * no write sorts the map. Terrain does not move, so evicting an old point * costs one refetch, never a wrong answer. */ function evictOverflow() { for (const key of mem.keys()) { if (mem.size <= CACHE_MAX_ENTRIES) return; mem.delete(key); } } /** Put the map in fetch-time order; a file is not guaranteed to be. */ function orderByAge() { const byAge = [...mem.entries()].sort((a, b) => a[1].at - b[1].at); mem.clear(); for (const [key, entry] of byAge) mem.set(key, entry); } /** @type {Map>>} single-flight per missing-point subset. */ const inflight = new Map(); let diskLoaded = false; let diskDirty = false; /** Load the on-disk cache into memory once, lazily (first request only). */ async function loadDiskOnce() { if (diskLoaded) return; diskLoaded = true; try { // A cache file past the ceiling is not read back into memory: loading it // would reinstate exactly the unbounded map the ceiling exists to stop. // The next flush rewrites a bounded one. // // "Cannot measure it" is not "too large": if stat is unavailable, fall // through and let readFile decide, so an absent file behaves exactly as // it did before this guard existed. let sizeBytes = null; try { sizeBytes = (await fsp.stat(CACHE_PATH)).size; } catch { sizeBytes = null; } if (sizeBytes !== null && sizeBytes > CACHE_MAX_DISK_BYTES) throw new Error('cache file too large'); const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' ? parsed.points : null; if (pointEntries) { for (const [key, entry] of Object.entries(pointEntries)) { if ( entry && Number.isFinite(entry.at) && validTerrainResult(entry.result) ) { mem.set(key, entry); } } } else if (parsed && typeof parsed === 'object') { // One-time migration from the former raw-batch cache. Zip only real, // positionally present results; an absent value never becomes height 0. for (const [rawPoints, entry] of Object.entries(parsed)) { const points = parseTerrainPoints(rawPoints); if ( !points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results) ) continue; for (let i = 0; i < points.length; i += 1) { const result = entry.results[i]; if (!validTerrainResult(result)) continue; const key = terrainPointKey(points[i]); const existing = mem.get(key); if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); } } diskDirty = mem.size > 0; } orderByAge(); evictOverflow(); } catch { /* no disk cache yet */ } // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of // hitting disk on every request. setInterval(async () => { if (!diskDirty) return; diskDirty = false; try { await fsp.mkdir(CACHE_DIR, { recursive: true }); const obj = { version: 2, points: Object.fromEntries(mem.entries()) }; await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8'); } catch (err) { diskDirty = true; // retry next tick console.warn('[terrain-heights-proxy] cache write failed'); } }, 15_000).unref?.(); } /** * Fetch all missing chunks sequentially, UPSTREAM_CHUNK points per call. * * The first try keeps its empirically required 30s timeout; network errors, * 429, and 5xx receive up to three jittered retries sharing a 10s added-time * budget, with Retry-After honored within that bound; a longer Retry-After * ends the retries at once so the 429 pause below starts without delay. * * A chunk that still fails yields nulls for its own positions rather than * rejecting: with several chunks per request, throwing would discard every * chunk that did resolve and re-fetch it on the next poll. The resolver * caches the resolved positions and reports the null ones as omitted. Total * failure (nothing resolved) still rejects, so the real upstream error * reaches the caller instead of a generic omission count. * @param {Array<[number, number]>} points * @returns {Promise>} */ async function fetchUpstreamAll(points) { if (Date.now() < rateLimitedUntil) { // Rate-limited: asking again only extends the limit. Cached points are // still served; the rest resolve on the page through its geoid. const error = new Error('HTTP 429'); error.status = 429; error.coolingDown = true; throw error; } const results = []; let firstError = null; for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { const chunk = points.slice(i, i + UPSTREAM_CHUNK); let chunkResults = []; // Re-checked per chunk for when a request spans several chunks again; // today requests are capped at one chunk (MAX_POINTS). if (Date.now() >= rateLimitedUntil) { try { chunkResults = await fetchTerrainChunkWithRetry(chunk); } catch (error) { firstError = firstError || error; // Concurrent 429s extend the pause, never cut it short. if (error?.status === 429) rateLimitedUntil = Math.max( rateLimitedUntil, Date.now() + Math.min( terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, RATE_LIMIT_PAUSE_MAX_MS, ), ); } } // Keep later chunks aligned even if a malformed upstream response omits // trailing positions. The resolver will reject each null individually. for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); } if (firstError && results.every((result) => result == null)) throw firstError; return results; } /** Coalesce concurrent requests for the same canonical missing-point list. */ function fetchMissingSingleFlight(points) { const key = points.map(terrainPointKey).join(';'); if (!inflight.has(key)) { const request = fetchUpstreamAll(points).finally(() => { if (inflight.get(key) === request) inflight.delete(key); }); inflight.set(key, request); } return inflight.get(key); } const installMiddleware = (server) => { server.middlewares.use('/api/terrain/heights', async (req, res) => { // Refuse cross-site browser requests (an or fetch from another // site) before the limiter, cache or upstream see them. Same-origin app // requests, non-browser loopback callers (MCP panel_request) and the app // served behind a reverse proxy pass. if (admitSameSiteKeyless(req, res)) return; const send = (status, bodyObj) => { if (res.headersSent) return; res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(bodyObj)); }; try { if (!allow(clientKey(req))) { res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '10', }); res.end(JSON.stringify({ error: 'Rate limit exceeded' })); return; } const parsedUrl = new URL(req.url || '', 'http://internal'); const rawPoints = parsedUrl.searchParams.get('points'); const points = parseTerrainPoints(rawPoints); if (!points) { send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers', }); return; } if (points.length > MAX_POINTS) { send(413, { error: `too many points (${points.length}); max ${MAX_POINTS} per request`, }); return; } await loadDiskOnce(); const outcome = await resolveTerrainHeightRequest({ points, cache: mem, fetchMissing: fetchMissingSingleFlight, ttlMs: TTL_MS, }); if (outcome.cacheChanged) { evictOverflow(); diskDirty = true; } if (outcome.upstreamError?.coolingDown) { // Logged once, when the pause began. } else if (outcome.upstreamError) { const pause = outcome.upstreamError.status === 429 && Date.now() < rateLimitedUntil ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)` : ''; console.warn( '[terrain-heights-proxy] refresh incomplete' + pause + ' — serving stale points when available', ); } else if (outcome.absentPoints > 0) { // Not a refresh failure. The upstream answered every position and // returned a null height for a few of them (~0.16%, transient); // the next poll re-asks and the client meanwhile resolves those // through its bundled geoid. Informational, not actionable. console.info( `[terrain-heights-proxy] ${outcome.absentPoints}` + ` of ${outcome.requestedPoints} position(s) had no upstream` + ' height this poll — retrying next cycle', ); } send(outcome.status, outcome.body); } catch (err) { console.error('[terrain-heights-proxy] request failed'); send(500, { error: 'terrain heights proxy error' }); } }); }; return { name: 'terrain-heights-proxy', configureServer: installMiddleware, configurePreviewServer: installMiddleware, }; }