fix(osm): stop using public Overpass by default (#648) (#742)

Traffic roads come from OpenFreeMap vector tiles, or TomTom flow tiles with a key (Hybrid by default). Military areas use OpenFreeMap polygons with a bundled worldwide name index; wide views show named points. ALPR reads an hourly OpenStreetMap extract. Cockpit place context and country, state and US county outlines use bundled Natural Earth and US Census data. Overpass is used only when an operator configures their own endpoint. Also: camera arrivals no longer land below ground, keyless terrain retries on 429, bounded relation geometry for configured Overpass, one shared OpenStreetMap credit, and separate tile and API transports for embedders. Co-authored-by: theGaryLarson <[email protected]> Co-authored-by: Milan Khanal <[email protected]>
2026-11-05 10:53:59 +00:00 · 2026-09-28 19:07:57 -05:00
1 parent 81eb44340d
commit f4fcf267c6
@@ -5,7 +5,11 @@ import {
  MILITARY_INSTALLATION_DISK_TTL_MS,
  MILITARY_INSTALLATION_STALE_MS,
 } from './military-installations/constants.js';
-import { fetchOverpassPayload } from './overpass/transport.js';
+import {
+  fetchOverpassPayload,
+  overpassNotConfigured,
+} from './overpass/transport.js';
+import { resolveOverpassUpstreams } from './overpass/constants.js';
 import {
  _militaryInstallationCache,
  trimMilitaryInstallationCache,
@@ -45,6 +49,7 @@ function militaryInstallationsProxy() {
      throw Object.assign(
        new Error('Mapped installation upstream unavailable'),
        {
+          retryAfterMs: upstream.retryAfterMs,
          installationReason: upstream.rateLimited
            ? 'rate_limited'
            : upstream.status === 504
@@ -83,14 +88,6 @@ function militaryInstallationsProxy() {
        res.end(JSON.stringify({ error: 'Method Not Allowed' }));
        return;
      }
-      if (!_militaryInstallationsRateLimiter(clientKey(req))) {
-        res.writeHead(429, {
-          'Content-Type': 'application/json',
-          'Retry-After': '5',
-        });
-        res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
-        return;
-      }
      const url = new URL(req.url, 'http://localhost');
      const requested = validMilitaryInstallationBox(url.searchParams);
      if (!requested) {
@@ -117,6 +114,36 @@ function militaryInstallationsProxy() {
        : militaryInstallationCacheKey(box);
      const now = Date.now();
      const cached = _militaryInstallationCache.get(key);
+      if (!resolveOverpassUpstreams().length) {
+        const stale =
+          cached || (await readMilitaryInstallationDisk(key, Infinity));
+        const disabled = overpassNotConfigured();
+        res.writeHead(200, {
+          'Content-Type': 'application/json',
+          'Cache-Control': 'no-store',
+          'X-Military-Installations': stale ? 'STALE' : 'DISABLED',
+        });
+        res.end(
+          stale
+            ? JSON.stringify({
+                ...stale.payload,
+                status: 'stale',
+                retrievedAt:
+                  stale.payload.retrievedAt ||
+                  new Date(stale.cachedAt).toISOString(),
+              })
+            : disabled.body,
+        );
+        return;
+      }
+      if (!_militaryInstallationsRateLimiter(clientKey(req))) {
+        res.writeHead(429, {
+          'Content-Type': 'application/json',
+          'Retry-After': '5',
+        });
+        res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
+        return;
+      }
      const preflight = await resolveMilitaryInstallationTier({
        cacheKey: key,
        memoryCache: _militaryInstallationCache,
@@ -176,6 +203,9 @@ function militaryInstallationsProxy() {
          return;
        }
        res.writeHead(503, {
+          ...(error?.retryAfterMs
+            ? { 'Retry-After': String(Math.ceil(error.retryAfterMs / 1000)) }
+            : {}),
          'Content-Type': 'application/json',
          'Cache-Control': 'no-store',
        });
@@ -183,6 +213,7 @@ function militaryInstallationsProxy() {
          JSON.stringify({
            error: 'Mapped installation context is temporarily unavailable',
            reason: militaryInstallationFailureReason(error),
+            retryAfterMs: error?.retryAfterMs,
          }),
        );
      }
@@ -3,6 +3,7 @@ import { readRequestBodyCapped } from './common/request.js';
 import {
  OVERPASS_MAX_BODY_BYTES,
  OVERPASS_MAX_CONCURRENT,
+  resolveOverpassUpstreams,
 } from './overpass/constants.js';
 import { sanitizeOverpassBody } from './overpass/query.js';
 import {
@@ -17,6 +18,7 @@ import {
 import {
  overpassPayloadIsData,
  fetchOverpassPayload,
+  overpassNotConfigured,
 } from './overpass/transport.js';
 import { installRouteMiddleware } from './places/routes.js';

@@ -41,9 +43,16 @@ const _overpassRateLimiter = makeRateLimiter({
 function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
  res.writeHead(payload.status, {
    'Content-Type': payload.contentType || 'application/json',
-    'Cache-Control': 'public, max-age=15',
+    'Cache-Control': overpassPayloadIsData(payload)
+      ? 'public, max-age=15'
+      : 'no-store',
    'X-Overpass-Cache': cacheStatus,
-    'X-Overpass-Upstream': payload.endpoint || 'unknown',
+    ...(Number.isFinite(payload.cachedAt)
+      ? { 'X-Overpass-Cached-At': new Date(payload.cachedAt).toISOString() }
+      : {}),
+    ...(payload.retryAfterMs
+      ? { 'Retry-After': String(Math.ceil(payload.retryAfterMs / 1000)) }
+      : {}),
  });
  res.end(payload.body || '');
 }
@@ -52,7 +61,7 @@ function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
 * Vite plugin: Overpass API proxy with response caching and request coalescing.
 *
 * Accepts POST requests at /api/overpass, normalizes the query body for
- * cache keying, and fans out to multiple Overpass mirrors with per-upstream
+ * cache keying, and uses only operator-configured endpoints with per-upstream
 * timeout and rate-limit detection. Successful responses are cached for
 * OVERPASS_CACHE_MS. Concurrent identical queries share a single upstream
 * request via the in-flight map.
@@ -66,6 +75,19 @@ function overpassProxy({ routing = {} } = {}) {
      // (a body-read failure would otherwise hit an out-of-scope reference).
      let cacheKey = null;
      try {
+        // Capability probe: lets clients skip queries when nothing is configured.
+        if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
+          res.writeHead(200, {
+            'Content-Type': 'application/json',
+            'Cache-Control': 'no-store',
+          });
+          res.end(
+            JSON.stringify({
+              configured: resolveOverpassUpstreams().length > 0,
+            }),
+          );
+          return;
+        }
        if (req.method !== 'POST') {
          res.writeHead(405, { 'Content-Type': 'application/json' });
          res.end(JSON.stringify({ error: 'Method Not Allowed' }));
@@ -104,12 +126,21 @@ function overpassProxy({ routing = {} } = {}) {

        // Normalize whitespace so semantically identical Overpass QL queries share cache entries
        cacheKey = safeBody.replace(/\s+/g, ' ').trim();
+        if (!resolveOverpassUpstreams().length) {
+          const cached = await readStaleOverpass(cacheKey);
+          sendOverpassResponse(
+            res,
+            cached || overpassNotConfigured(),
+            cached ? 'STALE' : 'DISABLED',
+          );
+          return;
+        }
        const preflight = await resolveOverpassPreflight({
          cacheKey,
          memoryCache: _overpassCache,
          inFlight: _overpassInFlight,
-          // Fresh-enough disk entries survive restarts and skip the public
-          // mirrors; boundary-class queries keep their month-long TTL.
+          // Fresh-enough disk entries survive restarts and skip upstream
+          // requests; boundary-class queries keep their month-long TTL.
          readDisk: () =>
            readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
          allowUpstream: () => _overpassRateLimiter(clientKey(req)),
@@ -194,7 +225,7 @@ function overpassProxy({ routing = {} } = {}) {
          sendOverpassResponse(res, stale, 'STALE');
          return;
        }
-        console.error('[Overpass Proxy]', e.message);
+        console.error('[Overpass Proxy] request failed');
        res.writeHead(502, { 'Content-Type': 'application/json' });
        res.end(JSON.stringify({ error: 'Overpass proxy error' }));
      }
@@ -3,30 +3,39 @@ import path from 'node:path';
 // ---------------------------------------------------------------------------
 // Overpass API proxy constants and cache state
 // ---------------------------------------------------------------------------
-/**
- * User-Agent sent to every Overpass mirror.
- *
- * The OSM API usage policy asks for a "Valid User-Agent identifying application
- * and version"; a generic proxy label is not one. A mirror is free to refuse a
- * client it cannot identify, and `src/overpassProxy.test.mjs` pins what that
- * costs: a refusal is never data, so the query falls through to whatever
- * mirrors are left. Keep this honest and stable — if it is ever refused, the
- * answer is less query volume, not a new name.
- */
+/** Stable application identity for operator-configured Overpass instances. */
 const OVERPASS_USER_AGENT =
  'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)';

-/** Ordered list of Overpass API mirrors; tried sequentially on failure/rate-limit. */
-const OVERPASS_UPSTREAMS = [
-  'https://overpass-api.de/api/interpreter',
-  'https://overpass.kumi.systems/api/interpreter',
-  'https://lz4.overpass-api.de/api/interpreter',
-  // Community full-planet instance (privateforge nonprofit) — added 2026-07-30
-  // when all three mirrors above refused this IP (likely a dev-traffic rate
-  // ban; refused connections fail in ms, so healthy mirrors above still win).
-  // Verified: planet coverage (Texas query), CORS *, ~5-20 s cold latency.
-  'https://overpass.private.coffee/api/interpreter',
-];
+/** Parse only operator-supplied HTTP(S) endpoints; private instances are allowed. */
+function parseOverpassUpstreams(raw) {
+  const endpoints = [];
+  for (const token of String(raw || '').split(',')) {
+    try {
+      const url = new URL(token.trim());
+      if (
+        !['http:', 'https:'].includes(url.protocol) ||
+        !url.hostname ||
+        url.hash
+      )
+        continue;
+      if (!endpoints.includes(url.href)) endpoints.push(url.href);
+    } catch {
+      // Invalid configuration never becomes an upstream or appears in logs.
+    }
+  }
+  return endpoints.slice(0, 8);
+}
+
+let upstreamMemo = { raw: null, endpoints: [] };
+
+/** Resolve after environment loading. Public Overpass instances are not used by default. */
+function resolveOverpassUpstreams() {
+  const raw = process.env.OVERPASS_UPSTREAMS || '';
+  if (upstreamMemo.raw !== raw)
+    upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) };
+  return [...upstreamMemo.endpoints];
+}

 /**
 * TTL for FRESH cached Overpass responses (ms). Road geometry is static for
@@ -142,7 +151,8 @@ export {
  OVERPASS_SIMPLIFY_MIN_POINTS,
  OVERPASS_SIMPLIFY_TOLERANCE_DEG,
  OVERPASS_MAX_RESPONSE_BYTES,
-  OVERPASS_UPSTREAMS,
+  parseOverpassUpstreams,
+  resolveOverpassUpstreams,
  OVERPASS_USER_AGENT,
  OVERPASS_TIMEOUT_MS,
 };
@@ -1,6 +1,6 @@
 import {
  OVERPASS_MAX_RESPONSE_BYTES,
-  OVERPASS_UPSTREAMS,
+  resolveOverpassUpstreams,
  OVERPASS_USER_AGENT,
  OVERPASS_TIMEOUT_MS,
 } from './constants.js';
@@ -60,99 +60,147 @@ function overpassPayloadIsData(payload) {
  );
 }

+const cooldowns = new Map();
+
+/** A stable, non-retryable capability response shared by every Overpass route. */
+export function overpassNotConfigured() {
+  return {
+    status: 503,
+    contentType: 'application/json',
+    body: JSON.stringify({
+      error: 'Detailed OpenStreetMap queries are not configured',
+      code: 'OVERPASS_NOT_CONFIGURED',
+      retryable: false,
+    }),
+  };
+}
+
+/** Honor Retry-After dates/seconds; absent values use bounded exponential backoff. */
+function retryDelay(value, failures, now) {
+  const seconds = Number(value);
+  const explicit =
+    value && Number.isFinite(seconds)
+      ? seconds * 1000
+      : Date.parse(value) - now;
+  return Number.isFinite(explicit)
+    ? Math.max(1000, explicit)
+    : Math.min(300_000, 30_000 * 2 ** Math.min(failures, 4));
+}
+
+function refusal(status, retryAfterMs) {
+  return {
+    status,
+    contentType: 'application/json',
+    rateLimited: status === 429 || status === 406,
+    retryAfterMs,
+    body: JSON.stringify({
+      error: 'Configured Overpass upstream unavailable',
+      code: 'OVERPASS_UNAVAILABLE',
+      retryable: true,
+      retryAfterMs,
+    }),
+  };
+}
+
 /**
- * Try each mirror once, retaining response-size and per-mirror timeout caps.
- * Refusals and body-level failures rotate; total failure returns the last
- * rate-limit payload, otherwise the first refusal, or throws a network error.
- * @param {string} body URL-encoded Overpass QL query body.
- * @param {number} [maxResponseBytes] Endpoint-specific response cap.
- * @param {object} [options] Server-only endpoint and I/O overrides for tests.
- * @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
+ * Query only the configured chain with capped reads, timeouts and per-endpoint cooldowns.
+ * Explicit endpoint/I/O overrides are server-only test seams. Empty data is valid.
 */
 async function fetchOverpassPayload(
  body,
  maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES,
  {
-    endpoints = OVERPASS_UPSTREAMS,
+    endpoints = resolveOverpassUpstreams(),
    fetchImpl = fetch,
    readBody = readResponseTextCapped,
    simplify = simplifyOverpassPayloadBody,
+    now = Date.now,
  } = {},
 ) {
-  let lastError = null;
-  let lastRateLimitPayload = null;
-  let lastRefusalPayload = null;
-
+  if (!endpoints.length) return overpassNotConfigured();
+  let failure = refusal(502, 30_000);
  for (const endpoint of endpoints) {
+    const previous = cooldowns.get(endpoint);
+    if (previous?.until > now()) {
+      failure = refusal(previous.status, previous.until - now());
+      continue;
+    }
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
-
    try {
-      const upstream = await fetchImpl(endpoint, {
+      const requestUrl = new URL(endpoint);
+      const authorization =
+        requestUrl.username || requestUrl.password
+          ? 'Basic ' +
+            Buffer.from(
+              `${decodeURIComponent(requestUrl.username)}:${decodeURIComponent(requestUrl.password)}`,
+            ).toString('base64')
+          : null;
+      requestUrl.username = '';
+      requestUrl.password = '';
+      const upstream = await fetchImpl(requestUrl.href, {
        method: 'POST',
+        redirect: 'error',
        headers: {
          'Content-Type': 'application/x-www-form-urlencoded',
          'User-Agent': OVERPASS_USER_AGENT,
+          ...(authorization ? { Authorization: authorization } : {}),
        },
        body,
        signal: controller.signal,
      });
-
      const responseBody = await readBody(upstream, maxResponseBytes);
-      const contentType =
-        upstream.headers.get('content-type') || 'application/json';
-      const status = upstream.status;
      const rateLimited =
-        status === 429 || overpassLooksRateLimited(responseBody);
-      const runtimeError = overpassLooksRuntimeError(responseBody);
-      const payload = {
-        status,
-        body: responseBody,
-        contentType,
-        endpoint,
-        rateLimited,
-        runtimeError,
-      };
-
-      if (rateLimited) {
-        lastRateLimitPayload = payload;
-        continue;
-      }
-      // A 200 body carrying a runtime error / timeout is a transient upstream
-      // failure — skip to the next mirror rather than returning or caching it.
-      if (runtimeError) {
-        lastError = new Error(`Overpass runtime error (${endpoint})`);
-        continue;
-      }
-      // Anything but 2xx is this mirror declining, not an answer. Only 5xx used
-      // to rotate, so a 4xx ended the fan-out and was returned — and cached —
-      // as data: a mirror refusing this client answers 406 while the others
-      // answer 200 to the very same request, so every Overpass-backed layer
-      // failed on an error page with healthy mirrors untried. The first
-      // refusal is kept so a genuinely bad query still reports what upstream
-      // said, but only after every mirror has had the chance to answer it.
-      if (status < 200 || status >= 300) {
-        if (!lastRefusalPayload) lastRefusalPayload = payload;
-        lastError = new Error(
-          `Overpass upstream returned ${status} (${endpoint})`,
+        upstream.status === 429 ||
+        upstream.status === 406 ||
+        overpassLooksRateLimited(responseBody);
+      if (
+        rateLimited ||
+        upstream.status < 200 ||
+        upstream.status >= 300 ||
+        overpassLooksRuntimeError(responseBody)
+      ) {
+        const failures = (previous?.failures || 0) + 1;
+        const delay = retryDelay(
+          upstream.headers.get('retry-after'),
+          failures - 1,
+          now(),
        );
+        const status = rateLimited
+          ? upstream.status === 406
+            ? 406
+            : 429
+          : 502;
+        cooldowns.set(endpoint, { until: now() + delay, failures, status });
+        while (cooldowns.size > 64)
+          cooldowns.delete(cooldowns.keys().next().value);
+        failure = refusal(status, delay);
        continue;
      }
-
-      // Success: decimate giant boundary geometry before it reaches the cache,
-      // the disk, or the client (what makes the 32 MB read cap safe to hold).
-      payload.body = simplify(payload.body);
-      return payload;
-    } catch (error) {
-      lastError = error;
+      const parsed = JSON.parse(responseBody);
+      if (!Array.isArray(parsed?.elements) || parsed.remark)
+        throw new Error('Malformed Overpass response');
+      cooldowns.delete(endpoint);
+      return {
+        status: upstream.status,
+        body: simplify(responseBody),
+        contentType: 'application/json',
+        // Never retain a secret-bearing endpoint in cache or response metadata.
+        endpoint: 'configured',
+        rateLimited: false,
+      };
+    } catch {
+      const failures = (previous?.failures || 0) + 1;
+      const delay = retryDelay(null, failures - 1, now());
+      cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
+      while (cooldowns.size > 64)
+        cooldowns.delete(cooldowns.keys().next().value);
+      failure = refusal(502, delay);
    } finally {
      clearTimeout(timeoutId);
    }
  }
-
-  if (lastRateLimitPayload) return lastRateLimitPayload;
-  if (lastRefusalPayload) return lastRefusalPayload;
-  throw lastError || new Error('All Overpass upstreams failed');
+  return failure;
 }

 export { overpassPayloadIsData, fetchOverpassPayload };
@@ -1,7 +1,7 @@
 import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
 import { coalesceProxyRequest } from '../common/http.js';
 import { fetchRegionalJson } from './http.js';
-import { normalizeRegionalPlace } from '../../../src/data/regionalModel.js';
+import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
 import {
  nominatimToGeocodeResult,
  nominatimViewboxFromBounds,
@@ -51,8 +51,8 @@ const NOMINATIM_SEARCH_MAX_QUERY = 200;
 // The pacer is deliberately module state while everything else here is
 // per-instance. One request per second is a budget for the whole application,
 // not for each provider object: two instances each pacing themselves would
-// send two requests a second between them. Reverse lookups and searches
-// therefore queue together.
+// send two requests a second between them. Only last-resort forward searches
+// use this queue.
 let _nominatimQueue = Promise.resolve();

 let _nominatimLastRequestAt = 0;
@@ -77,9 +77,7 @@ function abandonedError() {
 * Run one piece of upstream work, never closer than the policy spacing to the
 * last one.
 *
- * `bounded` applies the queue-depth and staleness limits. The cockpit's reverse
- * lookups are a slow background trickle and stay unbounded; the search route,
- * which a person can fire as fast as they can type, does not.
+ * `bounded` applies queue-depth and staleness limits to interactive forward searches.
 *
 * @param {() => Promise<unknown>} work
 * @param {{bounded?: boolean, signal?: AbortSignal}} [options]
@@ -111,30 +109,9 @@ function enqueueNominatim(work, { bounded = false, signal } = {}) {
  return task;
 }

-/** Construct the serialized Nominatim reverse adapter with a trusted endpoint. */
-export function createRegionalPlaceProvider({
-  endpoint = 'https://nominatim.openstreetmap.org/reverse',
-  requestJson = fetchRegionalJson,
-} = {}) {
-  function fetchRegionalPlace(point) {
-    return enqueueNominatim(async () => {
-      const params = new URLSearchParams({
-        format: 'jsonv2',
-        lat: point.latitude.toFixed(5),
-        lon: point.longitude.toFixed(5),
-        zoom: '10',
-        addressdetails: '1',
-        'accept-language': 'en',
-      });
-      const payload = await requestJson(`${endpoint}?${params}`, {
-        headers: NOMINATIM_HEADERS,
-        redirect: 'error',
-      });
-      return normalizeRegionalPlace(payload);
-    });
-  }
-
-  return fetchRegionalPlace;
+/** Construct the offline regional context provider from bundled Natural Earth polygons. */
+export function createRegionalPlaceProvider() {
+  return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
 }

 export const fetchRegionalPlace = createRegionalPlaceProvider();