Merge upstream/main into feature/catalonia-cctv-cameras

# Conflicts:
#	docs/CURRENT-STATE.md

server/providers/military-installations.js

@@ -5,7 +5,11 @@ import {
   MILITARY_INSTALLATION_DISK_TTL_MS,
   MILITARY_INSTALLATION_STALE_MS,
 } from './military-installations/constants.js';
-import { fetchOverpassPayload } from './overpass/transport.js';
+import {
+  fetchOverpassPayload,
+  overpassNotConfigured,
+} from './overpass/transport.js';
+import { resolveOverpassUpstreams } from './overpass/constants.js';
 import {
   _militaryInstallationCache,
   trimMilitaryInstallationCache,
@@ -45,6 +49,7 @@ function militaryInstallationsProxy() {
       throw Object.assign(
         new Error('Mapped installation upstream unavailable'),
         {
+          retryAfterMs: upstream.retryAfterMs,
           installationReason: upstream.rateLimited
             ? 'rate_limited'
             : upstream.status === 504
@@ -83,14 +88,6 @@ function militaryInstallationsProxy() {
         res.end(JSON.stringify({ error: 'Method Not Allowed' }));
         return;
       }
-      if (!_militaryInstallationsRateLimiter(clientKey(req))) {
-        res.writeHead(429, {
-          'Content-Type': 'application/json',
-          'Retry-After': '5',
-        });
-        res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
-        return;
-      }
       const url = new URL(req.url, 'http://localhost');
       const requested = validMilitaryInstallationBox(url.searchParams);
       if (!requested) {
@@ -117,6 +114,36 @@ function militaryInstallationsProxy() {
         : militaryInstallationCacheKey(box);
       const now = Date.now();
       const cached = _militaryInstallationCache.get(key);
+      if (!resolveOverpassUpstreams().length) {
+        const stale =
+          cached || (await readMilitaryInstallationDisk(key, Infinity));
+        const disabled = overpassNotConfigured();
+        res.writeHead(200, {
+          'Content-Type': 'application/json',
+          'Cache-Control': 'no-store',
+          'X-Military-Installations': stale ? 'STALE' : 'DISABLED',
+        });
+        res.end(
+          stale
+            ? JSON.stringify({
+                ...stale.payload,
+                status: 'stale',
+                retrievedAt:
+                  stale.payload.retrievedAt ||
+                  new Date(stale.cachedAt).toISOString(),
+              })
+            : disabled.body,
+        );
+        return;
+      }
+      if (!_militaryInstallationsRateLimiter(clientKey(req))) {
+        res.writeHead(429, {
+          'Content-Type': 'application/json',
+          'Retry-After': '5',
+        });
+        res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
+        return;
+      }
       const preflight = await resolveMilitaryInstallationTier({
         cacheKey: key,
         memoryCache: _militaryInstallationCache,
@@ -176,6 +203,9 @@ function militaryInstallationsProxy() {
           return;
         }
         res.writeHead(503, {
+          ...(error?.retryAfterMs
+            ? { 'Retry-After': String(Math.ceil(error.retryAfterMs / 1000)) }
+            : {}),
           'Content-Type': 'application/json',
           'Cache-Control': 'no-store',
         });
@@ -183,6 +213,7 @@ function militaryInstallationsProxy() {
           JSON.stringify({
             error: 'Mapped installation context is temporarily unavailable',
             reason: militaryInstallationFailureReason(error),
+            retryAfterMs: error?.retryAfterMs,
           }),
         );
       }

server/providers/overpass.js

@@ -3,6 +3,7 @@ import { readRequestBodyCapped } from './common/request.js';
 import {
   OVERPASS_MAX_BODY_BYTES,
   OVERPASS_MAX_CONCURRENT,
+  resolveOverpassUpstreams,
 } from './overpass/constants.js';
 import { sanitizeOverpassBody } from './overpass/query.js';
 import {
@@ -17,6 +18,7 @@ import {
 import {
   overpassPayloadIsData,
   fetchOverpassPayload,
+  overpassNotConfigured,
 } from './overpass/transport.js';
 import { installRouteMiddleware } from './places/routes.js';
 
@@ -41,9 +43,16 @@ const _overpassRateLimiter = makeRateLimiter({
 function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
   res.writeHead(payload.status, {
     'Content-Type': payload.contentType || 'application/json',
-    'Cache-Control': 'public, max-age=15',
+    'Cache-Control': overpassPayloadIsData(payload)
+      ? 'public, max-age=15'
+      : 'no-store',
     'X-Overpass-Cache': cacheStatus,
-    'X-Overpass-Upstream': payload.endpoint || 'unknown',
+    ...(Number.isFinite(payload.cachedAt)
+      ? { 'X-Overpass-Cached-At': new Date(payload.cachedAt).toISOString() }
+      : {}),
+    ...(payload.retryAfterMs
+      ? { 'Retry-After': String(Math.ceil(payload.retryAfterMs / 1000)) }
+      : {}),
   });
   res.end(payload.body || '');
 }
@@ -52,7 +61,7 @@ function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
  * Vite plugin: Overpass API proxy with response caching and request coalescing.
  *
  * Accepts POST requests at /api/overpass, normalizes the query body for
- * cache keying, and fans out to multiple Overpass mirrors with per-upstream
+ * cache keying, and uses only operator-configured endpoints with per-upstream
  * timeout and rate-limit detection. Successful responses are cached for
  * OVERPASS_CACHE_MS. Concurrent identical queries share a single upstream
  * request via the in-flight map.
@@ -66,6 +75,19 @@ function overpassProxy({ routing = {} } = {}) {
       // (a body-read failure would otherwise hit an out-of-scope reference).
       let cacheKey = null;
       try {
+        // Capability probe: lets clients skip queries when nothing is configured.
+        if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
+          res.writeHead(200, {
+            'Content-Type': 'application/json',
+            'Cache-Control': 'no-store',
+          });
+          res.end(
+            JSON.stringify({
+              configured: resolveOverpassUpstreams().length > 0,
+            }),
+          );
+          return;
+        }
         if (req.method !== 'POST') {
           res.writeHead(405, { 'Content-Type': 'application/json' });
           res.end(JSON.stringify({ error: 'Method Not Allowed' }));
@@ -104,12 +126,21 @@ function overpassProxy({ routing = {} } = {}) {
 
         // Normalize whitespace so semantically identical Overpass QL queries share cache entries
         cacheKey = safeBody.replace(/\s+/g, ' ').trim();
+        if (!resolveOverpassUpstreams().length) {
+          const cached = await readStaleOverpass(cacheKey);
+          sendOverpassResponse(
+            res,
+            cached || overpassNotConfigured(),
+            cached ? 'STALE' : 'DISABLED',
+          );
+          return;
+        }
         const preflight = await resolveOverpassPreflight({
           cacheKey,
           memoryCache: _overpassCache,
           inFlight: _overpassInFlight,
-          // Fresh-enough disk entries survive restarts and skip the public
-          // mirrors; boundary-class queries keep their month-long TTL.
+          // Fresh-enough disk entries survive restarts and skip upstream
+          // requests; boundary-class queries keep their month-long TTL.
           readDisk: () =>
             readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
           allowUpstream: () => _overpassRateLimiter(clientKey(req)),
@@ -194,7 +225,7 @@ function overpassProxy({ routing = {} } = {}) {
           sendOverpassResponse(res, stale, 'STALE');
           return;
         }
-        console.error('[Overpass Proxy]', e.message);
+        console.error('[Overpass Proxy] request failed');
         res.writeHead(502, { 'Content-Type': 'application/json' });
         res.end(JSON.stringify({ error: 'Overpass proxy error' }));
       }

server/providers/overpass/constants.js

@@ -3,30 +3,39 @@ import path from 'node:path';
 // ---------------------------------------------------------------------------
 // Overpass API proxy constants and cache state
 // ---------------------------------------------------------------------------
-/**
- * User-Agent sent to every Overpass mirror.
- *
- * The OSM API usage policy asks for a "Valid User-Agent identifying application
- * and version"; a generic proxy label is not one. A mirror is free to refuse a
- * client it cannot identify, and `src/overpassProxy.test.mjs` pins what that
- * costs: a refusal is never data, so the query falls through to whatever
- * mirrors are left. Keep this honest and stable — if it is ever refused, the
- * answer is less query volume, not a new name.
- */
+/** Stable application identity for operator-configured Overpass instances. */
 const OVERPASS_USER_AGENT =
   'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)';
 
-/** Ordered list of Overpass API mirrors; tried sequentially on failure/rate-limit. */
-const OVERPASS_UPSTREAMS = [
-  'https://overpass-api.de/api/interpreter',
-  'https://overpass.kumi.systems/api/interpreter',
-  'https://lz4.overpass-api.de/api/interpreter',
-  // Community full-planet instance (privateforge nonprofit) — added 2026-07-30
-  // when all three mirrors above refused this IP (likely a dev-traffic rate
-  // ban; refused connections fail in ms, so healthy mirrors above still win).
-  // Verified: planet coverage (Texas query), CORS *, ~5-20 s cold latency.
-  'https://overpass.private.coffee/api/interpreter',
-];
+/** Parse only operator-supplied HTTP(S) endpoints; private instances are allowed. */
+function parseOverpassUpstreams(raw) {
+  const endpoints = [];
+  for (const token of String(raw || '').split(',')) {
+    try {
+      const url = new URL(token.trim());
+      if (
+        !['http:', 'https:'].includes(url.protocol) ||
+        !url.hostname ||
+        url.hash
+      )
+        continue;
+      if (!endpoints.includes(url.href)) endpoints.push(url.href);
+    } catch {
+      // Invalid configuration never becomes an upstream or appears in logs.
+    }
+  }
+  return endpoints.slice(0, 8);
+}
+
+let upstreamMemo = { raw: null, endpoints: [] };
+
+/** Resolve after environment loading. Public Overpass instances are not used by default. */
+function resolveOverpassUpstreams() {
+  const raw = process.env.OVERPASS_UPSTREAMS || '';
+  if (upstreamMemo.raw !== raw)
+    upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) };
+  return [...upstreamMemo.endpoints];
+}
 
 /**
  * TTL for FRESH cached Overpass responses (ms). Road geometry is static for
@@ -142,7 +151,8 @@ export {
   OVERPASS_SIMPLIFY_MIN_POINTS,
   OVERPASS_SIMPLIFY_TOLERANCE_DEG,
   OVERPASS_MAX_RESPONSE_BYTES,
-  OVERPASS_UPSTREAMS,
+  parseOverpassUpstreams,
+  resolveOverpassUpstreams,
   OVERPASS_USER_AGENT,
   OVERPASS_TIMEOUT_MS,
 };

server/providers/overpass/transport.js

@@ -1,6 +1,6 @@
 import {
   OVERPASS_MAX_RESPONSE_BYTES,
-  OVERPASS_UPSTREAMS,
+  resolveOverpassUpstreams,
   OVERPASS_USER_AGENT,
   OVERPASS_TIMEOUT_MS,
 } from './constants.js';
@@ -60,99 +60,147 @@ function overpassPayloadIsData(payload) {
   );
 }
 
+const cooldowns = new Map();
+
+/** A stable, non-retryable capability response shared by every Overpass route. */
+export function overpassNotConfigured() {
+  return {
+    status: 503,
+    contentType: 'application/json',
+    body: JSON.stringify({
+      error: 'Detailed OpenStreetMap queries are not configured',
+      code: 'OVERPASS_NOT_CONFIGURED',
+      retryable: false,
+    }),
+  };
+}
+
+/** Honor Retry-After dates/seconds; absent values use bounded exponential backoff. */
+function retryDelay(value, failures, now) {
+  const seconds = Number(value);
+  const explicit =
+    value && Number.isFinite(seconds)
+      ? seconds * 1000
+      : Date.parse(value) - now;
+  return Number.isFinite(explicit)
+    ? Math.max(1000, explicit)
+    : Math.min(300_000, 30_000 * 2 ** Math.min(failures, 4));
+}
+
+function refusal(status, retryAfterMs) {
+  return {
+    status,
+    contentType: 'application/json',
+    rateLimited: status === 429 || status === 406,
+    retryAfterMs,
+    body: JSON.stringify({
+      error: 'Configured Overpass upstream unavailable',
+      code: 'OVERPASS_UNAVAILABLE',
+      retryable: true,
+      retryAfterMs,
+    }),
+  };
+}
+
 /**
- * Try each mirror once, retaining response-size and per-mirror timeout caps.
- * Refusals and body-level failures rotate; total failure returns the last
- * rate-limit payload, otherwise the first refusal, or throws a network error.
- * @param {string} body URL-encoded Overpass QL query body.
- * @param {number} [maxResponseBytes] Endpoint-specific response cap.
- * @param {object} [options] Server-only endpoint and I/O overrides for tests.
- * @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
+ * Query only the configured chain with capped reads, timeouts and per-endpoint cooldowns.
+ * Explicit endpoint/I/O overrides are server-only test seams. Empty data is valid.
  */
 async function fetchOverpassPayload(
   body,
   maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES,
   {
-    endpoints = OVERPASS_UPSTREAMS,
+    endpoints = resolveOverpassUpstreams(),
     fetchImpl = fetch,
     readBody = readResponseTextCapped,
     simplify = simplifyOverpassPayloadBody,
+    now = Date.now,
   } = {},
 ) {
-  let lastError = null;
-  let lastRateLimitPayload = null;
-  let lastRefusalPayload = null;
-
+  if (!endpoints.length) return overpassNotConfigured();
+  let failure = refusal(502, 30_000);
   for (const endpoint of endpoints) {
+    const previous = cooldowns.get(endpoint);
+    if (previous?.until > now()) {
+      failure = refusal(previous.status, previous.until - now());
+      continue;
+    }
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
-
     try {
-      const upstream = await fetchImpl(endpoint, {
+      const requestUrl = new URL(endpoint);
+      const authorization =
+        requestUrl.username || requestUrl.password
+          ? 'Basic ' +
+            Buffer.from(
+              `${decodeURIComponent(requestUrl.username)}:${decodeURIComponent(requestUrl.password)}`,
+            ).toString('base64')
+          : null;
+      requestUrl.username = '';
+      requestUrl.password = '';
+      const upstream = await fetchImpl(requestUrl.href, {
         method: 'POST',
+        redirect: 'error',
         headers: {
           'Content-Type': 'application/x-www-form-urlencoded',
           'User-Agent': OVERPASS_USER_AGENT,
+          ...(authorization ? { Authorization: authorization } : {}),
         },
         body,
         signal: controller.signal,
       });
-
       const responseBody = await readBody(upstream, maxResponseBytes);
-      const contentType =
-        upstream.headers.get('content-type') || 'application/json';
-      const status = upstream.status;
       const rateLimited =
-        status === 429 || overpassLooksRateLimited(responseBody);
-      const runtimeError = overpassLooksRuntimeError(responseBody);
-      const payload = {
-        status,
-        body: responseBody,
-        contentType,
-        endpoint,
-        rateLimited,
-        runtimeError,
-      };
-
-      if (rateLimited) {
-        lastRateLimitPayload = payload;
-        continue;
-      }
-      // A 200 body carrying a runtime error / timeout is a transient upstream
-      // failure — skip to the next mirror rather than returning or caching it.
-      if (runtimeError) {
-        lastError = new Error(`Overpass runtime error (${endpoint})`);
-        continue;
-      }
-      // Anything but 2xx is this mirror declining, not an answer. Only 5xx used
-      // to rotate, so a 4xx ended the fan-out and was returned — and cached —
-      // as data: a mirror refusing this client answers 406 while the others
-      // answer 200 to the very same request, so every Overpass-backed layer
-      // failed on an error page with healthy mirrors untried. The first
-      // refusal is kept so a genuinely bad query still reports what upstream
-      // said, but only after every mirror has had the chance to answer it.
-      if (status < 200 || status >= 300) {
-        if (!lastRefusalPayload) lastRefusalPayload = payload;
-        lastError = new Error(
-          `Overpass upstream returned ${status} (${endpoint})`,
+        upstream.status === 429 ||
+        upstream.status === 406 ||
+        overpassLooksRateLimited(responseBody);
+      if (
+        rateLimited ||
+        upstream.status < 200 ||
+        upstream.status >= 300 ||
+        overpassLooksRuntimeError(responseBody)
+      ) {
+        const failures = (previous?.failures || 0) + 1;
+        const delay = retryDelay(
+          upstream.headers.get('retry-after'),
+          failures - 1,
+          now(),
         );
+        const status = rateLimited
+          ? upstream.status === 406
+            ? 406
+            : 429
+          : 502;
+        cooldowns.set(endpoint, { until: now() + delay, failures, status });
+        while (cooldowns.size > 64)
+          cooldowns.delete(cooldowns.keys().next().value);
+        failure = refusal(status, delay);
         continue;
       }
-
-      // Success: decimate giant boundary geometry before it reaches the cache,
-      // the disk, or the client (what makes the 32 MB read cap safe to hold).
-      payload.body = simplify(payload.body);
-      return payload;
-    } catch (error) {
-      lastError = error;
+      const parsed = JSON.parse(responseBody);
+      if (!Array.isArray(parsed?.elements) || parsed.remark)
+        throw new Error('Malformed Overpass response');
+      cooldowns.delete(endpoint);
+      return {
+        status: upstream.status,
+        body: simplify(responseBody),
+        contentType: 'application/json',
+        // Never retain a secret-bearing endpoint in cache or response metadata.
+        endpoint: 'configured',
+        rateLimited: false,
+      };
+    } catch {
+      const failures = (previous?.failures || 0) + 1;
+      const delay = retryDelay(null, failures - 1, now());
+      cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
+      while (cooldowns.size > 64)
+        cooldowns.delete(cooldowns.keys().next().value);
+      failure = refusal(502, delay);
     } finally {
       clearTimeout(timeoutId);
     }
   }
-
-  if (lastRateLimitPayload) return lastRateLimitPayload;
-  if (lastRefusalPayload) return lastRefusalPayload;
-  throw lastError || new Error('All Overpass upstreams failed');
+  return failure;
 }
 
 export { overpassPayloadIsData, fetchOverpassPayload };

server/providers/regional/place.js

@@ -1,7 +1,7 @@
 import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
 import { coalesceProxyRequest } from '../common/http.js';
 import { fetchRegionalJson } from './http.js';
-import { normalizeRegionalPlace } from '../../../src/data/regionalModel.js';
+import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
 import {
   nominatimToGeocodeResult,
   nominatimViewboxFromBounds,
@@ -51,8 +51,8 @@ const NOMINATIM_SEARCH_MAX_QUERY = 200;
 // The pacer is deliberately module state while everything else here is
 // per-instance. One request per second is a budget for the whole application,
 // not for each provider object: two instances each pacing themselves would
-// send two requests a second between them. Reverse lookups and searches
-// therefore queue together.
+// send two requests a second between them. Only last-resort forward searches
+// use this queue.
 let _nominatimQueue = Promise.resolve();
 
 let _nominatimLastRequestAt = 0;
@@ -77,9 +77,7 @@ function abandonedError() {
  * Run one piece of upstream work, never closer than the policy spacing to the
  * last one.
  *
- * `bounded` applies the queue-depth and staleness limits. The cockpit's reverse
- * lookups are a slow background trickle and stay unbounded; the search route,
- * which a person can fire as fast as they can type, does not.
+ * `bounded` applies queue-depth and staleness limits to interactive forward searches.
  *
  * @param {() => Promise<unknown>} work
  * @param {{bounded?: boolean, signal?: AbortSignal}} [options]
@@ -111,30 +109,9 @@ function enqueueNominatim(work, { bounded = false, signal } = {}) {
   return task;
 }
 
-/** Construct the serialized Nominatim reverse adapter with a trusted endpoint. */
-export function createRegionalPlaceProvider({
-  endpoint = 'https://nominatim.openstreetmap.org/reverse',
-  requestJson = fetchRegionalJson,
-} = {}) {
-  function fetchRegionalPlace(point) {
-    return enqueueNominatim(async () => {
-      const params = new URLSearchParams({
-        format: 'jsonv2',
-        lat: point.latitude.toFixed(5),
-        lon: point.longitude.toFixed(5),
-        zoom: '10',
-        addressdetails: '1',
-        'accept-language': 'en',
-      });
-      const payload = await requestJson(`${endpoint}?${params}`, {
-        headers: NOMINATIM_HEADERS,
-        redirect: 'error',
-      });
-      return normalizeRegionalPlace(payload);
-    });
-  }
-
-  return fetchRegionalPlace;
+/** Construct the offline regional context provider from bundled Natural Earth polygons. */
+export function createRegionalPlaceProvider() {
+  return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
 }
 
 export const fetchRegionalPlace = createRegionalPlaceProvider();