benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-04 10:24:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: separate Overpass and installation services

Browse Source
This commit is contained in:
Sameh Khamis committed 2026-09-12 12:59:39 -07:00
1 parent 781613fbb8
commit 213edd80ce
14 files changed
+1455 -841
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

package.json
+7 -1
View File
@@ -2,7 +2,7 @@
"name": "gods-eye-view",
"private": true,
"version": "0.1.1",
"description": "A real-time intelligence console for planet Earth \u2014 photorealistic 3D globe, live aircraft/ships/satellites/earthquakes/CCTV, and hands-free voice control. Runs in a browser.",
"description": "A real-time intelligence console for planet Earth — photorealistic 3D globe, live aircraft/ships/satellites/earthquakes/CCTV, and hands-free voice control. Runs in a browser.",
"type": "module",
"license": "MIT",
"author": "Bilawal Sidhu",
@@ -102,6 +102,12 @@
},
"./server/providers/radio": {
"node": "./server/providers/radio.js"
},
"./server/providers/overpass": {
"node": "./server/providers/overpass.js"
},
"./server/providers/military-installations": {
"node": "./server/providers/military-installations.js"
}
}
}
scripts/package-boundaries.json
+145 -34
View File
@@ -10,27 +10,47 @@
"src/data/localGeojsonCore.js",
"src/data/localGeojsonLod.js"
],
"external": ["cesium"]
"external": [
"cesium"
]
},
"application": {
"exports": ["./application"],
"modules": ["src/app/application.js"],
"exports": [
"./application"
],
"modules": [
"src/app/application.js"
],
"external": []
},
"viewer": {
"exports": ["./application/viewer"],
"modules": ["src/app/viewer.js"],
"external": ["cesium"]
"exports": [
"./application/viewer"
],
"modules": [
"src/app/viewer.js"
],
"external": [
"cesium"
]
},
"vite-build": {
"runtime": "node",
"exports": ["./build/vite"],
"modules": ["build/vite.js"],
"external": ["vite-plugin-cesium"]
"exports": [
"./build/vite"
],
"modules": [
"build/vite.js"
],
"external": [
"vite-plugin-cesium"
]
},
"live-providers": {
"runtime": "node",
"exports": ["./server/providers/live"],
"exports": [
"./server/providers/live"
],
"modules": [
"server/providers/live.js",
"server/providers/common/http.js",
@@ -45,16 +65,24 @@
"src/data/aisStreamAdapter.js",
"src/data/aisWatchdog.js"
],
"external": ["ws"]
"external": [
"ws"
]
},
"adsb-lol-source": {
"exports": ["./sources/adsb-lol"],
"modules": ["src/data/adsbLolFallback.js"],
"exports": [
"./sources/adsb-lol"
],
"modules": [
"src/data/adsbLolFallback.js"
],
"external": []
},
"place-providers": {
"runtime": "node",
"exports": ["./server/providers/places"],
"exports": [
"./server/providers/places"
],
"modules": [
"server/providers/places.js",
"server/providers/places/google.js",
@@ -69,13 +97,19 @@
"external": []
},
"place-payloads": {
"exports": ["./sources/places"],
"modules": ["src/data/placeProviderPayloads.js"],
"exports": [
"./sources/places"
],
"modules": [
"src/data/placeProviderPayloads.js"
],
"external": []
},
"space-providers": {
"runtime": "node",
"exports": ["./server/providers/space"],
"exports": [
"./server/providers/space"
],
"modules": [
"server/providers/space.js",
"server/providers/space/celestrak.js",
@@ -86,13 +120,19 @@
"external": []
},
"space-requests": {
"exports": ["./sources/space"],
"modules": ["src/data/spaceProviderRequests.js"],
"exports": [
"./sources/space"
],
"modules": [
"src/data/spaceProviderRequests.js"
],
"external": []
},
"terrain-provider": {
"runtime": "node",
"exports": ["./server/providers/terrain"],
"exports": [
"./server/providers/terrain"
],
"modules": [
"server/providers/terrain.js",
"src/data/terrainHeightsProxy.js"
@@ -100,41 +140,70 @@
"external": []
},
"terrain-source": {
"exports": ["./sources/terrain"],
"modules": ["src/data/terrainHeightsProxy.js"],
"exports": [
"./sources/terrain"
],
"modules": [
"src/data/terrainHeightsProxy.js"
],
"external": []
},
"traffic-provider": {
"runtime": "node",
"exports": ["./server/providers/traffic"],
"modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"],
"exports": [
"./server/providers/traffic"
],
"modules": [
"server/providers/traffic.js",
"src/data/tomtomTiles.js"
],
"external": []
},
"traffic-source": {
"exports": ["./sources/traffic"],
"modules": ["src/data/tomtomTiles.js"],
"exports": [
"./sources/traffic"
],
"modules": [
"src/data/tomtomTiles.js"
],
"external": []
},
"firms-provider": {
"runtime": "node",
"exports": ["./server/providers/firms"],
"modules": ["server/providers/firms.js", "src/data/firmsCsv.js"],
"exports": [
"./server/providers/firms"
],
"modules": [
"server/providers/firms.js",
"src/data/firmsCsv.js"
],
"external": []
},
"gbfs-provider": {
"runtime": "node",
"exports": ["./server/providers/gbfs"],
"modules": ["server/providers/gbfs.js", "src/data/gbfsSource.js"],
"exports": [
"./server/providers/gbfs"
],
"modules": [
"server/providers/gbfs.js",
"src/data/gbfsSource.js"
],
"external": []
},
"gbfs-source": {
"exports": ["./sources/gbfs"],
"modules": ["src/data/gbfsSource.js"],
"exports": [
"./sources/gbfs"
],
"modules": [
"src/data/gbfsSource.js"
],
"external": []
},
"cctv-provider": {
"runtime": "node",
"exports": ["./server/providers/cctv"],
"exports": [
"./server/providers/cctv"
],
"modules": [
"server/providers/cctv.js",
"server/providers/cctv/catalog.js",
@@ -151,7 +220,9 @@
},
"radio-provider": {
"runtime": "node",
"exports": ["./server/providers/radio"],
"exports": [
"./server/providers/radio"
],
"modules": [
"server/providers/radio.js",
"server/providers/radio/catalog.js",
@@ -162,5 +233,45 @@
"server/providers/common/http.js"
],
"external": []
},
"overpass-provider": {
"runtime": "node",
"exports": [
"./server/providers/overpass"
],
"modules": [
"server/providers/common/geo.js",
"server/providers/common/http.js",
"server/providers/common/rate-limit.js",
"server/providers/common/request.js",
"server/providers/overpass.js",
"server/providers/overpass/cache.js",
"server/providers/overpass/constants.js",
"server/providers/overpass/geometry.js",
"server/providers/overpass/query.js",
"server/providers/overpass/transport.js",
"server/providers/places/routes.js",
"src/data/placeProviderPayloads.js"
],
"external": []
},
"military-installations-provider": {
"runtime": "node",
"exports": [
"./server/providers/military-installations"
],
"modules": [
"server/providers/common/http.js",
"server/providers/common/query.js",
"server/providers/common/rate-limit.js",
"server/providers/military-installations.js",
"server/providers/military-installations/cache.js",
"server/providers/military-installations/constants.js",
"server/providers/military-installations/query.js",
"server/providers/overpass/constants.js",
"server/providers/overpass/geometry.js",
"server/providers/overpass/transport.js"
],
"external": []
}
}
server/providers/common/request.js
+36
View File
@@ -0,0 +1,36 @@
/** Read a request body with a hard byte cap; throws { code:'BODY_TOO_LARGE' } past the cap. */
async function readRequestBodyCapped(req, maxBytes) {
const chunks = [];
let total = 0;
for await (const chunk of req) {
total += chunk.length;
if (total > maxBytes) {
const err = new Error('Request body too large');
err.code = 'BODY_TOO_LARGE';
throw err;
}
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
function readRequestBody(req, maxBytes = 1024 * 1024) {
return new Promise((resolve, reject) => {
let total = 0;
const chunks = [];
req.on('data', (chunk) => {
total += chunk.length;
if (total > maxBytes) {
reject(new Error(`Request body exceeds ${maxBytes} bytes`));
req.destroy();
return;
}
chunks.push(chunk);
});
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
req.on('error', reject);
});
}
export { readRequestBodyCapped, readRequestBody };
server/providers/local.js
+81 -806
View File
File diff suppressed because it is too large. Load diff
server/providers/military-installations.js
+152
View File
@@ -0,0 +1,152 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { MILITARY_INSTALLATION_ELEMENT_CAP, MILITARY_INSTALLATION_MAX_RESPONSE_BYTES, MILITARY_INSTALLATION_DISK_TTL_MS, MILITARY_INSTALLATION_STALE_MS } from './military-installations/constants.js';
import { fetchOverpassPayload } from './overpass/transport.js';
import { _militaryInstallationCache, trimMilitaryInstallationCache, writeMilitaryInstallationDisk, resolveMilitaryInstallationTier, readMilitaryInstallationDisk } from './military-installations/cache.js';
import { validMilitaryInstallationBox, quantizeMilitaryInstallationBox, militaryInstallationCacheKey, militaryInstallationFailureReason } from './military-installations/query.js';
import { coalesceProxyRequest } from './common/http.js';
const _militaryInstallationsRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 90, globalMax: 300 });
const _militaryInstallationInFlight = new Map();
function militaryInstallationsProxy() {
async function refresh(box, key) {
const bbox = `${box.south},${box.west},${box.north},${box.east}`;
const ql = `[out:json][timeout:20];(nwr["military"~"^(airfield|naval_base|range|barracks|base)$"](${bbox});nwr["landuse"="military"](${bbox}););out center tags geom ${MILITARY_INSTALLATION_ELEMENT_CAP};`;
const upstream = await fetchOverpassPayload(
`data=${encodeURIComponent(ql)}`,
MILITARY_INSTALLATION_MAX_RESPONSE_BYTES,
);
if (upstream.status >= 400 || upstream.rateLimited || upstream.runtimeError) {
throw Object.assign(new Error('Mapped installation upstream unavailable'), {
installationReason: upstream.rateLimited ? 'rate_limited'
: upstream.status === 504 ? 'timeout'
: upstream.runtimeError ? 'query_failed' : 'unavailable',
});
}
const parsed = JSON.parse(upstream.body);
const elements = Array.isArray(parsed?.elements)
? parsed.elements.slice(0, MILITARY_INSTALLATION_ELEMENT_CAP)
: [];
const payload = {
elements,
// Honest truncation flag — the client re-asks for its exact viewport so
// off-view features can never starve in-view ones. The cap travels with
// the payload so the client never has to hard-code it.
saturated: elements.length >= MILITARY_INSTALLATION_ELEMENT_CAP,
elementCap: MILITARY_INSTALLATION_ELEMENT_CAP,
retrievedAt: new Date().toISOString(),
status: 'ready',
};
const entry = { payload, cachedAt: Date.now() };
_militaryInstallationCache.set(key, entry);
trimMilitaryInstallationCache();
writeMilitaryInstallationDisk(key, entry);
return payload;
}
function install(middlewares) {
middlewares.use('/api/military-installations', async (req, res) => {
if (req.method !== 'GET') {
res.writeHead(405, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
if (!_militaryInstallationsRateLimiter(clientKey(req))) {
res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '5' });
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
const url = new URL(req.url, 'http://localhost');
const requested = validMilitaryInstallationBox(url.searchParams);
if (!requested) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'A non-dateline bbox no larger than 10 degrees is required' }));
return;
}
// Query the SNAPPED box, not the raw viewport: neighbouring views then
// share one cache entry, and an outward snap always covers what was asked.
// `exact=1` opts out — the client sends it after a SATURATED snapped
// response, so a truncated tile can never starve the actual viewport. It
// is keyed separately so exact and snapped answers never collide.
const exact = url.searchParams.get('exact') === '1';
const box = exact ? requested : quantizeMilitaryInstallationBox(requested);
// Key at the precision the query actually uses (see militaryInstallationCacheKey).
const key = exact
? `exact:${militaryInstallationCacheKey(box, 5)}`
: militaryInstallationCacheKey(box);
const now = Date.now();
const cached = _militaryInstallationCache.get(key);
const preflight = await resolveMilitaryInstallationTier({
cacheKey: key,
memoryCache: _militaryInstallationCache,
inFlight: _militaryInstallationInFlight,
readDisk: () => readMilitaryInstallationDisk(key, MILITARY_INSTALLATION_DISK_TTL_MS),
now,
});
if (preflight.source !== 'UPSTREAM') {
if (preflight.source === 'DISK') {
_militaryInstallationCache.set(key, preflight.entry);
trimMilitaryInstallationCache();
}
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'public, max-age=60', 'X-Military-Installations': preflight.source });
res.end(JSON.stringify({ ...preflight.entry.payload, status: 'cached' }));
return;
}
const request = coalesceProxyRequest(
_militaryInstallationInFlight,
key,
() => refresh(box, key),
);
try {
const payload = await request.promise;
res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'public, max-age=60',
'X-Military-Installations': request.shared ? 'INFLIGHT' : 'MISS',
});
res.end(JSON.stringify(payload));
} catch (error) {
if (cached && now - cached.cachedAt <= MILITARY_INSTALLATION_STALE_MS) {
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', 'X-Military-Installations': 'STALE' });
res.end(JSON.stringify({ ...cached.payload, status: 'stale' }));
return;
}
// Overpass is down: last-good mapped context at ANY age beats an empty
// layer (the same serve-stale rule the Overpass proxy applies).
const stale = await readMilitaryInstallationDisk(key, Infinity);
if (stale) {
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', 'X-Military-Installations': 'STALE-DISK' });
res.end(JSON.stringify({ ...stale.payload, status: 'stale' }));
return;
}
res.writeHead(503, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
res.end(JSON.stringify({ error: 'Mapped installation context is temporarily unavailable', reason: militaryInstallationFailureReason(error) }));
}
});
}
return {
name: 'military-installations-proxy',
configureServer(server) {
install(server.middlewares);
},
configurePreviewServer(server) {
install(server.middlewares);
},
};
}
export { militaryInstallationsProxy };
export { MILITARY_INSTALLATION_ELEMENT_CAP } from './military-installations/constants.js';
export { quantizeMilitaryInstallationBox } from './military-installations/query.js';
export { militaryInstallationCacheKey } from './military-installations/query.js';
export { resolveMilitaryInstallationTier } from './military-installations/cache.js';
export { migrateMilitaryInstallationEntry } from './military-installations/cache.js';
export { militaryInstallationDiskFresh } from './military-installations/cache.js';
export { militaryInstallationDiskPath } from './military-installations/cache.js';
export { readMilitaryInstallationDisk } from './military-installations/cache.js';
export { writeMilitaryInstallationDisk } from './military-installations/cache.js';
export { validMilitaryInstallationBox } from './military-installations/query.js';
export { militaryInstallationFailureReason } from './military-installations/query.js';
server/providers/military-installations/cache.js
+134
View File
@@ -0,0 +1,134 @@
import { MILITARY_INSTALLATION_CACHE_MS, MILITARY_INSTALLATION_ELEMENT_CAP, MILITARY_INSTALLATION_DISK_TTL_MS, MILITARY_INSTALLATION_DISK_DIR, MILITARY_INSTALLATION_MAX_CACHE } from './constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
const _militaryInstallationCache = new Map();
/**
* Resolve the READ tiers for one installation request, in order: fresh memory,
* then disk. Returns UPSTREAM when neither can answer.
*
* Disk is skipped while a request for this key is already in flight — the
* caller joins that instead of paying a read. Exported so the tier ORDER is
* testable against a real temp directory without a Vite server, mirroring
* resolveOverpassPreflight.
*
* @param {object} options
* @param {string} options.cacheKey
* @param {Map<string, {payload: object, cachedAt: number}>} options.memoryCache
* @param {Map<string, Promise>} options.inFlight
* @param {() => Promise<?{payload: object, cachedAt: number}>} options.readDisk
* @param {number} [options.now]
* @param {number} [options.cacheMs]
* @returns {Promise<{source: 'HIT'|'DISK'|'UPSTREAM', entry: ?object}>}
*/
async function resolveMilitaryInstallationTier({
cacheKey,
memoryCache,
inFlight,
readDisk,
now = Date.now(),
cacheMs = MILITARY_INSTALLATION_CACHE_MS,
}) {
const cached = memoryCache.get(cacheKey);
if (cached && now - cached.cachedAt <= cacheMs) return { source: 'HIT', entry: cached };
if (inFlight.has(cacheKey)) return { source: 'UPSTREAM', entry: null };
const disk = await readDisk();
return disk ? { source: 'DISK', entry: disk } : { source: 'UPSTREAM', entry: null };
}
/**
* Bring a stored installation entry up to the current payload shape.
*
* Entries written before the saturation guard shipped carry no `saturated`
* field, and the disk TTL is 30 DAYS — so without this a cached, truncated
* 700-element snapped response would keep skipping the exact-viewport retry for
* a month, quietly starving in-view sites. Saturation is DERIVED from the
* element count rather than invalidating those entries, so warm caches survive
* the upgrade.
* @param {?{payload: object, cachedAt: number}} entry
* @returns {?{payload: object, cachedAt: number}}
*/
function migrateMilitaryInstallationEntry(entry) {
if (!entry?.payload || typeof entry.payload.saturated === 'boolean') return entry;
const elements = Array.isArray(entry.payload.elements) ? entry.payload.elements : [];
return {
...entry,
payload: {
...entry.payload,
saturated: elements.length >= MILITARY_INSTALLATION_ELEMENT_CAP,
},
};
}
/** Whether a stored installation entry is still inside its TTL. */
function militaryInstallationDiskFresh(
entry,
maxAgeMs = MILITARY_INSTALLATION_DISK_TTL_MS,
now = Date.now(),
) {
if (!entry || !Number.isFinite(entry.cachedAt) || !Array.isArray(entry.payload?.elements)) return false;
return now - entry.cachedAt <= maxAgeMs;
}
/** Cache key -> stable disk-cache file path. */
function militaryInstallationDiskPath(cacheKey, dir = MILITARY_INSTALLATION_DISK_DIR) {
return path.join(dir, `${createHash('sha1').update(cacheKey).digest('hex')}.json`);
}
/**
* Read a disk-cached installation entry. maxAgeMs Infinity = any age (the
* serve-stale path when Overpass is down).
* @returns {Promise<?{payload: object, cachedAt: number}>}
*/
async function readMilitaryInstallationDisk(
cacheKey,
maxAgeMs,
dir = MILITARY_INSTALLATION_DISK_DIR,
) {
try {
const entry = JSON.parse(await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'));
if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null;
return migrateMilitaryInstallationEntry(entry);
} catch {
return null;
}
}
/**
* Persist one installation payload ATOMICALLY: serialize to a temp sibling,
* then rename over the target. A crash or a full disk mid-write leaves the
* PREVIOUS entry intact — an in-place overwrite would shred the last-good copy
* and take serve-stale down with it, exactly when it is needed most.
* @returns {Promise<boolean>} Whether the entry landed.
*/
async function writeMilitaryInstallationDisk(
cacheKey,
entry,
dir = MILITARY_INSTALLATION_DISK_DIR,
) {
const target = militaryInstallationDiskPath(cacheKey, dir);
// Same directory, so the rename is atomic on POSIX rather than a cross-device copy.
const temp = `${target}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
try {
await fsp.mkdir(dir, { recursive: true });
await fsp.writeFile(temp, JSON.stringify(entry));
await fsp.rename(temp, target);
return true;
} catch (err) {
console.warn('[Installations Proxy] disk cache write failed:', err?.message || err);
await fsp.rm(temp, { force: true }).catch(() => {});
return false;
}
}
function trimMilitaryInstallationCache() {
while (_militaryInstallationCache.size > MILITARY_INSTALLATION_MAX_CACHE) {
const oldest = _militaryInstallationCache.keys().next().value;
if (oldest === undefined) break;
_militaryInstallationCache.delete(oldest);
}
}
export { _militaryInstallationCache, trimMilitaryInstallationCache, writeMilitaryInstallationDisk, resolveMilitaryInstallationTier, readMilitaryInstallationDisk, migrateMilitaryInstallationEntry, militaryInstallationDiskFresh, militaryInstallationDiskPath };
server/providers/military-installations/constants.js
+48
View File
@@ -0,0 +1,48 @@
import path from 'node:path';
// ---------------------------------------------------------------------------
// Military-installation context proxy
// ---------------------------------------------------------------------------
// This narrow endpoint deliberately does not expose arbitrary Overpass QL to
// the browser. It returns only allow-listed mapped context and rejects global,
// cross-dateline, or oversized requests before touching public OSM mirrors.
const MILITARY_INSTALLATION_CACHE_MS = 5 * 60_000;
const MILITARY_INSTALLATION_STALE_MS = 60 * 60_000;
const MILITARY_INSTALLATION_MAX_CACHE = 80;
const MILITARY_INSTALLATION_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
/**
* Upstream element cap. A response that hits it exactly is SATURATED — Overpass
* truncated, so off-viewport features from the snapped bbox may have crowded out
* in-viewport ones. Callers re-ask for the exact viewport in that case.
*/
const MILITARY_INSTALLATION_ELEMENT_CAP = 700;
/**
* Disk-cache TTL for mapped installations (ms) — 30 days.
*
* Owner playtest 2026-08-18: "search nearby sites" was slow because every look
* around paid a live Overpass round trip, and the 5-minute in-memory tier died
* with the dev server. Mapped military features change on a survey timescale,
* not a session one, so a month-old answer is still the right answer — the same
* reasoning the Overpass proxy already applies to admin boundaries.
*/
const MILITARY_INSTALLATION_DISK_TTL_MS = 30 * 86_400_000;
/** Disk-cache directory for mapped installation payloads. */
const MILITARY_INSTALLATION_DISK_DIR = path.join(process.cwd(), '.gev-cache', 'military-installations');
/**
* Cache-key grid step in degrees (~5.5 km).
*
* The browser sends the raw view rectangle, so every pixel of pan minted a new
* key and a new upstream query. Snapping the bbox OUTWARD onto a coarse grid
* makes neighbouring viewports share one entry, and because the snap only ever
* grows the box, the cached answer is always a superset of what was asked for.
*/
const MILITARY_INSTALLATION_BBOX_STEP_DEG = 0.05;
export { MILITARY_INSTALLATION_ELEMENT_CAP, MILITARY_INSTALLATION_MAX_RESPONSE_BYTES, MILITARY_INSTALLATION_DISK_TTL_MS, MILITARY_INSTALLATION_STALE_MS, MILITARY_INSTALLATION_CACHE_MS, MILITARY_INSTALLATION_DISK_DIR, MILITARY_INSTALLATION_MAX_CACHE, MILITARY_INSTALLATION_BBOX_STEP_DEG };
server/providers/military-installations/query.js
+60
View File
@@ -0,0 +1,60 @@
import { MILITARY_INSTALLATION_BBOX_STEP_DEG } from './constants.js';
import { requiredFiniteQueryNumber } from '../common/query.js';
/**
* Snap a request bbox outward onto the shared installation cache grid.
* @param {{south:number, west:number, north:number, east:number}} box
* @param {number} [stepDeg]
* @returns {{south:number, west:number, north:number, east:number}}
*/
function quantizeMilitaryInstallationBox(box, stepDeg = MILITARY_INSTALLATION_BBOX_STEP_DEG) {
// Round the ratio first: 29.9999/0.05 lands a hair under an exact grid line
// in binary floating point, which would otherwise snap a whole cell too far.
const snap = (value, grow) => {
const cells = Number((value / stepDeg).toFixed(9));
return Number(((grow > 0 ? Math.ceil(cells) : Math.floor(cells)) * stepDeg).toFixed(6));
};
return {
south: Math.max(-90, snap(box.south, -1)),
west: Math.max(-180, snap(box.west, -1)),
north: Math.min(90, snap(box.north, 1)),
east: Math.min(180, snap(box.east, 1)),
};
}
/**
* Stable disk/memory cache key for an installation bbox.
*
* The key's precision must match the precision of the bounds the QUERY uses, or
* two different queries collide on one entry. Snapped boxes live on a 0.05 deg
* grid, so 3 decimals is exact for them; an `exact=1` request carries the raw
* viewport at 5 decimals and must be keyed at 5, otherwise two nearby exact
* viewports would share an answer and the second would be missing the edge
* strip it just exposed.
* @param {{south:number, west:number, north:number, east:number}} box
* @param {number} [decimals]
*/
function militaryInstallationCacheKey(box, decimals = 3) {
return [box.south, box.west, box.north, box.east]
.map((value) => value.toFixed(decimals))
.join(',');
}
function validMilitaryInstallationBox(params) {
const south = requiredFiniteQueryNumber(params, 'south');
const west = requiredFiniteQueryNumber(params, 'west');
const north = requiredFiniteQueryNumber(params, 'north');
const east = requiredFiniteQueryNumber(params, 'east');
if (![south, west, north, east].every(Number.isFinite)) return null;
if (south < -90 || north > 90 || west < -180 || east > 180 || south >= north || west >= east) return null;
if (north - south > 10 || east - west > 10) return null;
return { south, west, north, east };
}
/** Safe, evidence-based reason for an installation upstream failure. */
function militaryInstallationFailureReason(error) {
if (['rate_limited', 'timeout', 'query_failed'].includes(error?.installationReason)) return error.installationReason;
return ['AbortError', 'TimeoutError'].includes(error?.name) ? 'timeout' : 'unavailable';
}
export { validMilitaryInstallationBox, quantizeMilitaryInstallationBox, militaryInstallationCacheKey, militaryInstallationFailureReason };
server/providers/overpass.js
+190
View File
@@ -0,0 +1,190 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { readRequestBodyCapped } from './common/request.js';
import { OVERPASS_MAX_BODY_BYTES, OVERPASS_MAX_CONCURRENT } from './overpass/constants.js';
import { sanitizeOverpassBody } from './overpass/query.js';
import { resolveOverpassPreflight, _overpassCache, readOverpassDisk, overpassDiskTtlMs, readStaleOverpass, trimOverpassCache, writeOverpassDisk } from './overpass/cache.js';
import { overpassPayloadIsData, fetchOverpassPayload } from './overpass/transport.js';
import { installRouteMiddleware } from './places/routes.js';
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
const _overpassInFlight = new Map();
let _overpassConcurrent = 0;
const _overpassRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 90, globalMax: 300 });
/**
* Write a completed Overpass payload to the HTTP response.
*
* @param {import('http').ServerResponse} res - Node HTTP response.
* @param {{status:number,body:string,contentType:string,endpoint:string}} payload
* @param {string} [cacheStatus='MISS'] - 'HIT', 'MISS', or 'INFLIGHT'.
*/
function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
res.writeHead(payload.status, {
'Content-Type': payload.contentType || 'application/json',
'Cache-Control': 'public, max-age=15',
'X-Overpass-Cache': cacheStatus,
'X-Overpass-Upstream': payload.endpoint || 'unknown',
});
res.end(payload.body || '');
}
/**
* Vite plugin: Overpass API proxy with response caching and request coalescing.
*
* Accepts POST requests at /api/overpass, normalizes the query body for
* cache keying, and fans out to multiple Overpass mirrors with per-upstream
* timeout and rate-limit detection. Successful responses are cached for
* OVERPASS_CACHE_MS. Concurrent identical queries share a single upstream
* request via the in-flight map.
*
* @returns {import('vite').Plugin}
*/
function overpassProxy() {
const installMiddleware = (server) => {
server.middlewares.use('/api/overpass', async (req, res) => {
// Hoisted out of the try so the catch's serve-stale lookup can see it
// (a body-read failure would otherwise hit an out-of-scope reference).
let cacheKey = null;
try {
if (req.method !== 'POST') {
res.writeHead(405, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
// Collect POST body with a hard byte cap (Overpass QL queries are small)
let body;
try {
body = (await readRequestBodyCapped(req, OVERPASS_MAX_BODY_BYTES)).toString();
} catch (err) {
if (err?.code === 'BODY_TOO_LARGE') {
res.writeHead(413, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Overpass query too large' }));
return;
}
throw err;
}
if (!body) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Missing Overpass query body' }));
return;
}
// Validate + clamp the QL: reject unbounded/global queries and cap the
// server-side timeout so a tiny body can't request planet-scale work.
const sanitized = sanitizeOverpassBody(body);
if (!sanitized.ok) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: sanitized.error }));
return;
}
const safeBody = sanitized.body;
// Normalize whitespace so semantically identical Overpass QL queries share cache entries
cacheKey = safeBody.replace(/\s+/g, ' ').trim();
const preflight = await resolveOverpassPreflight({
cacheKey,
memoryCache: _overpassCache,
inFlight: _overpassInFlight,
// Fresh-enough disk entries survive restarts and skip the public
// mirrors; boundary-class queries keep their month-long TTL.
readDisk: () => readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
});
if (preflight.source === 'RATE_LIMITED') {
res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '5' });
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
if (preflight.source !== 'UPSTREAM') {
// A coalesced caller sees the same failure as the original request
// and must get the same last-good fallback, not the raw refusal.
if (!overpassPayloadIsData(preflight.payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
if (preflight.source === 'DISK') {
_overpassCache.set(cacheKey, preflight.payload);
trimOverpassCache();
}
sendOverpassResponse(res, preflight.payload, preflight.source);
return;
}
// From here onward the request is genuinely upstream-bound and has
// consumed one local limiter slot. Cache and dedupe hits above do not.
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '2' });
res.end(JSON.stringify({ error: 'Overpass proxy busy — try again shortly' }));
return;
}
_overpassConcurrent += 1;
const requestPromise = fetchOverpassPayload(safeBody)
.then((payload) => {
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
// refusal was written to memory AND disk — and boundary-class
// queries hold a month-long TTL, so a single 406 outlived the
// outage that caused it.
if (overpassPayloadIsData(payload)) {
const entry = { ...payload, cachedAt: Date.now() };
_overpassCache.set(cacheKey, entry);
trimOverpassCache();
writeOverpassDisk(cacheKey, entry);
}
return payload;
})
.finally(() => {
_overpassConcurrent -= 1;
_overpassInFlight.delete(cacheKey);
});
_overpassInFlight.set(cacheKey, requestPromise);
const payload = await requestPromise;
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
// error): last-good roads beat an empty layer — serve stale from
// memory or disk at ANY age before surfacing the failure.
if (!overpassPayloadIsData(payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
sendOverpassResponse(res, payload, 'MISS');
} catch (e) {
// Every mirror threw (network-level). Same serve-stale rule.
const stale = cacheKey
? await readStaleOverpass(cacheKey)
: null;
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
console.error('[Overpass Proxy]', e.message);
res.writeHead(502, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Overpass proxy error' }));
}
});
installRouteMiddleware(server.middlewares);
};
return {
name: 'overpass-proxy',
configureServer: installMiddleware,
configurePreviewServer: installMiddleware,
};
}
export { overpassProxy };
export { isOverpassBoundaryQuery } from './overpass/query.js';
export { simplifyOverpassPayloadBody } from './overpass/geometry.js';
export { readOverpassDisk } from './overpass/cache.js';
export { resolveOverpassPreflight } from './overpass/cache.js';
export { overpassPayloadIsData } from './overpass/transport.js';
export { fetchOverpassPayload } from './overpass/transport.js';
server/providers/overpass/cache.js
+102
View File
@@ -0,0 +1,102 @@
import { isOverpassBoundaryQuery } from './query.js';
import { OVERPASS_BOUNDARY_DISK_TTL_MS, OVERPASS_DISK_TTL_MS, OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES } from './constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
import { overpassPayloadIsData } from './transport.js';
/** @type {Map<string,{status:number,body:string,contentType:string,endpoint:string,cachedAt:number}>} */
const _overpassCache = new Map();
/** Disk TTL for a query: boundary geometry keeps for a month, the rest 7 days. */
function overpassDiskTtlMs(cacheKey) {
return isOverpassBoundaryQuery(cacheKey) ? OVERPASS_BOUNDARY_DISK_TTL_MS : OVERPASS_DISK_TTL_MS;
}
/** Normalized Overpass query -> stable disk-cache file path. */
function overpassDiskPath(cacheKey) {
return path.join(OVERPASS_DISK_DIR, `${createHash('sha1').update(cacheKey).digest('hex')}.json`);
}
/**
* Read a disk-cached Overpass payload. maxAgeMs Infinity = any age (the
* serve-stale path when every mirror is down).
* @returns {Promise<?Object>} Payload with cachedAt, or null.
*/
async function readOverpassDisk(cacheKey, maxAgeMs) {
try {
const raw = await fsp.readFile(overpassDiskPath(cacheKey), 'utf8');
const payload = JSON.parse(raw);
if (!payload || typeof payload.body !== 'string' || !Number.isFinite(payload.cachedAt)) return null;
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
// them on both fresh and stale reads so an upgrade can recover immediately.
if (!overpassPayloadIsData(payload)) return null;
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
return payload;
} catch {
return null;
}
}
/** Fire-and-forget disk write for a successful Overpass payload. */
function writeOverpassDisk(cacheKey, payload) {
fsp.mkdir(OVERPASS_DISK_DIR, { recursive: true })
.then(() => fsp.writeFile(overpassDiskPath(cacheKey), JSON.stringify(payload)))
.catch((err) => console.warn('[Overpass Proxy] disk cache write failed:', err?.message || err));
}
/**
* Resolve every cache/coalescing layer before admitting a request to the local
* upstream rate limiter. The injected limiter callback is invoked exactly once
* for a complete cache miss and never for memory, in-flight, or disk hits.
* Exported so the admission ordering can be tested without a Vite server.
*
* @param {object} options
* @param {string} options.cacheKey
* @param {Map<string, object>} options.memoryCache
* @param {Map<string, Promise<object>>} options.inFlight
* @param {()=>Promise<object|null>} options.readDisk
* @param {()=>boolean} options.allowUpstream
* @param {number} [options.now]
* @param {number} [options.cacheMs]
* @returns {Promise<{source:'HIT'|'INFLIGHT'|'DISK'|'UPSTREAM'|'RATE_LIMITED', payload:object|null}>}
*/
async function resolveOverpassPreflight({
cacheKey,
memoryCache,
inFlight,
readDisk,
allowUpstream,
now = Date.now(),
cacheMs = OVERPASS_CACHE_MS,
}) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs) return { source: 'HIT', payload: cached };
const pending = inFlight.get(cacheKey);
if (pending) return { source: 'INFLIGHT', payload: await pending };
const disk = await readDisk();
if (overpassPayloadIsData(disk)) return { source: 'DISK', payload: disk };
return allowUpstream()
? { source: 'UPSTREAM', payload: null }
: { source: 'RATE_LIMITED', payload: null };
}
/** Return only last-good Overpass data, regardless of its age. */
async function readStaleOverpass(cacheKey) {
const cached = _overpassCache.get(cacheKey);
return overpassPayloadIsData(cached) ? cached : readOverpassDisk(cacheKey, Infinity);
}
/** Evict oldest Overpass cache entries until size is within the cap. */
function trimOverpassCache() {
while (_overpassCache.size > OVERPASS_CACHE_MAX_ENTRIES) {
const oldestKey = _overpassCache.keys().next().value;
if (!oldestKey) break;
_overpassCache.delete(oldestKey);
}
}
export { readOverpassDisk, resolveOverpassPreflight, _overpassCache, overpassDiskTtlMs, readStaleOverpass, trimOverpassCache, writeOverpassDisk };
server/providers/overpass/constants.js
+108
View File
@@ -0,0 +1,108 @@
import path from 'node:path';
// ---------------------------------------------------------------------------
// Overpass API proxy constants and cache state
// ---------------------------------------------------------------------------
/** Ordered list of Overpass API mirrors; tried sequentially on failure/rate-limit. */
const OVERPASS_UPSTREAMS = [
'https://overpass-api.de/api/interpreter',
'https://overpass.kumi.systems/api/interpreter',
'https://lz4.overpass-api.de/api/interpreter',
// Community full-planet instance (privateforge nonprofit) — added 2026-07-30
// when all three mirrors above refused this IP (likely a dev-traffic rate
// ban; refused connections fail in ms, so healthy mirrors above still win).
// Verified: planet coverage (Texas query), CORS *, ~5-20 s cold latency.
'https://overpass.private.coffee/api/interpreter',
];
/**
* TTL for FRESH cached Overpass responses (ms). Road geometry is static for
* months — the original 45 s TTL forced a public-mirror round-trip on nearly
* every viewport revisit and left nothing to serve when the mirrors 502
* (field-test 2026-07-17: all three mirrors down during US morning peak =
* "traffic takes forever to load"). 24 h in memory; the disk layer below
* keeps 7 days and also survives dev-server restarts.
*/
const OVERPASS_CACHE_MS = 86_400_000;
/** Disk-cache TTL for Overpass responses (ms) — 7 days. */
const OVERPASS_DISK_TTL_MS = 7 * 86_400_000;
/**
* Disk-cache TTL for BOUNDARY-class queries (is_in / admin-relation pivots) — 30
* days. Admin boundaries change ≈never, and their pivots are the most expensive
* queries the app issues (multi-MB coastline geometry, 10–25 s on public mirrors —
* field test 2026-07-23: outline latency + the Sicily miss). Keeping them a month
* means each boundary is fetched roughly once per machine, ever.
*/
const OVERPASS_BOUNDARY_DISK_TTL_MS = 30 * 86_400_000;
/** Disk-cache directory for Overpass responses. */
const OVERPASS_DISK_DIR = path.join(process.cwd(), '.gev-cache', 'overpass');
/** Per-upstream fetch timeout (ms). */
const OVERPASS_TIMEOUT_MS = 22000;
/** Max entries in the Overpass response cache (LRU-like, oldest evicted first). */
const OVERPASS_CACHE_MAX_ENTRIES = 120;
// --- Abuse guards shared by the Overpass + route proxies --------------------
/** Max accepted POST body for the Overpass proxy (Overpass QL queries are tiny). */
const OVERPASS_MAX_BODY_BYTES = 24 * 1024;
// 24 KB
/**
* Hard cap on a single Overpass upstream response we will buffer into memory.
* 32 MB (was 12 MB): a dense island/state admin boundary at full `out geom`
* fidelity — Sicilia's Mediterranean coastline — can exceed 12 MB, and clipping
* it read as a permanent "transient" failure (field test 2026-07-23, Sicily
* never traced). The buffered payload is SIMPLIFIED server-side before it is
* cached or sent (simplifyOverpassPayloadBody), so the raised cap does not
* raise what clients receive or what the disk stores.
*/
const OVERPASS_MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
// 32 MB
/** Only payloads at least this large go through geometry simplification. */
const OVERPASS_SIMPLIFY_MIN_BYTES = 1_500_000;
/** Only per-element geometry arrays with at least this many points are simplified. */
const OVERPASS_SIMPLIFY_MIN_POINTS = 1200;
/**
* Douglas-Peucker tolerance (degrees, ≈44 m of latitude). Region/state boundary
* rings are drawn at regional camera scale and the client simplifies again for
* draw, so ~44 m fidelity is invisible; building footprints never reach the
* point threshold above and pass through untouched.
*/
const OVERPASS_SIMPLIFY_TOLERANCE_DEG = 0.0004;
/** Max concurrent in-flight upstream Overpass fetches across all distinct queries. */
const OVERPASS_MAX_CONCURRENT = 6;
/** Server-side timeout ceiling (seconds) we allow inside an Overpass QL query. */
const OVERPASS_MAX_QL_TIMEOUT = 30;
/** Max `around:` radius (m) — every app caller uses <= 1800 m. */
const OVERPASS_MAX_AROUND_M = 50000;
/** Max bbox span (degrees) — app bboxes are small viewport tiles. */
const OVERPASS_MAX_BBOX_DEG = 12;
/**
* Every Overpass element-type specifier, including the combined shortcuts
* (nwr/nw/nr/wr) and `rel`. Shared by the selector + area-element-deny regexes so
* they can't drift (a missing shortcut like `wr` was an area-scan bypass).
*/
const OVERPASS_ELEMENT_TYPES = 'node|way|relation|nwr|nw|nr|wr|rel';
/** Element-selector (incl. `area`) whose statements must be individually bounded. */
const OVERPASS_SELECTOR_RE = new RegExp(`\\b(?:${OVERPASS_ELEMENT_TYPES}|area)\\b`);
/** An element selector bounded BY an area — the country-scan abuse shape. */
const OVERPASS_AREA_ELEMENT_RE = new RegExp(`\\b(?:${OVERPASS_ELEMENT_TYPES})\\s*\\(\\s*area\\b`, 'i');
/** A single bbox 4-tuple `(s,w,n,e)` (non-global so it does not advance lastIndex). */
const OVERPASS_BBOX_RE = /\(\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*\)/;
export { OVERPASS_BOUNDARY_DISK_TTL_MS, OVERPASS_DISK_TTL_MS, OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES, OVERPASS_MAX_BODY_BYTES, OVERPASS_MAX_CONCURRENT, OVERPASS_MAX_AROUND_M, OVERPASS_MAX_BBOX_DEG, OVERPASS_AREA_ELEMENT_RE, OVERPASS_SELECTOR_RE, OVERPASS_BBOX_RE, OVERPASS_MAX_QL_TIMEOUT, OVERPASS_SIMPLIFY_MIN_BYTES, OVERPASS_SIMPLIFY_MIN_POINTS, OVERPASS_SIMPLIFY_TOLERANCE_DEG, OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, OVERPASS_TIMEOUT_MS };
server/providers/overpass/geometry.js
+92
View File
@@ -0,0 +1,92 @@
import { OVERPASS_SIMPLIFY_MIN_BYTES, OVERPASS_SIMPLIFY_MIN_POINTS, OVERPASS_SIMPLIFY_TOLERANCE_DEG } from './constants.js';
/** Iterative Douglas-Peucker on [{lat,lon},...] (planar-degree approx — fine at
* the ~44 m tolerance used here). Endpoints always kept. */
function douglasPeucker(points, toleranceDeg) {
const n = points.length;
if (n <= 2) return points;
const keep = new Uint8Array(n);
keep[0] = 1;
keep[n - 1] = 1;
const stack = [[0, n - 1]];
while (stack.length) {
const [a, b] = stack.pop();
if (b - a < 2) continue;
const ax = points[a].lon;
const ay = points[a].lat;
const vx = points[b].lon - ax;
const vy = points[b].lat - ay;
const c2 = vx * vx + vy * vy;
let worst = -1;
let worstDist = toleranceDeg;
for (let i = a + 1; i < b; i++) {
const wx = points[i].lon - ax;
const wy = points[i].lat - ay;
let d;
if (c2 === 0) {
d = Math.hypot(wx, wy);
} else {
const t = Math.max(0, Math.min(1, (vx * wx + vy * wy) / c2));
d = Math.hypot(wx - t * vx, wy - t * vy);
}
if (d > worstDist) {
worstDist = d;
worst = i;
}
}
if (worst >= 0) {
keep[worst] = 1;
stack.push([a, worst], [worst, b]);
}
}
const out = [];
for (let i = 0; i < n; i++) if (keep[i]) out.push(points[i]);
return out;
}
/** Simplify one element's geometry array in place if it is big enough. */
function simplifyElementGeometry(el, minPoints, toleranceDeg) {
if (Array.isArray(el?.geometry) && el.geometry.length >= minPoints) {
el.geometry = douglasPeucker(el.geometry, toleranceDeg);
}
if (Array.isArray(el?.members)) {
for (const member of el.members) {
if (Array.isArray(member?.geometry) && member.geometry.length >= minPoints) {
member.geometry = douglasPeucker(member.geometry, toleranceDeg);
}
}
}
}
/**
* Server-side geometry simplification for large Overpass `out geom` payloads.
* Region/state boundary pivots return multi-MB coastline rings whose fidelity
* nothing downstream needs (the client re-simplifies for draw); decimating them
* HERE shrinks the disk cache, the wire, and client parse time — and is what
* makes the raised read cap safe. Anything unparseable or below the thresholds
* passes through byte-identical. Exported for tests (opts override thresholds).
*
* @param {string} bodyText - Raw upstream JSON body.
* @returns {string} Possibly-simplified JSON body.
*/
function simplifyOverpassPayloadBody(bodyText, opts = {}) {
const minBytes = opts.minBytes ?? OVERPASS_SIMPLIFY_MIN_BYTES;
const minPoints = opts.minPoints ?? OVERPASS_SIMPLIFY_MIN_POINTS;
const toleranceDeg = opts.toleranceDeg ?? OVERPASS_SIMPLIFY_TOLERANCE_DEG;
if (typeof bodyText !== 'string' || bodyText.length < minBytes) return bodyText;
let data;
try {
data = JSON.parse(bodyText);
} catch {
return bodyText;
}
if (!Array.isArray(data?.elements)) return bodyText;
for (const el of data.elements) simplifyElementGeometry(el, minPoints, toleranceDeg);
try {
return JSON.stringify(data);
} catch {
return bodyText;
}
}
export { simplifyOverpassPayloadBody };
server/providers/overpass/query.js
+161
View File
@@ -0,0 +1,161 @@
import { OVERPASS_MAX_AROUND_M, OVERPASS_MAX_BBOX_DEG, OVERPASS_AREA_ELEMENT_RE, OVERPASS_SELECTOR_RE, OVERPASS_BBOX_RE, OVERPASS_MAX_QL_TIMEOUT } from './constants.js';
/**
* Whether a normalized Overpass query is BOUNDARY-class (admin `is_in` lookups
* and area→relation pivots) — the static, expensive geometry that earns the
* 30-day disk TTL. The enclosing-compound sweep and road fetches keep the
* default TTL. Exported for tests.
*/
function isOverpassBoundaryQuery(cacheKey) {
return /is_in\s*\(|\bpivot\b/i.test(String(cacheKey || ''));
}
/**
* Validate + clamp an Overpass form body. Defends the generic proxy against
* planet-scale abuse: requires exactly one `data` query in which EVERY element
* selector is individually spatially bounded (around / bbox / is_in / poly /
* area-set / pivot), rejects oversized radii and world-sized bboxes, and clamps
* every `[timeout:]` directive. Comments + quoted literals are stripped first so
* a fake bound inside a tag value can't satisfy the check.
*
* Every real app caller passes (annotations/locations/cctv use `around:`/`is_in`/
* `area.`/`pivot`; traffic uses a small `(s,w,n,e)` bbox); a mixed query that
* pairs one bounded selector with a global one is rejected.
*
* @returns {{ok:true, body:string} | {ok:false, error:string}}
*/
/**
* Single-pass lexer: blank out quoted literals (→ empty quotes) and strip line
* and block comments — recognizing each in one walk so a comment marker INSIDE a
* quoted string is treated as string content, not a comment (and vice versa).
* Chained regex replaces get the ordering wrong (a quoted slash-slash would hide
* the rest of the line), which is exactly the bypass this avoids.
*/
function stripOverpassNoise(src) {
let out = '';
let i = 0;
const n = src.length;
while (i < n) {
const c = src[i];
if (c === '"' || c === "'") {
const quote = c;
i += 1;
while (i < n) {
if (src[i] === '\\') { i += 2; continue; } // escaped char
if (src[i] === quote) { i += 1; break; } // closing quote
i += 1;
}
out += quote + quote; // collapse the literal to empty quotes
continue;
}
if (c === '/' && src[i + 1] === '*') {
i += 2;
while (i < n && !(src[i] === '*' && src[i + 1] === '/')) i += 1;
i += 2;
out += ' ';
continue;
}
if (c === '/' && src[i + 1] === '/') {
i += 2;
while (i < n && src[i] !== '\n') i += 1;
out += ' ';
continue;
}
out += c;
i += 1;
}
return out;
}
function sanitizeOverpassBody(rawBody) {
let params;
try { params = new URLSearchParams(rawBody); } catch { return { ok: false, error: 'Malformed query body' }; }
const all = params.getAll('data');
if (all.length !== 1) return { ok: false, error: 'Exactly one data query is required' };
const data = all[0];
if (!data || !data.trim()) return { ok: false, error: 'Missing Overpass data query' };
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
const stripped = stripOverpassNoise(data);
// Reject oversized radii in EVERY around form — point `around:r,lat,lon` AND the
// input-set form `around.set:r` — and parse the full numeric token so scientific
// notation (`5e7`) can't slip a planet-scale radius past the cap.
for (const m of stripped.matchAll(/around(?:\.\w+)?:\s*([\d.eE+-]+)/gi)) {
const radius = Number(m[1]);
if (!Number.isFinite(radius) || radius > OVERPASS_MAX_AROUND_M) {
return { ok: false, error: 'Overpass around radius too large' };
}
}
// Reject world-sized / oversized bboxes.
for (const m of stripped.matchAll(/\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)/g)) {
const s = Number(m[1]); const w = Number(m[2]); const n = Number(m[3]); const e = Number(m[4]);
if (Math.abs(n - s) > OVERPASS_MAX_BBOX_DEG || Math.abs(e - w) > OVERPASS_MAX_BBOX_DEG) {
return { ok: false, error: 'Overpass bbox too large' };
}
}
// Reject control-flow constructs the app never uses — their set/bound semantics
// are hard to validate statically. The app only uses plain selectors + is_in /
// area / pivot / recursion, so this denylist closes loop/transform escape hatches.
if (/\b(?:foreach|complete|retro|compare|convert|make)\b/i.test(stripped)) {
return { ok: false, error: 'Unsupported Overpass construct' };
}
// `poly:` has unchecked extent and the app never uses it — reject outright
// (position-independent, so tag filters can't hide it).
if (/\bpoly\s*:/i.test(stripped)) {
return { ok: false, error: 'Overpass poly filter not allowed' };
}
// Every selector statement must be individually bounded, WITH set provenance: a
// set counts as a bound only if it was assigned (->.set) by an already-bounded
// statement. So `way[...]->.a` (global assigned to a set) is rejected, while the
// app's `is_in(...)->.a; area.a[...]` and `area(id)->.x; rel(pivot.x)` validate.
const boundedSets = new Set();
for (let stmt of stripped.split(';')) {
stmt = stmt.trim();
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
// Strip output-set assignments (NOT input bounds), then strip bracket tag
// filters so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) can never be
// misread as a spatial bound. Bounds live in (...) / function calls / set
// refs, never inside [...], so the probe loses nothing real.
const outSets = [];
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => { outSets.push(name); return ' '; });
const probe = body.replace(/\[[^\]]*\]/g, ' ');
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
// between the selector and the area filter (way["highway"](area.a)) can't hide
// it. An area has unbounded extent (could be a whole country); the app only
// SELECTS admin areas (area.set) and pivots (rel(pivot.x)), never node/way/
// relation(area...). The probe collapses tags so `way (area.a)` is caught.
if (OVERPASS_AREA_ELEMENT_RE.test(probe)) {
return { ok: false, error: 'Overpass area-bounded element selector not allowed' };
}
const hasSelector = OVERPASS_SELECTOR_RE.test(probe);
const inputSets = [...probe.matchAll(/(?<!\d)\.([a-z_]\w*)/gi)].map((m) => m[1]);
const directBound = /around:\s*\d/.test(probe)
|| OVERPASS_BBOX_RE.test(probe)
|| /is_in\s*\(/.test(probe) // is_in(lat,lon) — the function form only
|| /\barea\s*\(/.test(probe); // area(id) — bounded as a set definition
const setBound = inputSets.some((s) => boundedSets.has(s));
const bounded = directBound || setBound;
if (hasSelector && !bounded) {
return { ok: false, error: 'Overpass query has an unbounded selector' };
}
// Only a bounded statement can mark its output sets as bounded.
if (bounded) for (const name of outSets) boundedSets.add(name);
}
const clamped = data.replace(
/\[timeout:\s*(\d+)\s*\]/gi,
(_, n) => `[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
);
return { ok: true, body: `data=${encodeURIComponent(clamped)}` };
}
export { isOverpassBoundaryQuery, sanitizeOverpassBody };
server/providers/overpass/transport.js
+139
View File
@@ -0,0 +1,139 @@
import { OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, OVERPASS_TIMEOUT_MS } from './constants.js';
import { readResponseTextCapped } from '../common/http.js';
import { simplifyOverpassPayloadBody } from './geometry.js';
/**
* Detect whether an Overpass API response body indicates rate-limiting.
*
* Checks for known rate-limit phrases in the body text regardless of
* HTTP status code, since some mirrors return 200 with an error payload.
*
* @param {string} bodyText - Upstream response body.
* @returns {boolean} True if the body looks rate-limited.
*/
function overpassLooksRateLimited(bodyText) {
const text = String(bodyText || '').toLowerCase();
return text.includes('rate_limited')
|| text.includes('quota of your ip address')
|| text.includes('dispatcher_client::request_read_and_idx::rate_limited')
|| text.includes('too many requests');
}
/**
* Detect an Overpass HTTP-200 body that is actually a runtime FAILURE (server-side
* timeout / out-of-memory) via its `remark`. These are transient upstream failures,
* not authoritative empty results, so they must not be returned or cached.
*/
function overpassLooksRuntimeError(bodyText) {
const text = String(bodyText || '').toLowerCase();
return text.includes('runtime error')
|| text.includes('timed out')
|| text.includes('out of memory');
}
/**
* True only for an upstream response that is actually Overpass data.
*
* The proxy caches on this and serves stale on its negation, so the two
* decisions cannot drift apart: a payload that is not data must never be
* written to the cache and must always be eligible for a stale replacement.
* @param {{status: number, rateLimited?: boolean, runtimeError?: boolean}} payload
* @returns {boolean}
*/
function overpassPayloadIsData(payload) {
const status = Number(payload?.status);
return Number.isFinite(status)
&& status >= 200 && status < 300
&& !payload.rateLimited
&& !payload.runtimeError;
}
/**
* Try each mirror once, retaining response-size and per-mirror timeout caps.
* Refusals and body-level failures rotate; total failure returns the last
* rate-limit payload, otherwise the first refusal, or throws a network error.
* @param {string} body URL-encoded Overpass QL query body.
* @param {number} [maxResponseBytes] Endpoint-specific response cap.
* @param {object} [options] Server-only endpoint and I/O overrides for tests.
* @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
*/
async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES, {
endpoints = OVERPASS_UPSTREAMS,
fetchImpl = fetch,
readBody = readResponseTextCapped,
simplify = simplifyOverpassPayloadBody,
} = {}) {
let lastError = null;
let lastRateLimitPayload = null;
let lastRefusalPayload = null;
for (const endpoint of endpoints) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
try {
const upstream = await fetchImpl(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'gods-eye-view-overpass-proxy/1.0',
},
body,
signal: controller.signal,
});
const responseBody = await readBody(upstream, maxResponseBytes);
const contentType = upstream.headers.get('content-type') || 'application/json';
const status = upstream.status;
const rateLimited = status === 429 || overpassLooksRateLimited(responseBody);
const runtimeError = overpassLooksRuntimeError(responseBody);
const payload = {
status,
body: responseBody,
contentType,
endpoint,
rateLimited,
runtimeError,
};
if (rateLimited) {
lastRateLimitPayload = payload;
continue;
}
// A 200 body carrying a runtime error / timeout is a transient upstream
// failure — skip to the next mirror rather than returning or caching it.
if (runtimeError) {
lastError = new Error(`Overpass runtime error (${endpoint})`);
continue;
}
// Anything but 2xx is this mirror declining, not an answer. Only 5xx used
// to rotate, so a 4xx ended the fan-out and was returned — and cached —
// as data: overpass-api.de and its lz4 alias answer 406 to this proxy's
// User-Agent while kumi.systems and private.coffee answer 200 to the very
// same request, so every Overpass-backed layer failed on an Apache error
// page with two healthy mirrors untried. The first refusal is kept so a
// genuinely bad query still reports what upstream said, but only after
// every mirror has had the chance to answer it.
if (status < 200 || status >= 300) {
if (!lastRefusalPayload) lastRefusalPayload = payload;
lastError = new Error(`Overpass upstream returned ${status} (${endpoint})`);
continue;
}
// Success: decimate giant boundary geometry before it reaches the cache,
// the disk, or the client (what makes the 32 MB read cap safe to hold).
payload.body = simplify(payload.body);
return payload;
} catch (error) {
lastError = error;
} finally {
clearTimeout(timeoutId);
}
}
if (lastRateLimitPayload) return lastRateLimitPayload;
if (lastRefusalPayload) return lastRefusalPayload;
throw lastError || new Error('All Overpass upstreams failed');
}
export { overpassPayloadIsData, fetchOverpassPayload };