mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
style: format extracted local services
This commit is contained in:
1 parent
233285fe12
commit
5223b6df5b
33 files changed
+1785
-830
No files matched your search
+145
-116
@@ -1,9 +1,23 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { readRequestBodyCapped } from './common/request.js';
|
||||
import { OVERPASS_MAX_BODY_BYTES, OVERPASS_MAX_CONCURRENT } from './overpass/constants.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
} from './overpass/constants.js';
|
||||
import { sanitizeOverpassBody } from './overpass/query.js';
|
||||
import { resolveOverpassPreflight, _overpassCache, readOverpassDisk, overpassDiskTtlMs, readStaleOverpass, trimOverpassCache, writeOverpassDisk } from './overpass/cache.js';
|
||||
import { overpassPayloadIsData, fetchOverpassPayload } from './overpass/transport.js';
|
||||
import {
|
||||
resolveOverpassPreflight,
|
||||
_overpassCache,
|
||||
readOverpassDisk,
|
||||
overpassDiskTtlMs,
|
||||
readStaleOverpass,
|
||||
trimOverpassCache,
|
||||
writeOverpassDisk,
|
||||
} from './overpass/cache.js';
|
||||
import {
|
||||
overpassPayloadIsData,
|
||||
fetchOverpassPayload,
|
||||
} from './overpass/transport.js';
|
||||
import { installRouteMiddleware } from './places/routes.js';
|
||||
|
||||
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
|
||||
@@ -11,7 +25,11 @@ const _overpassInFlight = new Map();
|
||||
|
||||
let _overpassConcurrent = 0;
|
||||
|
||||
const _overpassRateLimiter = makeRateLimiter({ windowMs: 60_000, max: 90, globalMax: 300 });
|
||||
const _overpassRateLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 90,
|
||||
globalMax: 300,
|
||||
});
|
||||
|
||||
/**
|
||||
* Write a completed Overpass payload to the HTTP response.
|
||||
@@ -43,136 +61,147 @@ function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
|
||||
*/
|
||||
function overpassProxy() {
|
||||
const installMiddleware = (server) => {
|
||||
server.middlewares.use('/api/overpass', async (req, res) => {
|
||||
// Hoisted out of the try so the catch's serve-stale lookup can see it
|
||||
// (a body-read failure would otherwise hit an out-of-scope reference).
|
||||
let cacheKey = null;
|
||||
server.middlewares.use('/api/overpass', async (req, res) => {
|
||||
// Hoisted out of the try so the catch's serve-stale lookup can see it
|
||||
// (a body-read failure would otherwise hit an out-of-scope reference).
|
||||
let cacheKey = null;
|
||||
try {
|
||||
if (req.method !== 'POST') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect POST body with a hard byte cap (Overpass QL queries are small)
|
||||
let body;
|
||||
try {
|
||||
if (req.method !== 'POST') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
body = (
|
||||
await readRequestBodyCapped(req, OVERPASS_MAX_BODY_BYTES)
|
||||
).toString();
|
||||
} catch (err) {
|
||||
if (err?.code === 'BODY_TOO_LARGE') {
|
||||
res.writeHead(413, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Overpass query too large' }));
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!body) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Missing Overpass query body' }));
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect POST body with a hard byte cap (Overpass QL queries are small)
|
||||
let body;
|
||||
try {
|
||||
body = (await readRequestBodyCapped(req, OVERPASS_MAX_BODY_BYTES)).toString();
|
||||
} catch (err) {
|
||||
if (err?.code === 'BODY_TOO_LARGE') {
|
||||
res.writeHead(413, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Overpass query too large' }));
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!body) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Missing Overpass query body' }));
|
||||
return;
|
||||
}
|
||||
// Validate + clamp the QL: reject unbounded/global queries and cap the
|
||||
// server-side timeout so a tiny body can't request planet-scale work.
|
||||
const sanitized = sanitizeOverpassBody(body);
|
||||
if (!sanitized.ok) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: sanitized.error }));
|
||||
return;
|
||||
}
|
||||
const safeBody = sanitized.body;
|
||||
|
||||
// Validate + clamp the QL: reject unbounded/global queries and cap the
|
||||
// server-side timeout so a tiny body can't request planet-scale work.
|
||||
const sanitized = sanitizeOverpassBody(body);
|
||||
if (!sanitized.ok) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: sanitized.error }));
|
||||
return;
|
||||
}
|
||||
const safeBody = sanitized.body;
|
||||
|
||||
// Normalize whitespace so semantically identical Overpass QL queries share cache entries
|
||||
cacheKey = safeBody.replace(/\s+/g, ' ').trim();
|
||||
const preflight = await resolveOverpassPreflight({
|
||||
cacheKey,
|
||||
memoryCache: _overpassCache,
|
||||
inFlight: _overpassInFlight,
|
||||
// Fresh-enough disk entries survive restarts and skip the public
|
||||
// mirrors; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () => readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
// Normalize whitespace so semantically identical Overpass QL queries share cache entries
|
||||
cacheKey = safeBody.replace(/\s+/g, ' ').trim();
|
||||
const preflight = await resolveOverpassPreflight({
|
||||
cacheKey,
|
||||
memoryCache: _overpassCache,
|
||||
inFlight: _overpassInFlight,
|
||||
// Fresh-enough disk entries survive restarts and skip the public
|
||||
// mirrors; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () =>
|
||||
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
});
|
||||
if (preflight.source === 'RATE_LIMITED') {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '5',
|
||||
});
|
||||
if (preflight.source === 'RATE_LIMITED') {
|
||||
res.writeHead(429, { 'Content-Type': 'application/json', 'Retry-After': '5' });
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
if (preflight.source !== 'UPSTREAM') {
|
||||
// A coalesced caller sees the same failure as the original request
|
||||
// and must get the same last-good fallback, not the raw refusal.
|
||||
if (!overpassPayloadIsData(preflight.payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (preflight.source === 'DISK') {
|
||||
_overpassCache.set(cacheKey, preflight.payload);
|
||||
trimOverpassCache();
|
||||
}
|
||||
sendOverpassResponse(res, preflight.payload, preflight.source);
|
||||
return;
|
||||
}
|
||||
|
||||
// From here onward the request is genuinely upstream-bound and has
|
||||
// consumed one local limiter slot. Cache and dedupe hits above do not.
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
|
||||
res.writeHead(503, { 'Content-Type': 'application/json', 'Retry-After': '2' });
|
||||
res.end(JSON.stringify({ error: 'Overpass proxy busy — try again shortly' }));
|
||||
return;
|
||||
}
|
||||
_overpassConcurrent += 1;
|
||||
const requestPromise = fetchOverpassPayload(safeBody)
|
||||
.then((payload) => {
|
||||
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
|
||||
// refusal was written to memory AND disk — and boundary-class
|
||||
// queries hold a month-long TTL, so a single 406 outlived the
|
||||
// outage that caused it.
|
||||
if (overpassPayloadIsData(payload)) {
|
||||
const entry = { ...payload, cachedAt: Date.now() };
|
||||
_overpassCache.set(cacheKey, entry);
|
||||
trimOverpassCache();
|
||||
writeOverpassDisk(cacheKey, entry);
|
||||
}
|
||||
return payload;
|
||||
})
|
||||
.finally(() => {
|
||||
_overpassConcurrent -= 1;
|
||||
_overpassInFlight.delete(cacheKey);
|
||||
});
|
||||
|
||||
_overpassInFlight.set(cacheKey, requestPromise);
|
||||
const payload = await requestPromise;
|
||||
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
|
||||
// error): last-good roads beat an empty layer — serve stale from
|
||||
// memory or disk at ANY age before surfacing the failure.
|
||||
if (!overpassPayloadIsData(payload)) {
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
if (preflight.source !== 'UPSTREAM') {
|
||||
// A coalesced caller sees the same failure as the original request
|
||||
// and must get the same last-good fallback, not the raw refusal.
|
||||
if (!overpassPayloadIsData(preflight.payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
}
|
||||
sendOverpassResponse(res, payload, 'MISS');
|
||||
} catch (e) {
|
||||
// Every mirror threw (network-level). Same serve-stale rule.
|
||||
const stale = cacheKey
|
||||
? await readStaleOverpass(cacheKey)
|
||||
: null;
|
||||
if (preflight.source === 'DISK') {
|
||||
_overpassCache.set(cacheKey, preflight.payload);
|
||||
trimOverpassCache();
|
||||
}
|
||||
sendOverpassResponse(res, preflight.payload, preflight.source);
|
||||
return;
|
||||
}
|
||||
|
||||
// From here onward the request is genuinely upstream-bound and has
|
||||
// consumed one local limiter slot. Cache and dedupe hits above do not.
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
|
||||
res.writeHead(503, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '2',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'Overpass proxy busy — try again shortly',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
_overpassConcurrent += 1;
|
||||
const requestPromise = fetchOverpassPayload(safeBody)
|
||||
.then((payload) => {
|
||||
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
|
||||
// refusal was written to memory AND disk — and boundary-class
|
||||
// queries hold a month-long TTL, so a single 406 outlived the
|
||||
// outage that caused it.
|
||||
if (overpassPayloadIsData(payload)) {
|
||||
const entry = { ...payload, cachedAt: Date.now() };
|
||||
_overpassCache.set(cacheKey, entry);
|
||||
trimOverpassCache();
|
||||
writeOverpassDisk(cacheKey, entry);
|
||||
}
|
||||
return payload;
|
||||
})
|
||||
.finally(() => {
|
||||
_overpassConcurrent -= 1;
|
||||
_overpassInFlight.delete(cacheKey);
|
||||
});
|
||||
|
||||
_overpassInFlight.set(cacheKey, requestPromise);
|
||||
const payload = await requestPromise;
|
||||
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
|
||||
// error): last-good roads beat an empty layer — serve stale from
|
||||
// memory or disk at ANY age before surfacing the failure.
|
||||
if (!overpassPayloadIsData(payload)) {
|
||||
const stale = await readStaleOverpass(cacheKey);
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
console.error('[Overpass Proxy]', e.message);
|
||||
res.writeHead(502, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Overpass proxy error' }));
|
||||
}
|
||||
});
|
||||
sendOverpassResponse(res, payload, 'MISS');
|
||||
} catch (e) {
|
||||
// Every mirror threw (network-level). Same serve-stale rule.
|
||||
const stale = cacheKey ? await readStaleOverpass(cacheKey) : null;
|
||||
if (stale) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
console.error('[Overpass Proxy]', e.message);
|
||||
res.writeHead(502, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Overpass proxy error' }));
|
||||
}
|
||||
});
|
||||
|
||||
installRouteMiddleware(server.middlewares);
|
||||
};
|
||||
installRouteMiddleware(server.middlewares);
|
||||
};
|
||||
return {
|
||||
name: 'overpass-proxy',
|
||||
configureServer: installMiddleware,
|
||||
|
||||
Reference in new issue
Block a user