style: format extracted local services

This commit is contained in:
Sameh Khamis committed 2026-09-12 13:01:49 -07:00
1 parent 233285fe12
commit 5223b6df5b
33 files changed
+1785 -830

No files matched your search

+45 -10
View File
@@ -1,5 +1,11 @@
import { isOverpassBoundaryQuery } from './query.js';
import { OVERPASS_BOUNDARY_DISK_TTL_MS, OVERPASS_DISK_TTL_MS, OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES } from './constants.js';
import {
OVERPASS_BOUNDARY_DISK_TTL_MS,
OVERPASS_DISK_TTL_MS,
OVERPASS_DISK_DIR,
OVERPASS_CACHE_MS,
OVERPASS_CACHE_MAX_ENTRIES,
} from './constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
@@ -10,12 +16,17 @@ const _overpassCache = new Map();
/** Disk TTL for a query: boundary geometry keeps for a month, the rest 7 days. */
function overpassDiskTtlMs(cacheKey) {
return isOverpassBoundaryQuery(cacheKey) ? OVERPASS_BOUNDARY_DISK_TTL_MS : OVERPASS_DISK_TTL_MS;
return isOverpassBoundaryQuery(cacheKey)
? OVERPASS_BOUNDARY_DISK_TTL_MS
: OVERPASS_DISK_TTL_MS;
}
/** Normalized Overpass query -> stable disk-cache file path. */
function overpassDiskPath(cacheKey) {
return path.join(OVERPASS_DISK_DIR, `${createHash('sha1').update(cacheKey).digest('hex')}.json`);
return path.join(
OVERPASS_DISK_DIR,
`${createHash('sha1').update(cacheKey).digest('hex')}.json`,
);
}
/**
@@ -27,7 +38,12 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
try {
const raw = await fsp.readFile(overpassDiskPath(cacheKey), 'utf8');
const payload = JSON.parse(raw);
if (!payload || typeof payload.body !== 'string' || !Number.isFinite(payload.cachedAt)) return null;
if (
!payload ||
typeof payload.body !== 'string' ||
!Number.isFinite(payload.cachedAt)
)
return null;
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
// them on both fresh and stale reads so an upgrade can recover immediately.
if (!overpassPayloadIsData(payload)) return null;
@@ -40,9 +56,17 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
/** Fire-and-forget disk write for a successful Overpass payload. */
function writeOverpassDisk(cacheKey, payload) {
fsp.mkdir(OVERPASS_DISK_DIR, { recursive: true })
.then(() => fsp.writeFile(overpassDiskPath(cacheKey), JSON.stringify(payload)))
.catch((err) => console.warn('[Overpass Proxy] disk cache write failed:', err?.message || err));
fsp
.mkdir(OVERPASS_DISK_DIR, { recursive: true })
.then(() =>
fsp.writeFile(overpassDiskPath(cacheKey), JSON.stringify(payload)),
)
.catch((err) =>
console.warn(
'[Overpass Proxy] disk cache write failed:',
err?.message || err,
),
);
}
/**
@@ -71,7 +95,8 @@ async function resolveOverpassPreflight({
cacheMs = OVERPASS_CACHE_MS,
}) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs) return { source: 'HIT', payload: cached };
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
return { source: 'HIT', payload: cached };
const pending = inFlight.get(cacheKey);
if (pending) return { source: 'INFLIGHT', payload: await pending };
@@ -87,7 +112,9 @@ async function resolveOverpassPreflight({
/** Return only last-good Overpass data, regardless of its age. */
async function readStaleOverpass(cacheKey) {
const cached = _overpassCache.get(cacheKey);
return overpassPayloadIsData(cached) ? cached : readOverpassDisk(cacheKey, Infinity);
return overpassPayloadIsData(cached)
? cached
: readOverpassDisk(cacheKey, Infinity);
}
/** Evict oldest Overpass cache entries until size is within the cap. */
@@ -99,4 +126,12 @@ function trimOverpassCache() {
}
}
export { readOverpassDisk, resolveOverpassPreflight, _overpassCache, overpassDiskTtlMs, readStaleOverpass, trimOverpassCache, writeOverpassDisk };
export {
readOverpassDisk,
resolveOverpassPreflight,
_overpassCache,
overpassDiskTtlMs,
readStaleOverpass,
trimOverpassCache,
writeOverpassDisk,
};
+30 -4
View File
@@ -97,12 +97,38 @@ const OVERPASS_MAX_BBOX_DEG = 12;
const OVERPASS_ELEMENT_TYPES = 'node|way|relation|nwr|nw|nr|wr|rel';
/** Element-selector (incl. `area`) whose statements must be individually bounded. */
const OVERPASS_SELECTOR_RE = new RegExp(`\\b(?:${OVERPASS_ELEMENT_TYPES}|area)\\b`);
const OVERPASS_SELECTOR_RE = new RegExp(
`\\b(?:${OVERPASS_ELEMENT_TYPES}|area)\\b`,
);
/** An element selector bounded BY an area — the country-scan abuse shape. */
const OVERPASS_AREA_ELEMENT_RE = new RegExp(`\\b(?:${OVERPASS_ELEMENT_TYPES})\\s*\\(\\s*area\\b`, 'i');
const OVERPASS_AREA_ELEMENT_RE = new RegExp(
`\\b(?:${OVERPASS_ELEMENT_TYPES})\\s*\\(\\s*area\\b`,
'i',
);
/** A single bbox 4-tuple `(s,w,n,e)` (non-global so it does not advance lastIndex). */
const OVERPASS_BBOX_RE = /\(\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*\)/;
const OVERPASS_BBOX_RE =
/\(\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*,\s*-?\d+(?:\.\d+)?\s*\)/;
export { OVERPASS_BOUNDARY_DISK_TTL_MS, OVERPASS_DISK_TTL_MS, OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES, OVERPASS_MAX_BODY_BYTES, OVERPASS_MAX_CONCURRENT, OVERPASS_MAX_AROUND_M, OVERPASS_MAX_BBOX_DEG, OVERPASS_AREA_ELEMENT_RE, OVERPASS_SELECTOR_RE, OVERPASS_BBOX_RE, OVERPASS_MAX_QL_TIMEOUT, OVERPASS_SIMPLIFY_MIN_BYTES, OVERPASS_SIMPLIFY_MIN_POINTS, OVERPASS_SIMPLIFY_TOLERANCE_DEG, OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, OVERPASS_TIMEOUT_MS };
export {
OVERPASS_BOUNDARY_DISK_TTL_MS,
OVERPASS_DISK_TTL_MS,
OVERPASS_DISK_DIR,
OVERPASS_CACHE_MS,
OVERPASS_CACHE_MAX_ENTRIES,
OVERPASS_MAX_BODY_BYTES,
OVERPASS_MAX_CONCURRENT,
OVERPASS_MAX_AROUND_M,
OVERPASS_MAX_BBOX_DEG,
OVERPASS_AREA_ELEMENT_RE,
OVERPASS_SELECTOR_RE,
OVERPASS_BBOX_RE,
OVERPASS_MAX_QL_TIMEOUT,
OVERPASS_SIMPLIFY_MIN_BYTES,
OVERPASS_SIMPLIFY_MIN_POINTS,
OVERPASS_SIMPLIFY_TOLERANCE_DEG,
OVERPASS_MAX_RESPONSE_BYTES,
OVERPASS_UPSTREAMS,
OVERPASS_TIMEOUT_MS,
};
+13 -4
View File
@@ -1,4 +1,8 @@
import { OVERPASS_SIMPLIFY_MIN_BYTES, OVERPASS_SIMPLIFY_MIN_POINTS, OVERPASS_SIMPLIFY_TOLERANCE_DEG } from './constants.js';
import {
OVERPASS_SIMPLIFY_MIN_BYTES,
OVERPASS_SIMPLIFY_MIN_POINTS,
OVERPASS_SIMPLIFY_TOLERANCE_DEG,
} from './constants.js';
/** Iterative Douglas-Peucker on [{lat,lon},...] (planar-degree approx — fine at
* the ~44 m tolerance used here). Endpoints always kept. */
@@ -51,7 +55,10 @@ function simplifyElementGeometry(el, minPoints, toleranceDeg) {
}
if (Array.isArray(el?.members)) {
for (const member of el.members) {
if (Array.isArray(member?.geometry) && member.geometry.length >= minPoints) {
if (
Array.isArray(member?.geometry) &&
member.geometry.length >= minPoints
) {
member.geometry = douglasPeucker(member.geometry, toleranceDeg);
}
}
@@ -73,7 +80,8 @@ function simplifyOverpassPayloadBody(bodyText, opts = {}) {
const minBytes = opts.minBytes ?? OVERPASS_SIMPLIFY_MIN_BYTES;
const minPoints = opts.minPoints ?? OVERPASS_SIMPLIFY_MIN_POINTS;
const toleranceDeg = opts.toleranceDeg ?? OVERPASS_SIMPLIFY_TOLERANCE_DEG;
if (typeof bodyText !== 'string' || bodyText.length < minBytes) return bodyText;
if (typeof bodyText !== 'string' || bodyText.length < minBytes)
return bodyText;
let data;
try {
data = JSON.parse(bodyText);
@@ -81,7 +89,8 @@ function simplifyOverpassPayloadBody(bodyText, opts = {}) {
return bodyText;
}
if (!Array.isArray(data?.elements)) return bodyText;
for (const el of data.elements) simplifyElementGeometry(el, minPoints, toleranceDeg);
for (const el of data.elements)
simplifyElementGeometry(el, minPoints, toleranceDeg);
try {
return JSON.stringify(data);
} catch {
+54 -17
View File
@@ -1,4 +1,11 @@
import { OVERPASS_MAX_AROUND_M, OVERPASS_MAX_BBOX_DEG, OVERPASS_AREA_ELEMENT_RE, OVERPASS_SELECTOR_RE, OVERPASS_BBOX_RE, OVERPASS_MAX_QL_TIMEOUT } from './constants.js';
import {
OVERPASS_MAX_AROUND_M,
OVERPASS_MAX_BBOX_DEG,
OVERPASS_AREA_ELEMENT_RE,
OVERPASS_SELECTOR_RE,
OVERPASS_BBOX_RE,
OVERPASS_MAX_QL_TIMEOUT,
} from './constants.js';
/**
* Whether a normalized Overpass query is BOUNDARY-class (admin `is_in` lookups
@@ -41,8 +48,14 @@ function stripOverpassNoise(src) {
const quote = c;
i += 1;
while (i < n) {
if (src[i] === '\\') { i += 2; continue; } // escaped char
if (src[i] === quote) { i += 1; break; } // closing quote
if (src[i] === '\\') {
i += 2;
continue;
} // escaped char
if (src[i] === quote) {
i += 1;
break;
} // closing quote
i += 1;
}
out += quote + quote; // collapse the literal to empty quotes
@@ -69,11 +82,17 @@ function stripOverpassNoise(src) {
function sanitizeOverpassBody(rawBody) {
let params;
try { params = new URLSearchParams(rawBody); } catch { return { ok: false, error: 'Malformed query body' }; }
try {
params = new URLSearchParams(rawBody);
} catch {
return { ok: false, error: 'Malformed query body' };
}
const all = params.getAll('data');
if (all.length !== 1) return { ok: false, error: 'Exactly one data query is required' };
if (all.length !== 1)
return { ok: false, error: 'Exactly one data query is required' };
const data = all[0];
if (!data || !data.trim()) return { ok: false, error: 'Missing Overpass data query' };
if (!data || !data.trim())
return { ok: false, error: 'Missing Overpass data query' };
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
@@ -89,9 +108,17 @@ function sanitizeOverpassBody(rawBody) {
}
}
// Reject world-sized / oversized bboxes.
for (const m of stripped.matchAll(/\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)/g)) {
const s = Number(m[1]); const w = Number(m[2]); const n = Number(m[3]); const e = Number(m[4]);
if (Math.abs(n - s) > OVERPASS_MAX_BBOX_DEG || Math.abs(e - w) > OVERPASS_MAX_BBOX_DEG) {
for (const m of stripped.matchAll(
/\(\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*,\s*(-?\d+(?:\.\d+)?)\s*\)/g,
)) {
const s = Number(m[1]);
const w = Number(m[2]);
const n = Number(m[3]);
const e = Number(m[4]);
if (
Math.abs(n - s) > OVERPASS_MAX_BBOX_DEG ||
Math.abs(e - w) > OVERPASS_MAX_BBOX_DEG
) {
return { ok: false, error: 'Overpass bbox too large' };
}
}
@@ -122,7 +149,10 @@ function sanitizeOverpassBody(rawBody) {
// misread as a spatial bound. Bounds live in (...) / function calls / set
// refs, never inside [...], so the probe loses nothing real.
const outSets = [];
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => { outSets.push(name); return ' '; });
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
outSets.push(name);
return ' ';
});
const probe = body.replace(/\[[^\]]*\]/g, ' ');
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
@@ -131,15 +161,21 @@ function sanitizeOverpassBody(rawBody) {
// SELECTS admin areas (area.set) and pivots (rel(pivot.x)), never node/way/
// relation(area...). The probe collapses tags so `way (area.a)` is caught.
if (OVERPASS_AREA_ELEMENT_RE.test(probe)) {
return { ok: false, error: 'Overpass area-bounded element selector not allowed' };
return {
ok: false,
error: 'Overpass area-bounded element selector not allowed',
};
}
const hasSelector = OVERPASS_SELECTOR_RE.test(probe);
const inputSets = [...probe.matchAll(/(?<!\d)\.([a-z_]\w*)/gi)].map((m) => m[1]);
const directBound = /around:\s*\d/.test(probe)
|| OVERPASS_BBOX_RE.test(probe)
|| /is_in\s*\(/.test(probe) // is_in(lat,lon) — the function form only
|| /\barea\s*\(/.test(probe); // area(id) — bounded as a set definition
const inputSets = [...probe.matchAll(/(?<!\d)\.([a-z_]\w*)/gi)].map(
(m) => m[1],
);
const directBound =
/around:\s*\d/.test(probe) ||
OVERPASS_BBOX_RE.test(probe) ||
/is_in\s*\(/.test(probe) || // is_in(lat,lon) — the function form only
/\barea\s*\(/.test(probe); // area(id) — bounded as a set definition
const setBound = inputSets.some((s) => boundedSets.has(s));
const bounded = directBound || setBound;
@@ -153,7 +189,8 @@ function sanitizeOverpassBody(rawBody) {
const clamped = data.replace(
/\[timeout:\s*(\d+)\s*\]/gi,
(_, n) => `[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
(_, n) =>
`[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
);
return { ok: true, body: `data=${encodeURIComponent(clamped)}` };
}
+40 -21
View File
@@ -1,4 +1,8 @@
import { OVERPASS_MAX_RESPONSE_BYTES, OVERPASS_UPSTREAMS, OVERPASS_TIMEOUT_MS } from './constants.js';
import {
OVERPASS_MAX_RESPONSE_BYTES,
OVERPASS_UPSTREAMS,
OVERPASS_TIMEOUT_MS,
} from './constants.js';
import { readResponseTextCapped } from '../common/http.js';
import { simplifyOverpassPayloadBody } from './geometry.js';
@@ -13,10 +17,12 @@ import { simplifyOverpassPayloadBody } from './geometry.js';
*/
function overpassLooksRateLimited(bodyText) {
const text = String(bodyText || '').toLowerCase();
return text.includes('rate_limited')
|| text.includes('quota of your ip address')
|| text.includes('dispatcher_client::request_read_and_idx::rate_limited')
|| text.includes('too many requests');
return (
text.includes('rate_limited') ||
text.includes('quota of your ip address') ||
text.includes('dispatcher_client::request_read_and_idx::rate_limited') ||
text.includes('too many requests')
);
}
/**
@@ -26,9 +32,11 @@ function overpassLooksRateLimited(bodyText) {
*/
function overpassLooksRuntimeError(bodyText) {
const text = String(bodyText || '').toLowerCase();
return text.includes('runtime error')
|| text.includes('timed out')
|| text.includes('out of memory');
return (
text.includes('runtime error') ||
text.includes('timed out') ||
text.includes('out of memory')
);
}
/**
@@ -42,10 +50,13 @@ function overpassLooksRuntimeError(bodyText) {
*/
function overpassPayloadIsData(payload) {
const status = Number(payload?.status);
return Number.isFinite(status)
&& status >= 200 && status < 300
&& !payload.rateLimited
&& !payload.runtimeError;
return (
Number.isFinite(status) &&
status >= 200 &&
status < 300 &&
!payload.rateLimited &&
!payload.runtimeError
);
}
/**
@@ -57,12 +68,16 @@ function overpassPayloadIsData(payload) {
* @param {object} [options] Server-only endpoint and I/O overrides for tests.
* @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
*/
async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES, {
endpoints = OVERPASS_UPSTREAMS,
fetchImpl = fetch,
readBody = readResponseTextCapped,
simplify = simplifyOverpassPayloadBody,
} = {}) {
async function fetchOverpassPayload(
body,
maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES,
{
endpoints = OVERPASS_UPSTREAMS,
fetchImpl = fetch,
readBody = readResponseTextCapped,
simplify = simplifyOverpassPayloadBody,
} = {},
) {
let lastError = null;
let lastRateLimitPayload = null;
let lastRefusalPayload = null;
@@ -83,9 +98,11 @@ async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPON
});
const responseBody = await readBody(upstream, maxResponseBytes);
const contentType = upstream.headers.get('content-type') || 'application/json';
const contentType =
upstream.headers.get('content-type') || 'application/json';
const status = upstream.status;
const rateLimited = status === 429 || overpassLooksRateLimited(responseBody);
const rateLimited =
status === 429 || overpassLooksRateLimited(responseBody);
const runtimeError = overpassLooksRuntimeError(responseBody);
const payload = {
status,
@@ -116,7 +133,9 @@ async function fetchOverpassPayload(body, maxResponseBytes = OVERPASS_MAX_RESPON
// every mirror has had the chance to answer it.
if (status < 200 || status >= 300) {
if (!lastRefusalPayload) lastRefusalPayload = payload;
lastError = new Error(`Overpass upstream returned ${status} (${endpoint})`);
lastError = new Error(
`Overpass upstream returned ${status} (${endpoint})`,
);
continue;
}