mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
505 lines
16 KiB
JavaScript
505 lines
16 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { readResponseJsonCapped } from '../common/http.js';
|
|
|
|
/**
|
|
* One gate for every request this server sends to a Nominatim instance.
|
|
*
|
|
* Place search and voice outlines share it, so the public instance sees one
|
|
* application: one request at a time, at least 1.1 s apart, a short queue, a
|
|
* daily ceiling per install, and a pause after the server says to stop.
|
|
*/
|
|
|
|
/** The public instance. Only this host is subject to the public-use limits. */
|
|
export const PUBLIC_NOMINATIM_SEARCH =
|
|
'https://nominatim.openstreetmap.org/search';
|
|
|
|
const PUBLIC_NOMINATIM_HOST = 'nominatim.openstreetmap.org';
|
|
|
|
/**
|
|
* The public usage policy asks for a User-Agent or Referer that identifies the
|
|
* application. The identity is stable: it is never rotated or varied.
|
|
*/
|
|
export const NOMINATIM_HEADERS = Object.freeze({
|
|
'User-Agent':
|
|
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
|
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
|
|
});
|
|
|
|
/** Absolute policy maximum is one request per second; 1.1 s absorbs jitter. */
|
|
export const NOMINATIM_MIN_SPACING_MS = 1100;
|
|
|
|
/** Requests allowed to wait for their turn; past this, refuse at once. */
|
|
export const NOMINATIM_MAX_PENDING = 4;
|
|
|
|
/** A queued request older than this has no reader left; drop it unsent. */
|
|
export const NOMINATIM_MAX_WAIT_MS = 10_000;
|
|
|
|
/** Default public requests per install per UTC day, outlines and search together. */
|
|
export const NOMINATIM_DEFAULT_DAILY_CAP = 50;
|
|
|
|
/**
|
|
* Pause after a refusal without Retry-After. A Retry-After from the server is
|
|
* honoured in full, however long: no local maximum may cause an earlier retry.
|
|
*/
|
|
const DEFAULT_BACKOFF_MS = 60_000;
|
|
/** A 403 from the public instance means this identity is blocked: stop longer. */
|
|
const BLOCKED_BACKOFF_MS = 3_600_000;
|
|
/** Pause after a server error or a failed retry. */
|
|
const OUTAGE_BACKOFF_MS = 30_000;
|
|
|
|
const DISABLED_VALUES = new Set(['', 'off', 'none', 'disabled', '0']);
|
|
|
|
/**
|
|
* Read the search endpoint setting.
|
|
*
|
|
* Unset means the public instance under its guards. Set to empty (or `off`)
|
|
* disables every Nominatim request. Any other value must be an http(s) URL
|
|
* without credentials, query or fragment; a bare host gets `/search`. A value
|
|
* that is not usable disables the service rather than falling back to the
|
|
* public instance.
|
|
*
|
|
* @param {Record<string, string|undefined>} [env]
|
|
* @returns {{endpoint: string|null, isPublic: boolean, dailyCap: number, invalid?: boolean}}
|
|
*/
|
|
export function resolveNominatimSettings(env = process.env) {
|
|
const dailyCap = parseDailyCap(env.NOMINATIM_DAILY_CAP);
|
|
const raw = env.NOMINATIM_URL;
|
|
if (raw === undefined)
|
|
return { endpoint: PUBLIC_NOMINATIM_SEARCH, isPublic: true, dailyCap };
|
|
const value = String(raw).trim();
|
|
if (DISABLED_VALUES.has(value.toLowerCase()))
|
|
return { endpoint: null, isPublic: false, dailyCap };
|
|
let url;
|
|
try {
|
|
url = new URL(value);
|
|
} catch {
|
|
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
|
}
|
|
if (
|
|
!['http:', 'https:'].includes(url.protocol) ||
|
|
!url.hostname ||
|
|
url.username ||
|
|
url.password ||
|
|
url.search ||
|
|
url.hash
|
|
)
|
|
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
|
if (url.pathname === '/' || url.pathname === '') url.pathname = '/search';
|
|
return {
|
|
endpoint: url.href.replace(/\/$/, ''),
|
|
// A trailing DNS dot names the same host.
|
|
isPublic:
|
|
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST,
|
|
dailyCap,
|
|
};
|
|
}
|
|
|
|
function parseDailyCap(raw) {
|
|
if (raw === undefined || String(raw).trim() === '')
|
|
return NOMINATIM_DEFAULT_DAILY_CAP;
|
|
const value = Number(raw);
|
|
return Number.isFinite(value) && value >= 0
|
|
? Math.floor(value)
|
|
: NOMINATIM_DEFAULT_DAILY_CAP;
|
|
}
|
|
|
|
function gateError(code, message, extra = {}) {
|
|
return Object.assign(new Error(message), { code, ...extra });
|
|
}
|
|
|
|
/** Seconds or an HTTP date → milliseconds from now, or null. */
|
|
export function parseRetryAfter(value, now = Date.now()) {
|
|
if (value == null || value === '') return null;
|
|
const seconds = Number(value);
|
|
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
|
|
const at = Date.parse(String(value));
|
|
return Number.isFinite(at) ? Math.max(0, at - now) : null;
|
|
}
|
|
|
|
/** UTC calendar day, the unit of the daily cap. */
|
|
function utcDay(ms) {
|
|
return new Date(ms).toISOString().slice(0, 10);
|
|
}
|
|
|
|
const sleepSync = (ms) =>
|
|
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
|
|
/**
|
|
* Public-service state shared by every process using the same file: today's
|
|
* request count and any pause the server asked for.
|
|
*
|
|
* Every change is a read-modify-write under an exclusive lock directory, and
|
|
* the file is replaced atomically, so two dev servers sharing a checkout add
|
|
* to one count instead of overwriting each other. Memory keeps the highest
|
|
* values this process has seen, so a failed or unreadable file never lowers
|
|
* the count or shortens a pause; a persistence failure is reported once and
|
|
* in `status()`.
|
|
*/
|
|
export function createGateStateStore({
|
|
file = null,
|
|
lockStaleMs = 5000,
|
|
lockWaitMs = 500,
|
|
onError = (error) =>
|
|
console.warn(
|
|
`[Nominatim] could not persist usage state: ${error?.message || error}`,
|
|
),
|
|
} = {}) {
|
|
let memory = { day: '', count: 0, pausedUntil: 0 };
|
|
let lastError = null;
|
|
|
|
const report = (error) => {
|
|
if (!lastError) onError(error);
|
|
lastError = error;
|
|
};
|
|
|
|
const merge = (a, b) => ({
|
|
day: a.day >= b.day ? a.day : b.day,
|
|
count:
|
|
a.day === b.day
|
|
? Math.max(a.count, b.count)
|
|
: a.day > b.day
|
|
? a.count
|
|
: b.count,
|
|
pausedUntil: Math.max(a.pausedUntil || 0, b.pausedUntil || 0),
|
|
});
|
|
|
|
function readDisk() {
|
|
if (!file) return null;
|
|
try {
|
|
const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
return {
|
|
day: typeof parsed?.day === 'string' ? parsed.day : '',
|
|
count: Number.isFinite(parsed?.count) ? Math.max(0, parsed.count) : 0,
|
|
pausedUntil: Number.isFinite(parsed?.pausedUntil)
|
|
? parsed.pausedUntil
|
|
: 0,
|
|
};
|
|
} catch (error) {
|
|
if (error?.code !== 'ENOENT') report(error);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function current() {
|
|
const disk = readDisk();
|
|
if (disk) memory = merge(memory, disk);
|
|
return memory;
|
|
}
|
|
|
|
function withLock(work) {
|
|
const lock = `${file}.lock`;
|
|
const deadline = Date.now() + lockWaitMs;
|
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
for (;;) {
|
|
try {
|
|
fs.mkdirSync(lock);
|
|
break;
|
|
} catch (error) {
|
|
if (error?.code !== 'EEXIST') throw error;
|
|
try {
|
|
if (Date.now() - fs.statSync(lock).mtimeMs > lockStaleMs)
|
|
fs.rmSync(lock, { recursive: true, force: true });
|
|
} catch {
|
|
// Another process released it meanwhile.
|
|
}
|
|
if (Date.now() > deadline)
|
|
throw new Error('usage state is locked by another process');
|
|
sleepSync(5);
|
|
}
|
|
}
|
|
try {
|
|
return work();
|
|
} finally {
|
|
fs.rmSync(lock, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
/** Apply `change` to the freshest state and persist it atomically. */
|
|
function update(change) {
|
|
if (!file) {
|
|
memory = merge(memory, change(memory));
|
|
return memory;
|
|
}
|
|
try {
|
|
withLock(() => {
|
|
const next = merge(memory, change(current()));
|
|
const temp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
|
fs.writeFileSync(temp, JSON.stringify(next));
|
|
fs.renameSync(temp, file);
|
|
memory = next;
|
|
lastError = null;
|
|
});
|
|
} catch (error) {
|
|
report(error);
|
|
memory = merge(memory, change(memory));
|
|
}
|
|
return memory;
|
|
}
|
|
|
|
return {
|
|
count(day) {
|
|
const state = current();
|
|
return state.day === day ? state.count : 0;
|
|
},
|
|
increment(day) {
|
|
return update((state) => ({
|
|
day,
|
|
count: state.day === day ? state.count + 1 : 1,
|
|
pausedUntil: state.pausedUntil,
|
|
})).count;
|
|
},
|
|
pausedUntil() {
|
|
return current().pausedUntil || 0;
|
|
},
|
|
pauseUntil(at) {
|
|
update((state) => ({ ...state, pausedUntil: at }));
|
|
},
|
|
status: () => ({
|
|
file,
|
|
persisted: Boolean(file) && !lastError,
|
|
error: lastError ? String(lastError.message || lastError) : null,
|
|
}),
|
|
};
|
|
}
|
|
|
|
/** Compatibility name for the state store. */
|
|
export const createUsageStore = createGateStateStore;
|
|
|
|
/**
|
|
* Construct a gate. `settings` may be a function so the environment is read
|
|
* after the dev server loads `.env`.
|
|
*
|
|
* @param {object} [options]
|
|
* @param {(() => object)|object} [options.settings]
|
|
* @param {typeof fetch} [options.fetchImpl]
|
|
* @param {() => number} [options.now]
|
|
* @param {(ms: number) => Promise<void>} [options.sleep]
|
|
* @param {ReturnType<typeof createUsageStore>} [options.usage]
|
|
*/
|
|
export function createNominatimGate({
|
|
settings = () => resolveNominatimSettings(),
|
|
fetchImpl = (...args) => globalThis.fetch(...args),
|
|
now = Date.now,
|
|
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
usage = createGateStateStore(),
|
|
minSpacingMs = NOMINATIM_MIN_SPACING_MS,
|
|
maxPending = NOMINATIM_MAX_PENDING,
|
|
maxWaitMs = NOMINATIM_MAX_WAIT_MS,
|
|
timeoutMs = 9000,
|
|
} = {}) {
|
|
let queue = Promise.resolve();
|
|
let lastStartedAt = -Infinity;
|
|
let pending = 0;
|
|
let pausedUntil = 0;
|
|
let upstreamRequests = 0;
|
|
let usageStore = usage;
|
|
|
|
const current = () =>
|
|
typeof settings === 'function' ? settings() : settings;
|
|
|
|
/**
|
|
* Pause until `ms` from now. The public service's pause is also written to
|
|
* the shared state so a restart or a second process honours it.
|
|
*/
|
|
function pause(ms, config) {
|
|
const until = now() + Math.max(0, ms);
|
|
if (until > pausedUntil) pausedUntil = until;
|
|
if (config?.isPublic) usageStore.pauseUntil(pausedUntil);
|
|
}
|
|
|
|
function pauseEnd(config) {
|
|
return config?.isPublic
|
|
? Math.max(pausedUntil, usageStore.pausedUntil())
|
|
: pausedUntil;
|
|
}
|
|
|
|
function pausedError(config) {
|
|
return gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
|
|
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
|
});
|
|
}
|
|
|
|
function admissionError(config) {
|
|
if (!config.endpoint)
|
|
return gateError('NOMINATIM_DISABLED', 'Place service is not configured');
|
|
if (now() < pauseEnd(config)) return pausedError(config);
|
|
if (config.isPublic && usageStore.count(utcDay(now())) >= config.dailyCap)
|
|
return gateError(
|
|
'NOMINATIM_DAILY_CAP',
|
|
'Daily place lookup allowance is used',
|
|
);
|
|
return null;
|
|
}
|
|
|
|
/** One upstream request; classifies refusals and pauses the gate. */
|
|
async function send(url, config, maxBytes, signal) {
|
|
if (config.isPublic) usageStore.increment(utcDay(now()));
|
|
upstreamRequests += 1;
|
|
lastStartedAt = now();
|
|
const controller = new AbortController();
|
|
const onAbort = () => controller.abort();
|
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
// One deadline covers the headers AND the body: a server that answers its
|
|
// headers and then stalls must not hold the shared queue.
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
return await exchange(url, config, maxBytes, signal, controller.signal);
|
|
} finally {
|
|
clearTimeout(timer);
|
|
signal?.removeEventListener('abort', onAbort);
|
|
}
|
|
}
|
|
|
|
async function exchange(url, config, maxBytes, signal, requestSignal) {
|
|
let response;
|
|
try {
|
|
response = await fetchImpl(url, {
|
|
headers: NOMINATIM_HEADERS,
|
|
redirect: 'error',
|
|
signal: requestSignal,
|
|
});
|
|
} catch (cause) {
|
|
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
|
throw gateError('NOMINATIM_TRANSPORT', 'Place service unreachable', {
|
|
cause,
|
|
transport: true,
|
|
});
|
|
}
|
|
if (!response.ok) {
|
|
void response.body?.cancel?.().catch(() => {});
|
|
const status = response.status;
|
|
const retryAfter = parseRetryAfter(
|
|
response.headers?.get?.('retry-after'),
|
|
now(),
|
|
);
|
|
if (status === 429 || status === 503) {
|
|
pause(retryAfter ?? DEFAULT_BACKOFF_MS, config);
|
|
throw gateError('NOMINATIM_REFUSED', 'Place service asked to wait', {
|
|
status,
|
|
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
|
});
|
|
}
|
|
if (status === 403 || status === 418) {
|
|
pause(retryAfter ?? BLOCKED_BACKOFF_MS, config);
|
|
throw gateError('NOMINATIM_REFUSED', 'Place service refused', {
|
|
status,
|
|
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
|
});
|
|
}
|
|
if (status >= 500) pause(retryAfter ?? OUTAGE_BACKOFF_MS, config);
|
|
throw gateError('NOMINATIM_UPSTREAM', `Upstream returned ${status}`, {
|
|
status,
|
|
});
|
|
}
|
|
try {
|
|
return await readResponseJsonCapped(response, maxBytes, requestSignal);
|
|
} catch (cause) {
|
|
if (cause?.code === 'RESPONSE_TOO_LARGE')
|
|
throw gateError('NOMINATIM_TOO_LARGE', 'Place answer too large');
|
|
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
|
throw gateError('NOMINATIM_TRANSPORT', 'Place answer unreadable', {
|
|
cause,
|
|
transport: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
/** Wait for the spacing slot, then check the request is still wanted. */
|
|
async function takeTurn(config, queuedAt, signal) {
|
|
const spacing = config.isPublic ? minSpacingMs : 0;
|
|
const waitMs = Math.max(0, spacing - (now() - lastStartedAt));
|
|
if (waitMs) await sleep(waitMs);
|
|
if (signal?.aborted || now() - queuedAt > maxWaitMs)
|
|
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
|
|
// An outage or refusal that happened while this waited stops it here.
|
|
const refused = admissionError(config);
|
|
if (refused) throw refused;
|
|
}
|
|
|
|
/**
|
|
* Queue one JSON request for `buildUrl(endpoint)`. At most one retry, and
|
|
* only after a transport failure; a failed retry pauses the gate.
|
|
*
|
|
* @param {(endpoint: string) => string} buildUrl
|
|
* @param {{signal?: AbortSignal, maxBytes?: number}} [options]
|
|
*/
|
|
function requestJson(buildUrl, { signal, maxBytes = 2 * 1024 * 1024 } = {}) {
|
|
const config = current();
|
|
const refused = admissionError(config);
|
|
if (refused) return Promise.reject(refused);
|
|
if (signal?.aborted)
|
|
return Promise.reject(
|
|
gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'),
|
|
);
|
|
if (pending >= maxPending)
|
|
return Promise.reject(
|
|
gateError('NOMINATIM_QUEUE_FULL', 'Place search queue is full'),
|
|
);
|
|
pending += 1;
|
|
const queuedAt = now();
|
|
const url = buildUrl(config.endpoint);
|
|
const task = queue.then(async () => {
|
|
try {
|
|
await takeTurn(config, queuedAt, signal);
|
|
try {
|
|
return await send(url, config, maxBytes, signal);
|
|
} catch (error) {
|
|
if (!error?.transport) throw error;
|
|
await takeTurn(config, now(), signal);
|
|
try {
|
|
return await send(url, config, maxBytes, signal);
|
|
} catch (retryError) {
|
|
if (retryError?.transport) pause(OUTAGE_BACKOFF_MS, config);
|
|
throw retryError;
|
|
}
|
|
}
|
|
} finally {
|
|
pending -= 1;
|
|
}
|
|
});
|
|
queue = task.catch(() => null);
|
|
return task;
|
|
}
|
|
|
|
return {
|
|
requestJson,
|
|
settings: current,
|
|
/** Share the daily count and pause through a file (composition chooses it). */
|
|
persistUsage(file) {
|
|
usageStore = createGateStateStore({ file });
|
|
},
|
|
stats: () => ({
|
|
upstreamRequests,
|
|
pending,
|
|
pausedForMs: Math.max(0, pauseEnd(current()) - now()),
|
|
usedToday: usageStore.count(utcDay(now())),
|
|
persistence: usageStore.status?.() || null,
|
|
}),
|
|
};
|
|
}
|
|
|
|
let sharedGate = createNominatimGate();
|
|
|
|
/** The process-wide gate: search and outlines share one budget and pacer. */
|
|
export function sharedNominatimGate() {
|
|
return sharedGate;
|
|
}
|
|
|
|
/** Replace the process-wide gate (tests, embedders with their own transport). */
|
|
export function setSharedNominatimGate(gate) {
|
|
sharedGate = gate || createNominatimGate();
|
|
return sharedGate;
|
|
}
|
|
|
|
/** Codes that mean "not now" rather than "no such place". */
|
|
export function isNominatimBusy(error) {
|
|
return [
|
|
'NOMINATIM_QUEUE_FULL',
|
|
'NOMINATIM_ABANDONED',
|
|
'NOMINATIM_PAUSED',
|
|
'NOMINATIM_REFUSED',
|
|
'NOMINATIM_DAILY_CAP',
|
|
].includes(error?.code);
|
|
}
|