mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Merge branch 'main' into codex/overpass-admission-review-fixes
This commit is contained in:
1407 files changed
+118138
-22562
No files matched your search
@@ -28,6 +28,7 @@ import {
|
||||
loadNycSourcesFromOpenData,
|
||||
loadTorontoSourcesFromOpenData,
|
||||
loadFloridaSourcesFromOpenData,
|
||||
loadA22SourcesFromOpenDataHub,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
@@ -165,6 +166,11 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
|
||||
load: loadTaiwanSources,
|
||||
},
|
||||
{
|
||||
name: 'a22',
|
||||
enabled: () => envEnabled('CCTV_A22_ENABLED'),
|
||||
load: loadA22SourcesFromOpenDataHub,
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
|
||||
|
||||
@@ -443,6 +443,39 @@ export const NYC_ANCHORS = [
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
|
||||
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
|
||||
* own camera list with coordinates, one request for the whole motorway
|
||||
* (13 cameras). Frames are stills on the operator's host, refreshed about
|
||||
* once a minute. */
|
||||
export const DEFAULT_A22_CCTV_URL =
|
||||
'https://tourism.api.opendatahub.com/v1/WebcamInfo?source=a22&pagesize=200&removenullvalues=true';
|
||||
/** The only origin A22 camera frames may come from: `km<N>.jpg` per camera. */
|
||||
export const A22_IMAGE_ORIGIN = 'https://www.autobrennero.it/WebCamImg/';
|
||||
/** Hard ceiling on the catalog body. The whole motorway is ~45 KB. */
|
||||
export const A22_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
|
||||
/** Orthometric ground elevation (m) at each camera, keyed by its motorway
|
||||
* kilometre, from the Copernicus GLO-90 DEM via Open-Meteo's elevation API.
|
||||
* The feed's coordinates are 2D and the road climbs from the Po valley (~20 m)
|
||||
* to the Brenner Pass (~1,370 m), so one flat prior would bury the Alpine
|
||||
* cameras. A camera not listed here uses A22_DEFAULT_GROUND_ELEVATION_M. */
|
||||
export const A22_GROUND_ELEVATION_BY_KM = Object.freeze({
|
||||
1: 1365,
|
||||
68: 375,
|
||||
77: 301,
|
||||
100: 219,
|
||||
129: 206,
|
||||
138: 204,
|
||||
159: 175,
|
||||
205: 188,
|
||||
228: 72,
|
||||
240: 38,
|
||||
272: 16,
|
||||
285: 19,
|
||||
313: 40,
|
||||
});
|
||||
export const A22_DEFAULT_GROUND_ELEVATION_M = 200;
|
||||
|
||||
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
|
||||
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
|
||||
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Readable } from 'node:stream';
|
||||
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
|
||||
import {
|
||||
A22_IMAGE_ORIGIN,
|
||||
CALGARY_IMAGE_ORIGIN,
|
||||
CCTV_FRAME_FETCH_TIMEOUT_MS,
|
||||
CCTV_FRAME_MAX_BODY_BYTES,
|
||||
@@ -528,6 +529,7 @@ for (const prefix of [
|
||||
HK_IMAGE_ORIGIN,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
A22_IMAGE_ORIGIN,
|
||||
]) {
|
||||
const host = new URL(prefix).hostname;
|
||||
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
|
||||
|
||||
@@ -421,6 +421,17 @@ export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** The A22 corridor, Brenner Pass to Modena, with slack either side. */
|
||||
export function isLikelyA22Coordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 44.5 &&
|
||||
lat <= 47.1 &&
|
||||
lon >= 10.5 &&
|
||||
lon <= 11.8
|
||||
);
|
||||
}
|
||||
|
||||
export function isLikelyFinlandCoordinate(lat, lon) {
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
|
||||
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
|
||||
|
||||
@@ -122,6 +122,11 @@ import {
|
||||
NYC_MAX_CATALOG_BYTES,
|
||||
NYC_GROUND_ELEVATION_M,
|
||||
NYC_ANCHORS,
|
||||
DEFAULT_A22_CCTV_URL,
|
||||
A22_IMAGE_ORIGIN,
|
||||
A22_MAX_CATALOG_BYTES,
|
||||
A22_GROUND_ELEVATION_BY_KM,
|
||||
A22_DEFAULT_GROUND_ELEVATION_M,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -145,6 +150,7 @@ import {
|
||||
isLikelyHongKongCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
isLikelyTorontoCoordinate,
|
||||
isLikelyA22Coordinate,
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
prioritizeSources,
|
||||
@@ -3645,3 +3651,123 @@ export async function loadFloridaSourcesFromOpenData() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One Open Data Hub `WebcamInfo` item (source `a22`) -> one catalog source,
|
||||
* or null.
|
||||
*
|
||||
* The frame must be the operator's own `WebCamImg/km<N>.jpg` for the
|
||||
* camera's motorway kilometre. The feed names the place and the kilometre but
|
||||
* no bearing, so the heading uses the shared id-hash fallback at low
|
||||
* confidence, like Calgary and Vegvesen; the ground elevation comes from the
|
||||
* per-kilometre DEM table because the road spans ~1,350 m of altitude.
|
||||
*
|
||||
* @param {object} item - One element of the API's `Items` array.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function a22WebcamToSource(item) {
|
||||
if (!item || typeof item !== 'object') return null;
|
||||
if (item.Active === false) return null;
|
||||
if (String(item.Source ?? '').toLowerCase() !== 'a22') return null;
|
||||
|
||||
const gps = Array.isArray(item.GpsInfo) ? item.GpsInfo[0] : null;
|
||||
const lat = toFiniteNumber(gps?.Latitude);
|
||||
const lon = toFiniteNumber(gps?.Longitude);
|
||||
if (!isLikelyA22Coordinate(lat, lon)) return null;
|
||||
|
||||
const km = String(item.Mapping?.a22?.km ?? '').trim();
|
||||
if (!/^\d{1,3}$/.test(km)) return null;
|
||||
const imageUrl = `${A22_IMAGE_ORIGIN}km${Number(km)}.jpg`;
|
||||
let image;
|
||||
try {
|
||||
image = new URL(String(item.Webcamurl ?? ''));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (image.username || image.password || image.href !== imageUrl) return null;
|
||||
|
||||
const place =
|
||||
String(item.Shortname ?? item.Webcamname?.it ?? '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim() || `Km ${Number(km)}`;
|
||||
const cameraId = `it-a22-km${Number(km)}`;
|
||||
const ground = A22_GROUND_ELEVATION_BY_KM[Number(km)];
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name: `A22 ${place} (km ${Number(km)})`,
|
||||
// One motorway-wide category in the camera picker; the place is already
|
||||
// in the name.
|
||||
city: 'A22 Brennero',
|
||||
cityId: 'a22',
|
||||
provider: 'Autostrada del Brennero',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -15,
|
||||
fovDeg: 50,
|
||||
rangeM: 160,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: Number.isFinite(ground)
|
||||
? ground
|
||||
: A22_DEFAULT_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'a22-opendatahub',
|
||||
license:
|
||||
'Webcam images © Autostrada del Brennero S.p.A., via the Open Data Hub (NOI Techpark)',
|
||||
code: cameraDisplayCode(place.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the Autostrada del Brennero (A22) webcams from the keyless Open Data
|
||||
* Hub tourism API. Frames are stills on www.autobrennero.it.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadA22SourcesFromOpenDataHub() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_A22_URL || DEFAULT_A22_CCTV_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] A22 catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] A22 camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(resp, A22_MAX_CATALOG_BYTES);
|
||||
const items = Array.isArray(payload?.Items) ? payload.Items : [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const item of items) {
|
||||
const camera = a22WebcamToSource(item);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
// North to south along the motorway, so the picker reads in road order.
|
||||
cameras.sort((a, b) => b.lat - a.lat);
|
||||
console.log(`[CCTV] Loaded A22 camera sources: ${cameras.length}`);
|
||||
return cameras;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] A22 camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,8 @@ const API_URL =
|
||||
// The service caps a single response at its maxRecordCount (2000); a peak
|
||||
// season can exceed that, so follow exceededTransferLimit with offset pages.
|
||||
const MAX_PAGES = 5;
|
||||
/** Longest a failed refresh keeps serving the last perimeter snapshot, marked stale. */
|
||||
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const exceededTransferLimit = (payload) =>
|
||||
payload?.exceededTransferLimit === true ||
|
||||
@@ -165,7 +167,14 @@ export function firePerimetersProxy({
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
if (previous) return { value: previous.value, stale: true };
|
||||
// A failed refresh serves the last answer, marked stale. A perimeter
|
||||
// snapshot only for a day: older than that, it is no picture of
|
||||
// current fires. (InciWeb entries keep their previous behaviour.)
|
||||
if (
|
||||
previous &&
|
||||
(key !== 'perimeters' || now() - previous.savedAt <= STALE_MAX_MS)
|
||||
)
|
||||
return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
+138
-6
@@ -1,9 +1,132 @@
|
||||
import path from 'node:path';
|
||||
import https from 'node:https';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { Readable } from 'node:stream';
|
||||
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
|
||||
|
||||
/**
|
||||
* The per-address connection errors behind a fetch failure, or null. Node 24
|
||||
* puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in
|
||||
* a ConnectTimeoutError one level deeper.
|
||||
* @param {unknown} err
|
||||
* @returns {?Array<unknown>}
|
||||
*/
|
||||
function connectAttemptErrors(err) {
|
||||
let cause = err?.cause;
|
||||
for (let depth = 0; cause && depth < 3; depth++) {
|
||||
if (Array.isArray(cause.errors)) return cause.errors;
|
||||
cause = cause.cause;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Describe a fetch failure with its socket-level cause. Node reports every
|
||||
* connection failure as `fetch failed`; the cause carries the error code and
|
||||
* the address that failed (never the URL, which embeds the MAP_KEY).
|
||||
* @param {unknown} err
|
||||
* @returns {string}
|
||||
*/
|
||||
export function describeFetchError(err) {
|
||||
const message = err?.message || String(err);
|
||||
const cause = err?.cause;
|
||||
if (!cause) return message;
|
||||
const attempts = connectAttemptErrors(err);
|
||||
const parts = attempts?.length
|
||||
? attempts.map((inner) => inner?.message || inner?.code || String(inner))
|
||||
: [cause.message || cause.code || String(cause)];
|
||||
return `${message} (${parts.join('; ')})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a fetch failed because every address it tried failed to connect.
|
||||
* Node's network-family auto-selection gives each address a short attempt
|
||||
* timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable
|
||||
* IPv6 address and the whole request fails (#68).
|
||||
* @param {unknown} err
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isAddressSelectionFailure(err) {
|
||||
return err instanceof TypeError && connectAttemptErrors(err)?.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET over IPv4 only, with one connection attempt bounded by the caller's
|
||||
* signal rather than by the auto-selection attempt timeout.
|
||||
* @param {string} url
|
||||
* @param {{signal?: AbortSignal}} [options]
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
export function fetchIpv4(url, { signal } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.request(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
signal,
|
||||
family: 4,
|
||||
autoSelectFamily: false,
|
||||
agent: false,
|
||||
},
|
||||
(response) => {
|
||||
// Nothing thrown here may escape: this callback runs outside the
|
||||
// promise, so an exception would be uncaught and crash the server.
|
||||
try {
|
||||
const status = response.statusCode || 500;
|
||||
// A `Response` only represents statuses up to 599; Node never hands
|
||||
// this callback a 1xx.
|
||||
if (status > 599)
|
||||
throw new Error(`FIRMS returned unsupported status ${status}`);
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
}
|
||||
const nullBody = NULL_BODY_STATUSES.has(status);
|
||||
const upstream = new Response(
|
||||
nullBody ? null : Readable.toWeb(response),
|
||||
{ status, statusText: response.statusMessage || '', headers },
|
||||
);
|
||||
if (nullBody) response.resume();
|
||||
resolve(upstream);
|
||||
} catch (error) {
|
||||
response.destroy();
|
||||
reject(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a FIRMS URL, retrying once over IPv4 when address selection fails.
|
||||
* Other failures (HTTP status, timeout, DNS) are returned or thrown as is.
|
||||
* @param {string} url
|
||||
* @param {{signal?: AbortSignal}} options
|
||||
* @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps]
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
export async function fetchFirms(
|
||||
url,
|
||||
options,
|
||||
{ fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {},
|
||||
) {
|
||||
try {
|
||||
return await fetchImpl(url, options);
|
||||
} catch (err) {
|
||||
if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err;
|
||||
warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`);
|
||||
return fetchIpv4Impl(url, options);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
|
||||
@@ -91,8 +214,12 @@ export function firmsProxy() {
|
||||
async function fetchSource(key, source) {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||
const signal = AbortSignal.timeout(60_000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const res = await fetchFirms(url, { signal });
|
||||
if (!res.ok) {
|
||||
// Release the connection now rather than when the deadline fires.
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
const records = parseFirmsCsv(
|
||||
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
|
||||
);
|
||||
@@ -121,7 +248,7 @@ export function firmsProxy() {
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
`[firms-proxy] ${source} fetch failed:`,
|
||||
err?.message || err,
|
||||
describeFetchError(err),
|
||||
);
|
||||
sources.push({ source, count: 0, ok: false });
|
||||
}
|
||||
@@ -156,8 +283,13 @@ export function firmsProxy() {
|
||||
statusInflight = (async () => {
|
||||
try {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const res = await fetchFirms(url, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
await res.body?.cancel().catch(() => {});
|
||||
throw new Error(`HTTP ${res.status}`);
|
||||
}
|
||||
const body = await res.json();
|
||||
const used = Number(body?.current_transactions);
|
||||
const limit = Number(body?.transaction_limit);
|
||||
@@ -167,7 +299,7 @@ export function firmsProxy() {
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[firms-proxy] mapkey status failed:',
|
||||
err?.message || err,
|
||||
describeFetchError(err),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
GHOSTMAPS_MAX_FILE_BYTES,
|
||||
GHOSTMAPS_REPO,
|
||||
buildGhostMapsCatalog,
|
||||
ghostMapsRawUrl,
|
||||
parseGitRefAdvertisement,
|
||||
} from '../../src/layers/ghostMaps/catalog.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from './common/http.js';
|
||||
|
||||
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
|
||||
// The repository is not ours, so there is no webhook: the watcher reads the
|
||||
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
|
||||
// REST API rate limit) every few seconds, and only a new commit costs REST
|
||||
// calls (tree + commit). Browsers hear about a new commit over server-sent
|
||||
// events and fetch only the products whose blob changed. Files are fetched by
|
||||
// commit-pinned raw URLs and kept on disk by blob sha.
|
||||
|
||||
const DEFAULT_POLL_MS = 15_000;
|
||||
const MIN_POLL_MS = 5_000;
|
||||
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
|
||||
const TIMEOUT_MS = 15_000;
|
||||
const FILE_TIMEOUT_MS = 120_000;
|
||||
const HEARTBEAT_MS = 25_000;
|
||||
const MAX_SUBSCRIBERS = 64;
|
||||
const MAX_REF_BYTES = 64 * 1024;
|
||||
const MAX_TREE_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
|
||||
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
|
||||
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
|
||||
|
||||
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
|
||||
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
|
||||
const seconds = Number(raw);
|
||||
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
|
||||
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* The repository watcher and file cache, independent of HTTP routing.
|
||||
* @returns {{start: Function, stop: Function, catalog: Function,
|
||||
* checkNow: Function, file: Function, subscribe: Function}}
|
||||
*/
|
||||
export function createGhostMapsWatcher({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
pollMs = ghostMapsPollMs(),
|
||||
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
|
||||
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
|
||||
log = (...args) => console.log(...args),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
let state = {
|
||||
commit: null,
|
||||
committedAt: null,
|
||||
message: null,
|
||||
products: [],
|
||||
checkedAt: null,
|
||||
changedAt: null,
|
||||
error: null,
|
||||
};
|
||||
let timer = null;
|
||||
let running = false;
|
||||
let checking = null;
|
||||
let failures = 0;
|
||||
let diskLoaded = false;
|
||||
const listeners = new Set();
|
||||
const downloads = new Map();
|
||||
|
||||
const apiHeaders = () => ({
|
||||
Accept: 'application/vnd.github+json',
|
||||
'User-Agent': USER_AGENT,
|
||||
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
|
||||
});
|
||||
|
||||
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
|
||||
const response = await fetchImpl(url, {
|
||||
headers,
|
||||
redirect: 'follow',
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
|
||||
status: response.status,
|
||||
});
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
async function readHead() {
|
||||
const response = await get(
|
||||
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
);
|
||||
const sha = parseGitRefAdvertisement(
|
||||
await readResponseTextCapped(response, MAX_REF_BYTES),
|
||||
);
|
||||
if (!sha) throw new Error('GhostMaps branch head not advertised');
|
||||
return sha;
|
||||
}
|
||||
|
||||
async function readCommit(sha) {
|
||||
const api = `https://api.github.com/repos/${REPO_PATH}`;
|
||||
const tree = await readResponseJsonCapped(
|
||||
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
|
||||
MAX_TREE_BYTES,
|
||||
);
|
||||
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
|
||||
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
|
||||
let committedAt = null;
|
||||
let message = null;
|
||||
try {
|
||||
const commit = await readResponseJsonCapped(
|
||||
await get(`${api}/commits/${sha}`, apiHeaders()),
|
||||
MAX_COMMIT_BYTES,
|
||||
);
|
||||
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
|
||||
message = String(commit?.commit?.message || '')
|
||||
.split('\n')[0]
|
||||
.slice(0, 200);
|
||||
} catch {
|
||||
// The commit date is decoration; the tree is what matters.
|
||||
}
|
||||
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
|
||||
}
|
||||
|
||||
async function loadDisk() {
|
||||
if (diskLoaded) return;
|
||||
diskLoaded = true;
|
||||
try {
|
||||
const saved = JSON.parse(
|
||||
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
|
||||
);
|
||||
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
|
||||
state = { ...state, ...saved, error: null };
|
||||
} catch {
|
||||
/* first run */
|
||||
}
|
||||
}
|
||||
|
||||
async function saveDisk() {
|
||||
try {
|
||||
await fsp.mkdir(cacheDir, { recursive: true });
|
||||
const { commit, committedAt, message, products, changedAt } = state;
|
||||
await fsp.writeFile(
|
||||
path.join(cacheDir, 'catalog.json'),
|
||||
JSON.stringify({ commit, committedAt, message, products, changedAt }),
|
||||
);
|
||||
} catch (error) {
|
||||
warn('[ghostmaps] could not save catalog:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function emit() {
|
||||
const event = publicCatalog();
|
||||
for (const listener of listeners) {
|
||||
try {
|
||||
listener(event);
|
||||
} catch {
|
||||
/* a broken subscriber must not stop the rest */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function check() {
|
||||
await loadDisk();
|
||||
const head = await readHead();
|
||||
state.checkedAt = now();
|
||||
if (head === state.commit) {
|
||||
if (state.error !== null) {
|
||||
state.error = null;
|
||||
emit();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const next = await readCommit(head);
|
||||
const previous = new Map(state.products.map((p) => [p.id, p.version]));
|
||||
const changed = next.products
|
||||
.filter((p) => previous.get(p.id) !== p.version)
|
||||
.map((p) => p.id);
|
||||
state = {
|
||||
...state,
|
||||
...next,
|
||||
commit: head,
|
||||
changedAt: now(),
|
||||
error: null,
|
||||
};
|
||||
log(
|
||||
`[ghostmaps] commit ${head.slice(0, 8)}${
|
||||
next.message ? ` "${next.message}"` : ''
|
||||
}; updated: ${changed.join(', ') || 'none'}`,
|
||||
);
|
||||
await saveDisk();
|
||||
emit();
|
||||
// Warm the default products so the first browser load is local.
|
||||
for (const product of state.products)
|
||||
if (product.defaultOn) file(product.id).catch(() => {});
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Run one check now, coalescing with one already in flight. */
|
||||
function checkNow() {
|
||||
if (!checking)
|
||||
checking = check()
|
||||
.then(
|
||||
(changed) => {
|
||||
failures = 0;
|
||||
return changed;
|
||||
},
|
||||
(error) => {
|
||||
failures += 1;
|
||||
state.error = error.message;
|
||||
state.checkedAt = now();
|
||||
warn('[ghostmaps] check failed:', error.message);
|
||||
emit();
|
||||
return false;
|
||||
},
|
||||
)
|
||||
.finally(() => {
|
||||
checking = null;
|
||||
});
|
||||
return checking;
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
clearTimeout(timer);
|
||||
if (!running) return;
|
||||
const delay = failures
|
||||
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
|
||||
: pollMs;
|
||||
timer = setTimeout(async () => {
|
||||
await checkNow();
|
||||
schedule();
|
||||
}, delay);
|
||||
timer.unref?.();
|
||||
}
|
||||
|
||||
/** Start watching (idempotent); the first check runs immediately. */
|
||||
function start() {
|
||||
if (running) return checking || Promise.resolve(false);
|
||||
running = true;
|
||||
const first = checkNow();
|
||||
first.then(schedule);
|
||||
return first;
|
||||
}
|
||||
|
||||
function stop() {
|
||||
running = false;
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
|
||||
function publicCatalog() {
|
||||
return {
|
||||
repo: GHOSTMAPS_REPO.url,
|
||||
branch: GHOSTMAPS_REPO.branch,
|
||||
commit: state.commit,
|
||||
committedAt: state.committedAt,
|
||||
message: state.message,
|
||||
checkedAt: state.checkedAt,
|
||||
changedAt: state.changedAt,
|
||||
pollMs,
|
||||
error: state.error,
|
||||
products: state.products.map(
|
||||
({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}) => ({
|
||||
id,
|
||||
title,
|
||||
defaultOn,
|
||||
fileName,
|
||||
fileTitle,
|
||||
version,
|
||||
size,
|
||||
dateMs,
|
||||
}),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The bytes of one product's current edition, from disk or a commit-pinned
|
||||
* download. Only catalog paths are ever fetched.
|
||||
*/
|
||||
async function file(id) {
|
||||
const product = state.products.find((p) => p.id === id);
|
||||
if (!product) return null;
|
||||
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
|
||||
try {
|
||||
return { product, bytes: await fsp.readFile(target) };
|
||||
} catch {
|
||||
/* not cached yet */
|
||||
}
|
||||
const commit = state.commit;
|
||||
let pending = downloads.get(product.version);
|
||||
if (!pending) {
|
||||
pending = (async () => {
|
||||
const response = await get(
|
||||
ghostMapsRawUrl(commit, product.path),
|
||||
{ 'User-Agent': USER_AGENT },
|
||||
FILE_TIMEOUT_MS,
|
||||
);
|
||||
const bytes = Buffer.from(
|
||||
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
|
||||
);
|
||||
await fsp.mkdir(path.dirname(target), { recursive: true });
|
||||
const temp = `${target}.${process.pid}.tmp`;
|
||||
await fsp.writeFile(temp, bytes);
|
||||
await fsp.rename(temp, target);
|
||||
await pruneFiles();
|
||||
return bytes;
|
||||
})().finally(() => downloads.delete(product.version));
|
||||
downloads.set(product.version, pending);
|
||||
}
|
||||
return { product, bytes: await pending };
|
||||
}
|
||||
|
||||
/** Keep only files that a current product still points to. */
|
||||
async function pruneFiles() {
|
||||
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
|
||||
const dir = path.join(cacheDir, 'files');
|
||||
for (const name of await fsp.readdir(dir).catch(() => [])) {
|
||||
if (!keep.has(name) && !name.endsWith('.tmp'))
|
||||
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
function subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
checkNow,
|
||||
catalog: publicCatalog,
|
||||
file,
|
||||
subscribe,
|
||||
subscribers: () => listeners.size,
|
||||
};
|
||||
}
|
||||
|
||||
/** Same-origin GhostMaps routes for dev and preview. */
|
||||
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
|
||||
const json = (res, status, value) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
|
||||
function events(req, res) {
|
||||
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
|
||||
return json(res, 429, { error: 'too_many_subscribers' });
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-store',
|
||||
Connection: 'keep-alive',
|
||||
'X-Accel-Buffering': 'no',
|
||||
});
|
||||
const send = (catalog) =>
|
||||
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
|
||||
res.write('retry: 5000\n\n');
|
||||
send(watcher.catalog());
|
||||
const unsubscribe = watcher.subscribe(send);
|
||||
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
|
||||
heartbeat.unref?.();
|
||||
const close = () => {
|
||||
clearInterval(heartbeat);
|
||||
unsubscribe();
|
||||
};
|
||||
req.on('close', close);
|
||||
res.on('error', close);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
const url = new URL(req.url || '/', 'http://local');
|
||||
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
|
||||
if (!['/catalog', '/events'].includes(url.pathname) && !match)
|
||||
return json(res, 404, { error: 'unknown_route' });
|
||||
// Watching starts with the first request, so an install that never turns
|
||||
// the layer on never contacts GitHub.
|
||||
const first = watcher.start();
|
||||
if (url.pathname === '/catalog') {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
return json(res, 200, watcher.catalog());
|
||||
}
|
||||
if (url.pathname === '/events') return events(req, res);
|
||||
try {
|
||||
if (!watcher.catalog().commit) await first;
|
||||
const found = await watcher.file(match[1]);
|
||||
if (!found) return json(res, 404, { error: 'unknown_product' });
|
||||
const pinned = url.searchParams.get('v') === found.product.version;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/vnd.google-earth.kmz',
|
||||
'Content-Length': found.bytes.length,
|
||||
'X-GhostMaps-Version': found.product.version,
|
||||
// A versioned URL names one immutable blob.
|
||||
'Cache-Control': pinned
|
||||
? 'private, max-age=31536000, immutable'
|
||||
: 'no-store',
|
||||
});
|
||||
return res.end(found.bytes);
|
||||
} catch {
|
||||
return json(res, 502, { error: 'ghostmaps_unavailable' });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'ghostmaps',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/ghostmaps', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtemp, readdir, rm } from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
createGhostMapsWatcher,
|
||||
ghostMapsPollMs,
|
||||
ghostMapsProxy,
|
||||
} from './ghostMaps.js';
|
||||
|
||||
const MASTER =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
|
||||
const KINETIC =
|
||||
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
|
||||
const sha = (c) => c.repeat(40);
|
||||
|
||||
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
|
||||
function fakeGitHub() {
|
||||
const repo = { head: sha('1'), trees: new Map(), calls: [] };
|
||||
repo.trees.set(sha('1'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('a'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
const body = (text, status = 200) =>
|
||||
new Response(text, {
|
||||
status,
|
||||
headers: { 'content-length': String(Buffer.byteLength(text)) },
|
||||
});
|
||||
const fetchImpl = async (url) => {
|
||||
repo.calls.push(url);
|
||||
if (url.endsWith('/info/refs?service=git-upload-pack'))
|
||||
return body(
|
||||
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
|
||||
);
|
||||
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
|
||||
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
|
||||
if (/\/commits\//.test(url))
|
||||
return body(
|
||||
JSON.stringify({
|
||||
commit: {
|
||||
message: 'Oct 18 Update\nmore',
|
||||
committer: { date: '2026-10-18T00:00:00Z' },
|
||||
},
|
||||
}),
|
||||
);
|
||||
if (url.startsWith('https://raw.githubusercontent.com/'))
|
||||
return body('KMZ!');
|
||||
return body('nope', 404);
|
||||
};
|
||||
return { repo, fetchImpl };
|
||||
}
|
||||
|
||||
async function withCache(fn) {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
|
||||
try {
|
||||
await fn(dir);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
// checkNow announces the catalog before background warm-up finishes. Join the
|
||||
// same file reads before changing editions or removing the temporary cache.
|
||||
async function warmDefaults(watcher) {
|
||||
await Promise.all(
|
||||
watcher
|
||||
.catalog()
|
||||
.products.filter((product) => product.defaultOn)
|
||||
.map((product) => watcher.file(product.id)),
|
||||
);
|
||||
}
|
||||
|
||||
test('a new commit is announced once, naming only the products that changed', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
const first = watcher.catalog();
|
||||
assert.equal(first.commit, sha('1'));
|
||||
assert.equal(first.message, 'Oct 18 Update');
|
||||
assert.deepEqual(
|
||||
first.products.map((p) => p.id),
|
||||
['cip', 'kinetic-30d'],
|
||||
);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// Same head: one ref read, no REST calls, no event.
|
||||
repo.calls.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(repo.calls.length, 1);
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
...repo.trees.get(sha('1')),
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
]);
|
||||
assert.equal(await watcher.checkNow(), true);
|
||||
await warmDefaults(watcher);
|
||||
assert.equal(events.length, 2);
|
||||
const cip = events[1].products.find((p) => p.id === 'cip');
|
||||
assert.equal(cip.version, sha('c'));
|
||||
assert.match(cip.fileName, /October_18/);
|
||||
assert.equal(
|
||||
events[1].products.find((p) => p.id === 'kinetic-30d').version,
|
||||
sha('b'),
|
||||
);
|
||||
// Repository paths stay on the server.
|
||||
assert.equal('path' in cip, false);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { repo, fetchImpl } = fakeGitHub();
|
||||
const watcher = createGhostMapsWatcher({
|
||||
fetchImpl,
|
||||
cacheDir,
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
await watcher.checkNow();
|
||||
const found = await watcher.file('kinetic-30d');
|
||||
assert.equal(found.bytes.toString(), 'KMZ!');
|
||||
const raw = repo.calls.filter((url) =>
|
||||
url.includes('raw.githubusercontent'),
|
||||
);
|
||||
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
|
||||
assert.ok(
|
||||
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
|
||||
);
|
||||
const before = repo.calls.length;
|
||||
await watcher.file('kinetic-30d');
|
||||
assert.equal(repo.calls.length, before, 'second read comes from disk');
|
||||
assert.equal(await watcher.file('not-a-product'), null);
|
||||
|
||||
// When the edition changes, the old blob leaves the disk cache.
|
||||
await watcher.file('cip');
|
||||
repo.head = sha('2');
|
||||
repo.trees.set(sha('2'), [
|
||||
{
|
||||
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
|
||||
type: 'blob',
|
||||
size: 4,
|
||||
sha: sha('c'),
|
||||
},
|
||||
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
|
||||
]);
|
||||
await watcher.checkNow();
|
||||
await watcher.file('cip');
|
||||
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
|
||||
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
const quiet = { log: () => {}, warn: () => {} };
|
||||
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
|
||||
await online.checkNow();
|
||||
await warmDefaults(online);
|
||||
const offline = createGhostMapsWatcher({
|
||||
fetchImpl: async () => new Response('down', { status: 503 }),
|
||||
cacheDir,
|
||||
...quiet,
|
||||
});
|
||||
assert.equal(await offline.checkNow(), false);
|
||||
const catalog = offline.catalog();
|
||||
assert.equal(catalog.commit, sha('1'));
|
||||
assert.equal(catalog.products.length, 2);
|
||||
assert.match(catalog.error, /HTTP 503/);
|
||||
}));
|
||||
|
||||
test('an unchanged head clears an upstream outage and announces recovery once', () =>
|
||||
withCache(async (cacheDir) => {
|
||||
const { fetchImpl } = fakeGitHub();
|
||||
let unavailable = false;
|
||||
let checkedAt = 1000;
|
||||
const requests = [];
|
||||
const watcher = createGhostMapsWatcher({
|
||||
cacheDir,
|
||||
now: () => checkedAt,
|
||||
fetchImpl: async (url, options) => {
|
||||
requests.push(url);
|
||||
if (unavailable) return new Response('down', { status: 503 });
|
||||
return fetchImpl(url, options);
|
||||
},
|
||||
log: () => {},
|
||||
warn: () => {},
|
||||
});
|
||||
const events = [];
|
||||
watcher.subscribe((catalog) => events.push(catalog));
|
||||
await watcher.checkNow();
|
||||
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
|
||||
const healthy = watcher.catalog();
|
||||
assert.equal(events.length, 1);
|
||||
|
||||
unavailable = true;
|
||||
checkedAt = 2000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.match(watcher.catalog().error, /HTTP 503/);
|
||||
assert.equal(events.length, 2);
|
||||
|
||||
unavailable = false;
|
||||
checkedAt = 3000;
|
||||
requests.length = 0;
|
||||
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
|
||||
const recovered = watcher.catalog();
|
||||
assert.equal(recovered.error, null);
|
||||
assert.equal(recovered.checkedAt, checkedAt);
|
||||
assert.equal(recovered.commit, healthy.commit);
|
||||
assert.equal(recovered.changedAt, healthy.changedAt);
|
||||
assert.deepEqual(recovered.products, healthy.products);
|
||||
assert.equal(requests.length, 1, 'recovery needs only the ref read');
|
||||
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
|
||||
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
|
||||
assert.equal(events[2].error, null);
|
||||
assert.equal(events[2].checkedAt, checkedAt);
|
||||
|
||||
checkedAt = 4000;
|
||||
assert.equal(await watcher.checkNow(), false);
|
||||
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
|
||||
watcher.stop();
|
||||
}));
|
||||
|
||||
test('poll interval is configurable with a 5 s floor', () => {
|
||||
assert.equal(ghostMapsPollMs(undefined), 15000);
|
||||
assert.equal(ghostMapsPollMs('30'), 30000);
|
||||
assert.equal(ghostMapsPollMs('1'), 5000);
|
||||
assert.equal(ghostMapsPollMs('soon'), 15000);
|
||||
});
|
||||
|
||||
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
|
||||
let started = 0;
|
||||
const watcher = {
|
||||
start: () => {
|
||||
started += 1;
|
||||
return Promise.resolve(false);
|
||||
},
|
||||
catalog: () => ({ commit: sha('1'), products: [] }),
|
||||
file: async () => null,
|
||||
subscribe: () => () => {},
|
||||
subscribers: () => 0,
|
||||
};
|
||||
let handler;
|
||||
ghostMapsProxy({ watcher }).configureServer({
|
||||
middlewares: { use: (_route, fn) => (handler = fn) },
|
||||
});
|
||||
const call = async (method, url) => {
|
||||
const res = {
|
||||
destroyed: false,
|
||||
writeHead(status, headers) {
|
||||
this.status = status;
|
||||
this.headers = headers;
|
||||
},
|
||||
end(body) {
|
||||
this.body = String(body ?? '');
|
||||
},
|
||||
};
|
||||
await handler({ method, url }, res);
|
||||
return res;
|
||||
};
|
||||
assert.equal((await call('GET', '/nope')).status, 404);
|
||||
assert.equal((await call('POST', '/catalog')).status, 405);
|
||||
assert.equal((await call('GET', '/file/../../etc')).status, 404);
|
||||
assert.equal(started, 0);
|
||||
assert.equal((await call('GET', '/file/cip')).status, 404);
|
||||
const catalog = await call('GET', '/catalog');
|
||||
assert.equal(catalog.status, 200);
|
||||
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
|
||||
assert.equal(started, 2);
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import path from 'node:path';
|
||||
import { openSkyProxy } from './aircraft/opensky.js';
|
||||
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
|
||||
import { tomtomProxy } from './traffic.js';
|
||||
@@ -7,7 +8,7 @@ import { adsbdbProxy } from './aircraft/enrichment.js';
|
||||
import { overpassProxy } from './overpass.js';
|
||||
import { militaryInstallationsProxy } from './military-installations.js';
|
||||
import { regionalBriefProxy } from './regional/briefing.js';
|
||||
import { geocodeProxy } from './regional/place.js';
|
||||
import { geocodeProxy, NOMINATIM_CACHE_DIR } from './regional/place.js';
|
||||
import { weatherEffectsProxy } from './regional/weather-effects.js';
|
||||
import { cctvProxy } from './cctv.js';
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
@@ -26,14 +27,27 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { ghostMapsProxy } from './ghostMaps.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
/**
|
||||
* Construct the local provider plugins in their established order.
|
||||
* `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live.
|
||||
* `nominatimStorageDir` binds an owned absolute directory before construction;
|
||||
* omission retains the durable per-source cache and policy state.
|
||||
*/
|
||||
function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
function localProviderPlugins({
|
||||
realtime,
|
||||
gemini,
|
||||
nominatimStorageDir = NOMINATIM_CACHE_DIR,
|
||||
} = {}) {
|
||||
if (
|
||||
typeof nominatimStorageDir !== 'string' ||
|
||||
!nominatimStorageDir.trim() ||
|
||||
!path.isAbsolute(nominatimStorageDir)
|
||||
)
|
||||
throw new TypeError('nominatimStorageDir must be an absolute directory');
|
||||
return [
|
||||
openSkyProxy(),
|
||||
celestrakProxy(),
|
||||
@@ -45,7 +59,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
overpassProxy(),
|
||||
militaryInstallationsProxy(),
|
||||
regionalBriefProxy(),
|
||||
geocodeProxy(),
|
||||
geocodeProxy({ storageDir: nominatimStorageDir }),
|
||||
weatherEffectsProxy(),
|
||||
cctvProxy({ sourceRoot: defaultSourceRoot }),
|
||||
radioBrowserProxy(),
|
||||
@@ -64,6 +78,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
ghostMapsProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSite } from './common/same-site.js';
|
||||
import {
|
||||
MILITARY_INSTALLATION_ELEMENT_CAP,
|
||||
MILITARY_INSTALLATION_MAX_RESPONSE_BYTES,
|
||||
@@ -97,6 +98,7 @@ function militaryInstallationsProxy() {
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
if (admitSameSite(req, res)) return;
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
const requested = validMilitaryInstallationBox(url.searchParams);
|
||||
if (!requested) {
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { SNAPSHOT_FEED_STATES } from '../../../src/data/layerSnapshot.js';
|
||||
import {
|
||||
HUD_SUMMARY_INSTRUCTIONS,
|
||||
HUD_SUMMARY_MAX_CHARS,
|
||||
HUD_SUMMARY_MAX_UNSPACED_CHARS,
|
||||
HUD_SUMMARY_RESPONSE_FORMAT,
|
||||
hudSummaryInput,
|
||||
hudSummaryInstructions,
|
||||
hudSummaryIsUnspaced,
|
||||
hudSummaryLocale,
|
||||
keylessHudSummaryResponse,
|
||||
} from '../../../src/hudSummaryResponse.js';
|
||||
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
@@ -18,17 +25,74 @@ function extractOpenAiResponseText(data) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function toFiveWordHudSummary(value) {
|
||||
return String(value || '')
|
||||
.replace(/[^\p{L}\p{N}\s-]/gu, ' ')
|
||||
.trim()
|
||||
/** Parse the summary and normalize the canonical feed state. */
|
||||
function parseHudSummaryReply(text, locale) {
|
||||
let reply = null;
|
||||
try {
|
||||
reply = JSON.parse(text);
|
||||
} catch {
|
||||
/* not JSON: no summary */
|
||||
}
|
||||
const feedState = SNAPSHOT_FEED_STATES.includes(reply?.feedState)
|
||||
? reply.feedState
|
||||
: null;
|
||||
return {
|
||||
summary: toHudSummary(reply?.summary, locale),
|
||||
feedState,
|
||||
};
|
||||
}
|
||||
|
||||
function toHudSummary(value, requested) {
|
||||
const locale = hudSummaryLocale(requested);
|
||||
const words = String(value || '')
|
||||
.normalize('NFC')
|
||||
// Marks, and apostrophes and dots inside a word, belong to it: ar vowel
|
||||
// signs, fr d'Orléans, ja サンフランシスコ・ベイエリア.
|
||||
.replace(/[^\p{L}\p{M}\p{N}\s'’·・-]/gu, ' ')
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.slice(0, 5)
|
||||
.join(' ');
|
||||
.map((word) => word.replace(/^['’·・-]+|['’·・-]+$/gu, ''))
|
||||
.filter(Boolean);
|
||||
// Characters, not words: the model counts Rue de Rivoli as one word.
|
||||
const max = hudSummaryIsUnspaced(locale)
|
||||
? HUD_SUMMARY_MAX_UNSPACED_CHARS
|
||||
: HUD_SUMMARY_MAX_CHARS;
|
||||
return capAtWord(words.join(' '), max, locale);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cut `text` to `max` characters at a word boundary; a first word longer
|
||||
* than that is cut where the cap falls.
|
||||
*/
|
||||
function capAtWord(text, max, locale) {
|
||||
const graphemes = [
|
||||
...new Intl.Segmenter(locale, { granularity: 'grapheme' }).segment(text),
|
||||
];
|
||||
if (graphemes.length <= max) return text;
|
||||
const cut = graphemes[max].index;
|
||||
let end = 0;
|
||||
for (const { index, segment } of new Intl.Segmenter(locale, {
|
||||
granularity: 'word',
|
||||
}).segment(text)) {
|
||||
if (index + segment.length > cut) break;
|
||||
end = index + segment.length;
|
||||
}
|
||||
return text.slice(0, end).trim() || text.slice(0, cut);
|
||||
}
|
||||
|
||||
async function handleHudSummary(req, res) {
|
||||
// Capability probe: lets the HUD skip its paid place lookups when there is
|
||||
// no summary to feed them to.
|
||||
if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
configured: !keylessHudSummaryResponse(process.env.OPENAI_API_KEY),
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (req.method !== 'POST') {
|
||||
res.statusCode = 405;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
@@ -64,14 +128,18 @@ async function handleHudSummary(req, res) {
|
||||
model:
|
||||
process.env.OPENAI_HUD_SUMMARY_MODEL ||
|
||||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
|
||||
instructions: HUD_SUMMARY_INSTRUCTIONS,
|
||||
input: JSON.stringify(context),
|
||||
instructions: hudSummaryInstructions(context?.locale),
|
||||
input: hudSummaryInput(context),
|
||||
text: { format: HUD_SUMMARY_RESPONSE_FORMAT },
|
||||
reasoning: { effort: 'minimal' },
|
||||
max_output_tokens: 100,
|
||||
max_output_tokens: 200,
|
||||
}),
|
||||
});
|
||||
const data = await response.json().catch(() => ({}));
|
||||
const summary = toFiveWordHudSummary(extractOpenAiResponseText(data));
|
||||
const { summary, feedState } = parseHudSummaryReply(
|
||||
extractOpenAiResponseText(data),
|
||||
context?.locale,
|
||||
);
|
||||
res.statusCode = response.ok && summary ? 200 : response.status || 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -80,6 +148,7 @@ async function handleHudSummary(req, res) {
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
summary: summary || null,
|
||||
feedState,
|
||||
// Never relay `data.error.message`: that is OpenAI's own wording, and
|
||||
// it carries request ids, organization hints and quota phrasing.
|
||||
error: response.ok ? null : 'OpenAI HUD summary request failed',
|
||||
@@ -97,4 +166,4 @@ async function handleHudSummary(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
export { handleHudSummary };
|
||||
export { handleHudSummary, parseHudSummaryReply };
|
||||
@@ -10,7 +10,7 @@ function realtimeInstructions(
|
||||
'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
|
||||
'Only control the app by calling the provided tools. Never invent tool names or arguments.',
|
||||
'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
|
||||
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
|
||||
'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
|
||||
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
|
||||
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
|
||||
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
|
||||
@@ -18,13 +18,12 @@ function realtimeInstructions(
|
||||
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
|
||||
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
|
||||
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
|
||||
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
|
||||
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
|
||||
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
|
||||
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
|
||||
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
|
||||
'Do not mention disabled layers or stale selections.',
|
||||
@@ -47,15 +46,15 @@ function realtimeInstructions(
|
||||
// string is the whole rollback.
|
||||
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
|
||||
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
|
||||
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
|
||||
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
|
||||
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
|
||||
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'HUD requests ("hud on/off", "switch to minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
|
||||
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
|
||||
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
|
||||
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
|
||||
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
|
||||
'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
|
||||
'Confirmations echo the RESULTING state, never the request: "HUD minimal layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
|
||||
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
|
||||
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
|
||||
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
|
||||
|
||||
@@ -44,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
select_nearest_aircraft: {
|
||||
description:
|
||||
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
|
||||
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -111,12 +111,18 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
set_panel_open: {
|
||||
description: 'Open or close a GEV UI panel/dropdown.',
|
||||
description:
|
||||
'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
|
||||
$position: 1,
|
||||
},
|
||||
set_voice_inactivity_timeout: {
|
||||
description:
|
||||
'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
|
||||
$position: 1,
|
||||
},
|
||||
set_context_mode: {
|
||||
description:
|
||||
'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
|
||||
'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -129,23 +135,23 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
control_cockpit: {
|
||||
description:
|
||||
'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
|
||||
'Explicit Cockpit requests only: enter from a selected/tracked aircraft (establishes Contacts itself; no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
action: {
|
||||
description:
|
||||
'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
|
||||
'previous/next (or prev) cycles nearby Cockpit contacts.',
|
||||
$position: 2,
|
||||
},
|
||||
targetLayer: {
|
||||
description:
|
||||
'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
|
||||
'Optional next/previous layer filter (e.g. military-only).',
|
||||
$position: 2,
|
||||
},
|
||||
aircraftClass: {
|
||||
description:
|
||||
'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
|
||||
'Optional next/previous aircraft class filter (e.g. helicopter).',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { normalizePlaceName as foldName } from '../../../src/data/placeNames.js';
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
/** Entry files this cache writes: a 40-hex-digit key hash. */
|
||||
const CACHE_FILE_RE = /^[0-9a-f]{40}\.json$/;
|
||||
|
||||
/** How long an answer is kept, by what it was. */
|
||||
export const NOMINATIM_CACHE_TTL_MS = Object.freeze({
|
||||
outline: 90 * DAY_MS,
|
||||
place: 30 * DAY_MS,
|
||||
notFound: DAY_MS,
|
||||
});
|
||||
|
||||
/**
|
||||
* Comparable form of a place name for matching results to the ask: the
|
||||
* shared place-name key (see `normalizePlaceName` in placeNames.js, so
|
||||
* "St. Paul" and "Saint Paul" agree), bounded in length.
|
||||
*
|
||||
* @param {string} value
|
||||
* @returns {string}
|
||||
*/
|
||||
export function normalizePlaceName(value) {
|
||||
return foldName(value).slice(0, 200);
|
||||
}
|
||||
|
||||
/**
|
||||
* Memory + disk cache for Nominatim answers.
|
||||
*
|
||||
* Every entry carries its own expiry. Memory is an LRU bounded by entries;
|
||||
* disk is bounded by entries and bytes, with expired files removed during a
|
||||
* periodic sweep. A null `dir` keeps everything in memory.
|
||||
*/
|
||||
export function createNominatimCache({
|
||||
dir = null,
|
||||
maxMemoryEntries = 256,
|
||||
maxDiskEntries = 4000,
|
||||
maxDiskBytes = 64 * 1024 * 1024,
|
||||
maxEntryBytes = 2 * 1024 * 1024,
|
||||
sweepEvery = 50,
|
||||
now = Date.now,
|
||||
} = {}) {
|
||||
const memory = new Map();
|
||||
let writesSinceSweep = 0;
|
||||
let sweeping = null;
|
||||
|
||||
const digestFor = (key) =>
|
||||
crypto.createHash('sha256').update(key).digest('hex');
|
||||
|
||||
const fileFor = (key) =>
|
||||
path.join(dir, `${digestFor(key).slice(0, 40)}.json`);
|
||||
|
||||
function remember(key, entry) {
|
||||
memory.delete(key);
|
||||
memory.set(key, entry);
|
||||
while (memory.size > maxMemoryEntries)
|
||||
memory.delete(memory.keys().next().value);
|
||||
}
|
||||
|
||||
async function get(key) {
|
||||
const hit = memory.get(key);
|
||||
if (hit) {
|
||||
if (hit.expiresAt > now()) {
|
||||
remember(key, hit);
|
||||
return hit;
|
||||
}
|
||||
memory.delete(key);
|
||||
}
|
||||
if (!dir) return null;
|
||||
try {
|
||||
const entry = JSON.parse(await fs.readFile(fileFor(key), 'utf8'));
|
||||
if (entry?.keyDigest !== digestFor(key) || !(entry.expiresAt > now()))
|
||||
return null;
|
||||
remember(key, entry);
|
||||
return entry;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function set(key, payload, ttlMs) {
|
||||
const entry = {
|
||||
keyDigest: digestFor(key),
|
||||
payload,
|
||||
storedAt: now(),
|
||||
expiresAt: now() + ttlMs,
|
||||
};
|
||||
remember(key, entry);
|
||||
if (!dir) return;
|
||||
const body = JSON.stringify(entry);
|
||||
if (Buffer.byteLength(body) > maxEntryBytes) return;
|
||||
let temp = null;
|
||||
try {
|
||||
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
|
||||
await fs.chmod(dir, 0o700);
|
||||
const file = fileFor(key);
|
||||
temp = `${file}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`;
|
||||
await fs.writeFile(temp, body, { mode: 0o600, flag: 'wx' });
|
||||
await fs.rename(temp, file);
|
||||
temp = null;
|
||||
} catch {
|
||||
return;
|
||||
} finally {
|
||||
if (temp) await fs.rm(temp, { force: true }).catch(() => {});
|
||||
}
|
||||
writesSinceSweep += 1;
|
||||
if (writesSinceSweep >= sweepEvery) {
|
||||
writesSinceSweep = 0;
|
||||
await sweep();
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove expired files, then the oldest until both disk bounds hold. */
|
||||
async function sweep() {
|
||||
if (!dir) return;
|
||||
if (sweeping) return sweeping;
|
||||
sweeping = (async () => {
|
||||
let names;
|
||||
try {
|
||||
names = await fs.readdir(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const files = [];
|
||||
for (const name of names) {
|
||||
// Only this cache's own entry files; anything else in the directory
|
||||
// is left alone.
|
||||
if (!CACHE_FILE_RE.test(name)) continue;
|
||||
const file = path.join(dir, name);
|
||||
let stat;
|
||||
let text;
|
||||
try {
|
||||
[stat, text] = await Promise.all([
|
||||
fs.stat(file),
|
||||
fs.readFile(file, 'utf8'),
|
||||
]);
|
||||
} catch {
|
||||
// Unreadable right now (gone, busy, too many open files): an error
|
||||
// reading is not evidence the entry is bad, so it stays.
|
||||
continue;
|
||||
}
|
||||
let expiresAt;
|
||||
try {
|
||||
expiresAt = JSON.parse(text)?.expiresAt;
|
||||
} catch {
|
||||
expiresAt = null; // corrupt: removed below
|
||||
}
|
||||
if (!(expiresAt > now())) {
|
||||
await fs.rm(file, { force: true }).catch(() => {});
|
||||
continue;
|
||||
}
|
||||
files.push({ file, size: stat.size, at: stat.mtimeMs });
|
||||
}
|
||||
files.sort((a, b) => a.at - b.at);
|
||||
let bytes = files.reduce((sum, entry) => sum + entry.size, 0);
|
||||
while (
|
||||
files.length &&
|
||||
(files.length > maxDiskEntries || bytes > maxDiskBytes)
|
||||
) {
|
||||
const oldest = files.shift();
|
||||
bytes -= oldest.size;
|
||||
await fs.rm(oldest.file, { force: true }).catch(() => {});
|
||||
}
|
||||
})().finally(() => {
|
||||
sweeping = null;
|
||||
});
|
||||
return sweeping;
|
||||
}
|
||||
|
||||
return { get, set, sweep };
|
||||
}
|
||||
@@ -0,0 +1,955 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* One gate for every request this server sends to a Nominatim instance.
|
||||
*
|
||||
* Place search and voice outlines share it. The service is off until an operator
|
||||
* configures an endpoint. Explicit public requests coordinate fetch invocation
|
||||
* across processes sharing this installation's state file, at least 1.1 s apart.
|
||||
* Independent installations do not share an app-wide limiter; these local guards
|
||||
* do not establish compliance for an application's combined public traffic.
|
||||
*/
|
||||
|
||||
/** The public instance. Only this host is subject to the public-use limits. */
|
||||
export const PUBLIC_NOMINATIM_SEARCH =
|
||||
'https://nominatim.openstreetmap.org/search';
|
||||
|
||||
const PUBLIC_NOMINATIM_HOST = 'nominatim.openstreetmap.org';
|
||||
|
||||
/**
|
||||
* The public usage policy asks for a User-Agent or Referer that identifies the
|
||||
* application. The identity is stable: it is never rotated or varied.
|
||||
*/
|
||||
const NOMINATIM_HEADERS = Object.freeze({
|
||||
'User-Agent':
|
||||
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
|
||||
});
|
||||
|
||||
/** Absolute policy maximum is one request per second; 1.1 s absorbs jitter. */
|
||||
const NOMINATIM_MIN_SPACING_MS = 1100;
|
||||
|
||||
/** Requests allowed to wait for their turn; past this, refuse at once. */
|
||||
export const NOMINATIM_MAX_PENDING = 4;
|
||||
|
||||
/** A queued request older than this has no reader left; drop it unsent. */
|
||||
const NOMINATIM_MAX_WAIT_MS = 10_000;
|
||||
|
||||
/** Default public requests per install per UTC day, outlines and search together. */
|
||||
const NOMINATIM_DEFAULT_DAILY_CAP = 50;
|
||||
|
||||
/**
|
||||
* Pause after a refusal without Retry-After. A Retry-After from the server is
|
||||
* honoured in full, however long: no local maximum may cause an earlier retry.
|
||||
*/
|
||||
const DEFAULT_BACKOFF_MS = 60_000;
|
||||
/** A 403 from the public instance means this identity is blocked: stop longer. */
|
||||
const BLOCKED_BACKOFF_MS = 3_600_000;
|
||||
/** Pause after a server error or a failed retry. */
|
||||
const OUTAGE_BACKOFF_MS = 30_000;
|
||||
/** How old an unfinished dispatch must be before the next lock owner reclaims it. */
|
||||
const DISPATCH_RECLAIM_MS = 10 * 60_000;
|
||||
|
||||
const DISABLED_VALUES = new Set(['', 'off', 'none', 'disabled', '0']);
|
||||
|
||||
/**
|
||||
* node:sqlite, loaded on first public-instance use. Node 24 still flags it
|
||||
* experimental and prints that once per process; only that one warning is
|
||||
* dropped, and only while the module loads.
|
||||
*/
|
||||
let sqlite = null;
|
||||
function sqliteModule() {
|
||||
if (sqlite) return sqlite;
|
||||
const emitWarning = process.emitWarning;
|
||||
process.emitWarning = function (warning, ...rest) {
|
||||
const type = typeof rest[0] === 'string' ? rest[0] : rest[0]?.type;
|
||||
if (type === 'ExperimentalWarning' && /SQLite/.test(String(warning)))
|
||||
return;
|
||||
return emitWarning.call(this, warning, ...rest);
|
||||
};
|
||||
try {
|
||||
sqlite = process.getBuiltinModule('node:sqlite');
|
||||
} finally {
|
||||
process.emitWarning = emitWarning;
|
||||
}
|
||||
return sqlite;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the search endpoint setting.
|
||||
*
|
||||
* Unset, empty or `off` disables every Nominatim request. An explicit endpoint
|
||||
* must be an http(s) URL
|
||||
* without credentials, query or fragment; a bare host gets `/search`. A value
|
||||
* that is not usable disables the service rather than falling back to the
|
||||
* public instance.
|
||||
*
|
||||
* @param {Record<string, string|undefined>} [env]
|
||||
* @returns {{endpoint: string|null, isPublic: boolean, dailyCap: number, invalid?: boolean}}
|
||||
*/
|
||||
export function resolveNominatimSettings(env = process.env) {
|
||||
const dailyCap = parseDailyCap(env.NOMINATIM_DAILY_CAP);
|
||||
const raw = env.NOMINATIM_URL;
|
||||
if (raw === undefined) return { endpoint: null, isPublic: false, dailyCap };
|
||||
const value = String(raw).trim();
|
||||
if (DISABLED_VALUES.has(value.toLowerCase()))
|
||||
return { endpoint: null, isPublic: false, dailyCap };
|
||||
let url;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
}
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
!url.hostname ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.search ||
|
||||
url.hash
|
||||
)
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
const publicHost =
|
||||
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST;
|
||||
if (publicHost && url.protocol !== 'https:')
|
||||
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
|
||||
if (url.pathname === '/' || url.pathname === '') url.pathname = '/search';
|
||||
return {
|
||||
endpoint: url.href.replace(/\/$/, ''),
|
||||
// A trailing DNS dot names the same host.
|
||||
isPublic: publicHost,
|
||||
dailyCap,
|
||||
};
|
||||
}
|
||||
|
||||
function parseDailyCap(raw) {
|
||||
if (raw === undefined || String(raw).trim() === '')
|
||||
return NOMINATIM_DEFAULT_DAILY_CAP;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) && value >= 0
|
||||
? Math.floor(value)
|
||||
: NOMINATIM_DEFAULT_DAILY_CAP;
|
||||
}
|
||||
|
||||
function gateError(code, message, extra = {}) {
|
||||
return Object.assign(new Error(message), { code, ...extra });
|
||||
}
|
||||
|
||||
const HTTP_MONTHS = Object.freeze([
|
||||
'Jan',
|
||||
'Feb',
|
||||
'Mar',
|
||||
'Apr',
|
||||
'May',
|
||||
'Jun',
|
||||
'Jul',
|
||||
'Aug',
|
||||
'Sep',
|
||||
'Oct',
|
||||
'Nov',
|
||||
'Dec',
|
||||
]);
|
||||
const HTTP_WEEKDAYS = Object.freeze([
|
||||
'Sun',
|
||||
'Mon',
|
||||
'Tue',
|
||||
'Wed',
|
||||
'Thu',
|
||||
'Fri',
|
||||
'Sat',
|
||||
]);
|
||||
const HTTP_WEEKDAYS_LONG = Object.freeze([
|
||||
'Sunday',
|
||||
'Monday',
|
||||
'Tuesday',
|
||||
'Wednesday',
|
||||
'Thursday',
|
||||
'Friday',
|
||||
'Saturday',
|
||||
]);
|
||||
|
||||
/** Strict RFC 9110 HTTP-date, including the two obsolete recipient formats. */
|
||||
function parseHttpDate(raw, now) {
|
||||
let match = raw.match(
|
||||
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
|
||||
);
|
||||
let weekday, day, month, year, hour, minute, second;
|
||||
if (match) {
|
||||
[, weekday, day, month, year, hour, minute, second] = match;
|
||||
weekday = HTTP_WEEKDAYS.indexOf(weekday);
|
||||
} else {
|
||||
match = raw.match(
|
||||
/^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
|
||||
);
|
||||
if (match) {
|
||||
[, weekday, day, month, year, hour, minute, second] = match;
|
||||
weekday = HTTP_WEEKDAYS_LONG.indexOf(weekday);
|
||||
const currentYear = new Date(now).getUTCFullYear();
|
||||
if (!Number.isFinite(currentYear)) return null;
|
||||
year = Math.floor(currentYear / 100) * 100 + Number(year);
|
||||
if (year > currentYear + 50) year -= 100;
|
||||
} else {
|
||||
match = raw.match(
|
||||
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([ \d]\d) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/,
|
||||
);
|
||||
if (!match) return null;
|
||||
[, weekday, month, day, hour, minute, second, year] = match;
|
||||
weekday = HTTP_WEEKDAYS.indexOf(weekday);
|
||||
}
|
||||
}
|
||||
day = Number(day);
|
||||
month = HTTP_MONTHS.indexOf(month);
|
||||
year = Number(year);
|
||||
hour = Number(hour);
|
||||
minute = Number(minute);
|
||||
second = Number(second);
|
||||
if (
|
||||
weekday < 0 ||
|
||||
month < 0 ||
|
||||
year < 1601 ||
|
||||
hour > 23 ||
|
||||
minute > 59 ||
|
||||
second > 59
|
||||
)
|
||||
return null;
|
||||
const at = Date.UTC(year, month, day, hour, minute, second);
|
||||
const date = new Date(at);
|
||||
return date.getUTCFullYear() === year &&
|
||||
date.getUTCMonth() === month &&
|
||||
date.getUTCDate() === day &&
|
||||
date.getUTCHours() === hour &&
|
||||
date.getUTCMinutes() === minute &&
|
||||
date.getUTCSeconds() === second &&
|
||||
date.getUTCDay() === weekday
|
||||
? at
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Seconds or an HTTP date → milliseconds from now, or null. */
|
||||
export function parseRetryAfter(value, now = Date.now()) {
|
||||
if (value == null) return null;
|
||||
const raw = String(value).trim();
|
||||
if (!raw) return null;
|
||||
if (/^\d+$/.test(raw)) {
|
||||
const seconds = Number(raw);
|
||||
const milliseconds = seconds * 1000;
|
||||
return Number.isSafeInteger(milliseconds) ? milliseconds : null;
|
||||
}
|
||||
// Numeric-looking but invalid delay-seconds must not be reinterpreted as a date.
|
||||
if (/^[+-]?(?:\d|\.\d)/.test(raw)) return null;
|
||||
const at = parseHttpDate(raw, now);
|
||||
return at === null ? null : Math.max(0, at - now);
|
||||
}
|
||||
|
||||
/** UTC calendar day, the unit of the daily cap. */
|
||||
function utcDay(ms) {
|
||||
return new Date(ms).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Public-service state shared by every process using the same file: today's
|
||||
* request count, the next reserved start, and any pause the server asked for.
|
||||
*
|
||||
* Every change is a read-modify-write under an OS-owned SQLite transaction,
|
||||
* and the JSON file is replaced atomically, so two dev servers sharing a
|
||||
* checkout add to one count instead of overwriting each other. A process exit
|
||||
* releases only that process's transaction; there is no stale path to delete
|
||||
* and therefore no stale reclaimer that can remove a successor's live lock.
|
||||
* Memory keeps the highest values this process has seen, so a failed or
|
||||
* unreadable file never lowers the count or shortens a pause; a persistence
|
||||
* failure is reported once and in `status()`.
|
||||
*/
|
||||
export function createGateStateStore({
|
||||
file = null,
|
||||
lockWaitMs = 500,
|
||||
onError = (error) =>
|
||||
console.warn(
|
||||
`[Nominatim] could not persist usage state: ${error?.message || error}`,
|
||||
),
|
||||
} = {}) {
|
||||
let memory = { day: '', count: 0, pausedUntil: 0, nextStartAt: 0 };
|
||||
let lastError = null;
|
||||
|
||||
const report = (error) => {
|
||||
if (!lastError) onError(error);
|
||||
lastError = error;
|
||||
};
|
||||
|
||||
const pauseMarkerPrefix = file ? `${path.basename(file)}.pause.` : null;
|
||||
const dispatchMarkerPrefix = file ? `${path.basename(file)}.dispatch.` : null;
|
||||
|
||||
// A crash or persistence failure during dispatch must never reopen admission
|
||||
// using an older timestamp. A dispatch removes its own marker after
|
||||
// persisting the post-invocation deadline. A marker is only ever seen under
|
||||
// the lock after its owner left the lock, so one that names a start older
|
||||
// than DISPATCH_RECLAIM_MS (far beyond any request's timeout) is reclaimed
|
||||
// by the next lock owner, which restarts spacing from its own clock.
|
||||
function pendingDispatches({ strict = false } = {}) {
|
||||
if (!file) return [];
|
||||
try {
|
||||
return fs
|
||||
.readdirSync(path.dirname(file))
|
||||
.filter((name) => name.startsWith(dispatchMarkerPrefix));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Under the lock: remove dispatch markers whose start is older than
|
||||
* DISPATCH_RECLAIM_MS and report whether any were. Any other marker (newer,
|
||||
* or naming no start) still refuses admission.
|
||||
*/
|
||||
function reclaimDispatches(at) {
|
||||
const names = pendingDispatches({ strict: true });
|
||||
for (const name of names) {
|
||||
const since = /^since-(\d+)$/.exec(
|
||||
name.slice(dispatchMarkerPrefix.length).split('.', 1)[0],
|
||||
);
|
||||
const started = since ? Number(since[1]) : NaN;
|
||||
if (!Number.isSafeInteger(started) || at - started < DISPATCH_RECLAIM_MS)
|
||||
throw new Error('unfinished durable Nominatim dispatch');
|
||||
}
|
||||
for (const name of names)
|
||||
fs.unlinkSync(path.join(path.dirname(file), name));
|
||||
return names.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A refusal must remain visible even while another process owns the SQLite
|
||||
* transaction. Immutable marker names make that write atomic and monotonic:
|
||||
* concurrent writers cannot shorten one another's pause, and the next lock
|
||||
* owner folds every marker into the normal JSON state.
|
||||
*/
|
||||
function pendingPauses({ strict = false } = {}) {
|
||||
if (!file) return [];
|
||||
let names;
|
||||
try {
|
||||
names = fs.readdirSync(path.dirname(file));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return [];
|
||||
}
|
||||
const pauses = [];
|
||||
for (const name of names) {
|
||||
if (!name.startsWith(pauseMarkerPrefix)) continue;
|
||||
const encoded = name.slice(pauseMarkerPrefix.length).split('.', 1)[0];
|
||||
if (!/^\d+$/.test(encoded)) continue;
|
||||
const pausedUntil = Number(encoded);
|
||||
if (!Number.isSafeInteger(pausedUntil)) {
|
||||
const error = new Error('invalid durable Nominatim pause marker');
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
continue;
|
||||
}
|
||||
pauses.push({
|
||||
path: path.join(path.dirname(file), name),
|
||||
pausedUntil,
|
||||
});
|
||||
}
|
||||
return pauses;
|
||||
}
|
||||
|
||||
function markPause(at) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const pausedUntil = Math.ceil(at);
|
||||
const marker = `${file}.pause.${pausedUntil}.${process.pid}.${process.hrtime.bigint()}`;
|
||||
fs.writeFileSync(marker, '', { flag: 'wx' });
|
||||
return pausedUntil;
|
||||
}
|
||||
|
||||
function clearPersistedPauses(pausedUntil) {
|
||||
for (const marker of pendingPauses()) {
|
||||
if (marker.pausedUntil > pausedUntil) continue;
|
||||
try {
|
||||
fs.unlinkSync(marker.path);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') report(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const merge = (a, b) => ({
|
||||
day: a.day >= b.day ? a.day : b.day,
|
||||
count:
|
||||
a.day === b.day
|
||||
? Math.max(a.count, b.count)
|
||||
: a.day > b.day
|
||||
? a.count
|
||||
: b.count,
|
||||
pausedUntil: Math.max(a.pausedUntil || 0, b.pausedUntil || 0),
|
||||
nextStartAt: Math.max(a.nextStartAt || 0, b.nextStartAt || 0),
|
||||
});
|
||||
|
||||
const parseState = (parsed) => {
|
||||
const validDay =
|
||||
parsed?.day === '' ||
|
||||
(typeof parsed?.day === 'string' &&
|
||||
/^\d{4}-\d{2}-\d{2}$/.test(parsed.day) &&
|
||||
new Date(`${parsed.day}T00:00:00.000Z`)
|
||||
.toISOString()
|
||||
.startsWith(parsed.day));
|
||||
const validCount =
|
||||
Number.isInteger(parsed?.count) &&
|
||||
parsed.count >= 0 &&
|
||||
(parsed.day !== '' || parsed.count === 0);
|
||||
const validPausedUntil =
|
||||
Number.isFinite(parsed?.pausedUntil) && parsed.pausedUntil >= 0;
|
||||
// nextStartAt was added after the original state format. Its absence is
|
||||
// the one supported migration; a present value must still be trustworthy.
|
||||
const validNextStartAt =
|
||||
parsed?.nextStartAt === undefined ||
|
||||
(Number.isFinite(parsed.nextStartAt) && parsed.nextStartAt >= 0);
|
||||
if (
|
||||
!parsed ||
|
||||
Array.isArray(parsed) ||
|
||||
typeof parsed !== 'object' ||
|
||||
!validDay ||
|
||||
!validCount ||
|
||||
!validPausedUntil ||
|
||||
!validNextStartAt
|
||||
)
|
||||
throw new Error('invalid durable Nominatim usage state');
|
||||
return {
|
||||
day: parsed.day,
|
||||
count: parsed.count,
|
||||
pausedUntil: parsed.pausedUntil,
|
||||
nextStartAt: parsed.nextStartAt ?? 0,
|
||||
};
|
||||
};
|
||||
|
||||
function readDisk({ strict = false } = {}) {
|
||||
if (!file) {
|
||||
if (strict)
|
||||
throw new Error('no durable Nominatim usage state is configured');
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return parseState(JSON.parse(fs.readFileSync(file, 'utf8')));
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return null;
|
||||
report(error);
|
||||
if (strict) throw error;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function current() {
|
||||
const disk = readDisk();
|
||||
if (disk) memory = merge(memory, disk);
|
||||
const pending = pendingPauses();
|
||||
if (pending.length)
|
||||
memory = merge(memory, {
|
||||
...memory,
|
||||
pausedUntil: Math.max(...pending.map((item) => item.pausedUntil)),
|
||||
});
|
||||
return memory;
|
||||
}
|
||||
|
||||
function withLock(work) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
const waitMs = Number.isFinite(lockWaitMs)
|
||||
? Math.max(0, Math.floor(lockWaitMs))
|
||||
: 500;
|
||||
const database = new (sqliteModule().DatabaseSync)(`${file}.lock.sqlite`, {
|
||||
timeout: waitMs,
|
||||
});
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
transactionOpen = true;
|
||||
const result = work();
|
||||
database.exec('COMMIT');
|
||||
transactionOpen = false;
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (transactionOpen) {
|
||||
try {
|
||||
database.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Closing the connection below still releases this owner's lock.
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
try {
|
||||
database.close();
|
||||
} catch {
|
||||
// The transaction has already been committed or rolled back. Closing
|
||||
// is best effort during exceptional teardown and never removes a lock.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function writeDisk(next) {
|
||||
const temp = `${file}.${process.pid}.${process.hrtime.bigint()}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(temp, JSON.stringify(next));
|
||||
fs.renameSync(temp, file);
|
||||
} finally {
|
||||
try {
|
||||
fs.unlinkSync(temp);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') report(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Apply `change` to the freshest state and persist it atomically. */
|
||||
function update(change) {
|
||||
if (!file) {
|
||||
memory = merge(memory, change(memory));
|
||||
return memory;
|
||||
}
|
||||
try {
|
||||
withLock(() => {
|
||||
const next = merge(memory, change(current()));
|
||||
writeDisk(next);
|
||||
memory = next;
|
||||
clearPersistedPauses(next.pausedUntil);
|
||||
lastError = null;
|
||||
});
|
||||
} catch (error) {
|
||||
report(error);
|
||||
memory = merge(memory, change(memory));
|
||||
}
|
||||
return memory;
|
||||
}
|
||||
|
||||
return {
|
||||
count(day) {
|
||||
const state = current();
|
||||
return state.day === day ? state.count : 0;
|
||||
},
|
||||
increment(day) {
|
||||
return update((state) => ({
|
||||
day,
|
||||
count: state.day === day ? state.count + 1 : 1,
|
||||
pausedUntil: state.pausedUntil,
|
||||
nextStartAt: state.nextStartAt,
|
||||
})).count;
|
||||
},
|
||||
dispatch(
|
||||
day,
|
||||
{ clock, dailyCap, minSpacingMs, latestStartAt = Infinity },
|
||||
start,
|
||||
) {
|
||||
if (!file) {
|
||||
const error = new Error(
|
||||
'no durable Nominatim usage state is configured',
|
||||
);
|
||||
report(error);
|
||||
return { status: 'unavailable', error };
|
||||
}
|
||||
let value;
|
||||
try {
|
||||
let result;
|
||||
withLock(() => {
|
||||
const reclaimed = reclaimDispatches(clock());
|
||||
const disk = readDisk({ strict: true });
|
||||
if (disk) memory = merge(memory, disk);
|
||||
const pauses = pendingPauses({ strict: true });
|
||||
const state = merge(memory, {
|
||||
...memory,
|
||||
pausedUntil: Math.max(
|
||||
memory.pausedUntil,
|
||||
...pauses.map((p) => p.pausedUntil),
|
||||
),
|
||||
});
|
||||
const at = clock();
|
||||
if (reclaimed) {
|
||||
state.nextStartAt = Math.max(state.nextStartAt, at + minSpacingMs);
|
||||
writeDisk(state);
|
||||
memory = state;
|
||||
}
|
||||
if (at > latestStartAt) {
|
||||
result = { status: 'abandoned' };
|
||||
return;
|
||||
}
|
||||
if (state.pausedUntil > at) {
|
||||
result = { status: 'paused', retryAfterMs: state.pausedUntil - at };
|
||||
return;
|
||||
}
|
||||
const count = state.day === day ? state.count : 0;
|
||||
if (count >= dailyCap) {
|
||||
result = { status: 'cap' };
|
||||
return;
|
||||
}
|
||||
if (state.nextStartAt > at) {
|
||||
result =
|
||||
state.nextStartAt > latestStartAt
|
||||
? { status: 'abandoned' }
|
||||
: { status: 'wait', waitMs: state.nextStartAt - at };
|
||||
return;
|
||||
}
|
||||
const marker = `${file}.dispatch.since-${at}.${process.pid}.${process.hrtime.bigint()}`;
|
||||
fs.writeFileSync(marker, '', { flag: 'wx' });
|
||||
const next = {
|
||||
day,
|
||||
count: count + 1,
|
||||
pausedUntil: state.pausedUntil,
|
||||
nextStartAt: at + minSpacingMs,
|
||||
};
|
||||
writeDisk(next);
|
||||
memory = next;
|
||||
const latePause = Math.max(
|
||||
next.pausedUntil,
|
||||
...pendingPauses({ strict: true }).map((p) => p.pausedUntil),
|
||||
);
|
||||
if (clock() > latestStartAt) result = { status: 'abandoned' };
|
||||
else if (latePause > clock())
|
||||
result = { status: 'paused', retryAfterMs: latePause - clock() };
|
||||
else {
|
||||
// Invoke, but never await, the transport while still owning the lock.
|
||||
try {
|
||||
value = start();
|
||||
} catch (error) {
|
||||
value = Promise.reject(error);
|
||||
}
|
||||
result = { status: 'started', value };
|
||||
}
|
||||
const after = {
|
||||
...next,
|
||||
pausedUntil: latePause,
|
||||
nextStartAt: Math.max(next.nextStartAt, clock() + minSpacingMs),
|
||||
};
|
||||
writeDisk(after);
|
||||
memory = after;
|
||||
fs.unlinkSync(marker);
|
||||
clearPersistedPauses(after.pausedUntil);
|
||||
lastError = null;
|
||||
});
|
||||
return result;
|
||||
} catch (error) {
|
||||
// The caller aborts its transport on admission failure. Observe a started
|
||||
// promise even when persistence failed, without masking the store error.
|
||||
if (value?.then) Promise.resolve(value).catch(() => {});
|
||||
report(error);
|
||||
return { status: 'unavailable', error };
|
||||
}
|
||||
},
|
||||
pausedUntil() {
|
||||
return current().pausedUntil || 0;
|
||||
},
|
||||
pauseUntil(at) {
|
||||
if (file) {
|
||||
try {
|
||||
at = markPause(at);
|
||||
} catch (error) {
|
||||
report(error);
|
||||
}
|
||||
}
|
||||
update((state) => ({ ...state, pausedUntil: at }));
|
||||
},
|
||||
status: () => {
|
||||
const dispatchPending = pendingDispatches().length > 0;
|
||||
return {
|
||||
file,
|
||||
durable: Boolean(file),
|
||||
persisted: Boolean(file) && !lastError && !dispatchPending,
|
||||
dispatchPending,
|
||||
error: lastError
|
||||
? String(lastError.message || lastError)
|
||||
: dispatchPending
|
||||
? 'unfinished durable Nominatim dispatch'
|
||||
: null,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a gate. `settings` may be a function so the environment is read
|
||||
* after the dev server loads `.env`.
|
||||
*
|
||||
* @param {object} [options]
|
||||
* @param {(() => object)|object} [options.settings]
|
||||
* @param {typeof fetch} [options.fetchImpl]
|
||||
* @param {() => number} [options.now]
|
||||
* @param {(ms: number) => Promise<void>} [options.sleep]
|
||||
* @param {ReturnType<typeof createGateStateStore>} [options.usage]
|
||||
*/
|
||||
export function createNominatimGate({
|
||||
settings = () => resolveNominatimSettings(),
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = Date.now,
|
||||
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
usage = createGateStateStore(),
|
||||
minSpacingMs = NOMINATIM_MIN_SPACING_MS,
|
||||
maxPending = NOMINATIM_MAX_PENDING,
|
||||
maxWaitMs = NOMINATIM_MAX_WAIT_MS,
|
||||
timeoutMs = 9000,
|
||||
} = {}) {
|
||||
let queue = Promise.resolve();
|
||||
let pending = 0;
|
||||
const pausedUntilByEndpoint = new Map();
|
||||
let upstreamRequests = 0;
|
||||
let usageStore = usage;
|
||||
|
||||
const current = () =>
|
||||
typeof settings === 'function' ? settings() : settings;
|
||||
|
||||
/**
|
||||
* Pause until `ms` from now. The public service's pause is also written to
|
||||
* the shared state so a restart or a second process honours it.
|
||||
*/
|
||||
function pause(ms, config) {
|
||||
const until = now() + Math.max(0, ms);
|
||||
const key = String(config?.endpoint || '');
|
||||
if (until > (pausedUntilByEndpoint.get(key) || 0))
|
||||
pausedUntilByEndpoint.set(key, until);
|
||||
if (config?.isPublic) usageStore.pauseUntil(until);
|
||||
}
|
||||
|
||||
function pauseEnd(config) {
|
||||
const endpointPause =
|
||||
pausedUntilByEndpoint.get(String(config?.endpoint || '')) || 0;
|
||||
return config?.isPublic
|
||||
? Math.max(endpointPause, usageStore.pausedUntil())
|
||||
: endpointPause;
|
||||
}
|
||||
|
||||
function pausedError(config) {
|
||||
return gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
|
||||
function admissionError(config) {
|
||||
if (!config.endpoint)
|
||||
return gateError('NOMINATIM_DISABLED', 'Place service is not configured');
|
||||
if (now() < pauseEnd(config)) return pausedError(config);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** One upstream request; classifies refusals and pauses the gate. */
|
||||
async function send(url, config, maxBytes, signal, queuedAt) {
|
||||
const controller = new AbortController();
|
||||
const onAbort = () => controller.abort();
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
// One deadline covers the headers AND the body: a server that answers its
|
||||
// headers and then stalls must not hold the shared queue.
|
||||
let timer;
|
||||
const start = () => {
|
||||
upstreamRequests += 1;
|
||||
timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
return fetchImpl(url, {
|
||||
headers: NOMINATIM_HEADERS,
|
||||
redirect: 'error',
|
||||
signal: controller.signal,
|
||||
});
|
||||
};
|
||||
try {
|
||||
return await exchange(
|
||||
config,
|
||||
maxBytes,
|
||||
signal,
|
||||
controller.signal,
|
||||
queuedAt,
|
||||
start,
|
||||
);
|
||||
} catch (error) {
|
||||
controller.abort();
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
async function exchange(
|
||||
config,
|
||||
maxBytes,
|
||||
signal,
|
||||
requestSignal,
|
||||
queuedAt,
|
||||
start,
|
||||
) {
|
||||
const turn = await takeTurn(config, queuedAt, signal, start);
|
||||
let response;
|
||||
try {
|
||||
response = await turn.value;
|
||||
} catch (cause) {
|
||||
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
||||
throw gateError('NOMINATIM_TRANSPORT', 'Place service unreachable', {
|
||||
cause,
|
||||
transport: true,
|
||||
});
|
||||
}
|
||||
if (!response.ok) {
|
||||
void response.body?.cancel?.().catch(() => {});
|
||||
const status = response.status;
|
||||
const retryAfter = parseRetryAfter(
|
||||
response.headers?.get?.('retry-after'),
|
||||
now(),
|
||||
);
|
||||
if (status === 429 || status === 503) {
|
||||
pause(retryAfter ?? DEFAULT_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_REFUSED', 'Place service asked to wait', {
|
||||
status,
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
if (status === 403 || status === 418) {
|
||||
pause(retryAfter ?? BLOCKED_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_REFUSED', 'Place service refused', {
|
||||
status,
|
||||
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
|
||||
});
|
||||
}
|
||||
if (status >= 500) pause(retryAfter ?? OUTAGE_BACKOFF_MS, config);
|
||||
throw gateError('NOMINATIM_UPSTREAM', `Upstream returned ${status}`, {
|
||||
status,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return await readResponseJsonCapped(response, maxBytes, requestSignal);
|
||||
} catch (cause) {
|
||||
if (cause?.code === 'RESPONSE_TOO_LARGE')
|
||||
throw gateError('NOMINATIM_TOO_LARGE', 'Place answer too large');
|
||||
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
|
||||
throw gateError('NOMINATIM_TRANSPORT', 'Place answer unreadable', {
|
||||
cause,
|
||||
transport: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Revalidate public dispatch under shared ownership after every wake. */
|
||||
async function takeTurn(config, queuedAt, signal, start) {
|
||||
while (true) {
|
||||
if (signal?.aborted || now() - queuedAt > maxWaitMs)
|
||||
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
|
||||
const refused = admissionError(config);
|
||||
if (refused) throw refused;
|
||||
// An operator-selected endpoint is not governed by the public instance's
|
||||
// shared budget, pacing, or durable-state requirement.
|
||||
if (!config.isPublic) {
|
||||
try {
|
||||
return { value: start() };
|
||||
} catch (error) {
|
||||
return { value: Promise.reject(error) };
|
||||
}
|
||||
}
|
||||
const reserved = usageStore.dispatch(
|
||||
utcDay(now()),
|
||||
{
|
||||
clock: now,
|
||||
dailyCap: config.dailyCap,
|
||||
minSpacingMs,
|
||||
latestStartAt: queuedAt + maxWaitMs,
|
||||
},
|
||||
start,
|
||||
);
|
||||
if (reserved.status === 'unavailable')
|
||||
throw gateError(
|
||||
'NOMINATIM_STATE_UNAVAILABLE',
|
||||
'Public place service usage state is unavailable',
|
||||
{ cause: reserved.error },
|
||||
);
|
||||
if (reserved.status === 'cap')
|
||||
throw gateError(
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
'Daily place lookup allowance is used',
|
||||
);
|
||||
if (reserved.status === 'paused')
|
||||
throw gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
|
||||
retryAfterMs: reserved.retryAfterMs,
|
||||
});
|
||||
if (reserved.status === 'abandoned')
|
||||
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
|
||||
if (reserved.status === 'started') return reserved;
|
||||
if (reserved.status === 'wait') await sleep(reserved.waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue one JSON request for `buildUrl(endpoint)`. At most one retry, and
|
||||
* only after a transport failure; a failed retry pauses the gate.
|
||||
*
|
||||
* @param {(endpoint: string) => string} buildUrl
|
||||
* @param {{signal?: AbortSignal, maxBytes?: number}} [options]
|
||||
*/
|
||||
function requestJson(buildUrl, { signal, maxBytes = 2 * 1024 * 1024 } = {}) {
|
||||
const config = current();
|
||||
const refused = admissionError(config);
|
||||
if (refused) return Promise.reject(refused);
|
||||
if (signal?.aborted)
|
||||
return Promise.reject(
|
||||
gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'),
|
||||
);
|
||||
if (pending >= maxPending)
|
||||
return Promise.reject(
|
||||
gateError('NOMINATIM_QUEUE_FULL', 'Place search queue is full'),
|
||||
);
|
||||
pending += 1;
|
||||
const queuedAt = now();
|
||||
const url = buildUrl(config.endpoint);
|
||||
const task = queue.then(async () => {
|
||||
try {
|
||||
try {
|
||||
return await send(url, config, maxBytes, signal, queuedAt);
|
||||
} catch (error) {
|
||||
if (!error?.transport) throw error;
|
||||
try {
|
||||
return await send(url, config, maxBytes, signal, now());
|
||||
} catch (retryError) {
|
||||
if (retryError?.transport) pause(OUTAGE_BACKOFF_MS, config);
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
pending -= 1;
|
||||
}
|
||||
});
|
||||
queue = task.catch(() => null);
|
||||
return task;
|
||||
}
|
||||
|
||||
return {
|
||||
requestJson,
|
||||
settings: current,
|
||||
/** Share public admission, daily count and pauses through one install's file. */
|
||||
persistUsage(file) {
|
||||
usageStore = createGateStateStore({ file });
|
||||
},
|
||||
stats: () => ({
|
||||
upstreamRequests,
|
||||
pending,
|
||||
pausedForMs: Math.max(0, pauseEnd(current()) - now()),
|
||||
usedToday: usageStore.count(utcDay(now())),
|
||||
persistence: usageStore.status?.() || null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
let sharedGate = createNominatimGate();
|
||||
|
||||
/** The process-wide gate: search and outlines share one budget and pacer. */
|
||||
export function sharedNominatimGate() {
|
||||
return sharedGate;
|
||||
}
|
||||
|
||||
/** Replace the process-wide gate (tests, embedders with their own transport). */
|
||||
export function setSharedNominatimGate(gate) {
|
||||
sharedGate = gate || createNominatimGate();
|
||||
return sharedGate;
|
||||
}
|
||||
|
||||
/** Codes that mean "not now" rather than "no such place". */
|
||||
export function isNominatimBusy(error) {
|
||||
return [
|
||||
'NOMINATIM_QUEUE_FULL',
|
||||
'NOMINATIM_ABANDONED',
|
||||
'NOMINATIM_PAUSED',
|
||||
'NOMINATIM_REFUSED',
|
||||
'NOMINATIM_DAILY_CAP',
|
||||
].includes(error?.code);
|
||||
}
|
||||
@@ -0,0 +1,654 @@
|
||||
import { normalizePlaceName } from './nominatimCache.js';
|
||||
import {
|
||||
containsAnchor,
|
||||
pointInRing,
|
||||
} from '../../../src/data/adminBoundaries.js';
|
||||
import {
|
||||
douglasPeucker,
|
||||
ringAreaM2,
|
||||
} from '../../../src/sources/featureGeometry.js';
|
||||
|
||||
/**
|
||||
* Which Nominatim results may answer which voice ask, and how their polygons
|
||||
* are bounded. Pure: no I/O.
|
||||
*
|
||||
* A search returns whatever matches the words best. "Mission District" can
|
||||
* come back as a car-charging station and "Bandra" as a railway station, so a
|
||||
* result is accepted only when its OSM class (and, for broad classes, its
|
||||
* type) suits the kind of thing that was asked for, it carries a polygon, and
|
||||
* its name actually answers the words.
|
||||
*/
|
||||
|
||||
/** Ask kinds the outline route answers. */
|
||||
export const OUTLINE_KINDS = Object.freeze([
|
||||
'city',
|
||||
'admin',
|
||||
'neighborhood',
|
||||
'landmark',
|
||||
'building',
|
||||
]);
|
||||
|
||||
/** Simplification tolerance (degrees) sent upstream, per ask kind. */
|
||||
export const OUTLINE_POLYGON_THRESHOLD = Object.freeze({
|
||||
city: 0.0005,
|
||||
admin: 0.0005,
|
||||
neighborhood: 0.0005,
|
||||
landmark: 0.00005,
|
||||
building: 0.00005,
|
||||
});
|
||||
|
||||
/**
|
||||
* Size limits for one outline. Vertices beyond the budget are simplified away;
|
||||
* more parts, or more holes in one part, than these limits refuse the outline
|
||||
* whole, because a truncated multipart place would misrepresent it.
|
||||
*/
|
||||
const OUTLINE_MAX_VERTICES = 6000;
|
||||
const OUTLINE_MAX_PARTS = 24;
|
||||
const OUTLINE_MAX_HOLES = 16;
|
||||
|
||||
const ANY = '*';
|
||||
|
||||
const SETTLEMENT_PLACES = [
|
||||
'city',
|
||||
'town',
|
||||
'village',
|
||||
'municipality',
|
||||
'borough',
|
||||
'hamlet',
|
||||
];
|
||||
|
||||
const CIVIC_AMENITIES = [
|
||||
'university',
|
||||
'college',
|
||||
'school',
|
||||
'kindergarten',
|
||||
'hospital',
|
||||
'clinic',
|
||||
'place_of_worship',
|
||||
'monastery',
|
||||
'townhall',
|
||||
'courthouse',
|
||||
'library',
|
||||
'theatre',
|
||||
'cinema',
|
||||
'arts_centre',
|
||||
'community_centre',
|
||||
'conference_centre',
|
||||
'events_venue',
|
||||
'exhibition_centre',
|
||||
'music_venue',
|
||||
'planetarium',
|
||||
'marketplace',
|
||||
'prison',
|
||||
'grave_yard',
|
||||
'fire_station',
|
||||
'police',
|
||||
'embassy',
|
||||
'public_building',
|
||||
];
|
||||
|
||||
const LANDMARK_TOURISM = [
|
||||
'attraction',
|
||||
'theme_park',
|
||||
'zoo',
|
||||
'aquarium',
|
||||
'museum',
|
||||
'gallery',
|
||||
'viewpoint',
|
||||
'camp_site',
|
||||
'hotel',
|
||||
'resort',
|
||||
];
|
||||
|
||||
const MAN_MADE_STRUCTURES = [
|
||||
'tower',
|
||||
'lighthouse',
|
||||
'pier',
|
||||
'bridge',
|
||||
'works',
|
||||
'dam',
|
||||
'observatory',
|
||||
'wastewater_plant',
|
||||
'water_works',
|
||||
'breakwater',
|
||||
];
|
||||
|
||||
/**
|
||||
* Accepted `class → types` per ask kind. `*` accepts every type of that
|
||||
* class. Anything absent is rejected: roads, railways, stations, shops,
|
||||
* guideposts and single facilities never answer an area ask.
|
||||
*/
|
||||
const OUTLINE_ACCEPT = Object.freeze({
|
||||
city: {
|
||||
boundary: ['administrative'],
|
||||
place: SETTLEMENT_PLACES,
|
||||
},
|
||||
admin: {
|
||||
boundary: ['administrative'],
|
||||
place: ['country', 'state', 'province', 'region', 'county', 'district'],
|
||||
},
|
||||
neighborhood: {
|
||||
boundary: ['administrative'],
|
||||
place: ['suburb', 'quarter', 'neighbourhood', 'borough', 'city_block'],
|
||||
},
|
||||
landmark: {
|
||||
leisure: ANY,
|
||||
landuse: ANY,
|
||||
amenity: CIVIC_AMENITIES,
|
||||
tourism: LANDMARK_TOURISM,
|
||||
building: ANY,
|
||||
natural: ANY,
|
||||
water: ANY,
|
||||
waterway: ['riverbank', 'dock'],
|
||||
historic: ANY,
|
||||
man_made: MAN_MADE_STRUCTURES,
|
||||
office: ['government', 'diplomatic'],
|
||||
aeroway: ['aerodrome', 'heliport', 'terminal'],
|
||||
military: ANY,
|
||||
boundary: ['national_park', 'protected_area'],
|
||||
place: ['island', 'islet', 'square'],
|
||||
},
|
||||
// Single structures only: no campuses, grounds, parks or camp sites.
|
||||
building: {
|
||||
building: ANY,
|
||||
amenity: [
|
||||
'place_of_worship',
|
||||
'townhall',
|
||||
'courthouse',
|
||||
'library',
|
||||
'theatre',
|
||||
'cinema',
|
||||
'arts_centre',
|
||||
'community_centre',
|
||||
'conference_centre',
|
||||
'exhibition_centre',
|
||||
'music_venue',
|
||||
'planetarium',
|
||||
'fire_station',
|
||||
'police',
|
||||
'embassy',
|
||||
'public_building',
|
||||
],
|
||||
tourism: ['museum', 'gallery', 'hotel', 'attraction'],
|
||||
historic: ['building', 'castle', 'church', 'fort', 'palace', 'monastery'],
|
||||
man_made: ['tower', 'lighthouse', 'observatory'],
|
||||
office: ANY,
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Administrative evidence per ask kind: a recognized address type is required.
|
||||
* A supplied numeric place_rank must also fit the ask's broad band (country 4,
|
||||
* state 8, county 10-12, city 12-16, suburb 18-20, neighbourhood 20-22).
|
||||
*/
|
||||
const ADMIN_EVIDENCE = Object.freeze({
|
||||
city: {
|
||||
types: ['city', 'town', 'village', 'municipality', 'borough', 'hamlet'],
|
||||
ranks: [12, 18],
|
||||
},
|
||||
admin: {
|
||||
types: [
|
||||
'country',
|
||||
'state',
|
||||
'province',
|
||||
'region',
|
||||
'county',
|
||||
'district',
|
||||
'state_district',
|
||||
],
|
||||
ranks: [2, 14],
|
||||
},
|
||||
neighborhood: {
|
||||
types: ['suburb', 'quarter', 'neighbourhood', 'city_district', 'borough'],
|
||||
ranks: [17, 24],
|
||||
},
|
||||
});
|
||||
|
||||
/** Whether supplied administrative evidence contradicts the accepted identity. */
|
||||
function adminLevelFits(kind, row) {
|
||||
const evidence = ADMIN_EVIDENCE[kind];
|
||||
if (!evidence) return true;
|
||||
const place = String(row?.category || row?.class).toLowerCase() === 'place';
|
||||
// A place row already has an accepted type. Missing address evidence must
|
||||
// not erase it, but explicit contradictory evidence must not bypass checks.
|
||||
if (!place || Object.hasOwn(row, 'addresstype')) {
|
||||
if (typeof row?.addresstype !== 'string') return false;
|
||||
const type = row.addresstype.toLowerCase();
|
||||
if (
|
||||
!evidence.types.includes(type) &&
|
||||
!(place && type === String(row.type).toLowerCase())
|
||||
)
|
||||
return false;
|
||||
}
|
||||
// Rank can corroborate or contradict an explicit identity, never invent it.
|
||||
// JSONv2 supplies a number; coercing arrays or strings hides malformed data.
|
||||
if (!Object.hasOwn(row, 'place_rank')) return true;
|
||||
const rank = row.place_rank;
|
||||
return (
|
||||
Number.isInteger(rank) &&
|
||||
// Boundary rank bands do not describe every accepted place type.
|
||||
(place || (rank >= evidence.ranks[0] && rank <= evidence.ranks[1]))
|
||||
);
|
||||
}
|
||||
|
||||
/** Largest footprint (m²) a non-`building` result may have for a building ask. */
|
||||
const BUILDING_MAX_AREA_M2 = 150_000;
|
||||
|
||||
/** Whether a result's class/type suits the ask kind. */
|
||||
export function acceptsOutlineClass(kind, osmClass, osmType) {
|
||||
const rules = OUTLINE_ACCEPT[kind];
|
||||
if (!rules) return false;
|
||||
const types = rules[String(osmClass || '').toLowerCase()];
|
||||
if (!types) return false;
|
||||
return types === ANY || types.includes(String(osmType || '').toLowerCase());
|
||||
}
|
||||
|
||||
const STOP_WORDS = new Set(['the', 'of', 'de', 'la', 'le', 'el', 'and']);
|
||||
|
||||
/** Scripts written without spaces between words. */
|
||||
const UNSPACED_SCRIPT_RE =
|
||||
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
|
||||
|
||||
/** Significant words of the part of a query before the first comma. */
|
||||
function queryWords(query) {
|
||||
const head = String(query ?? '').split(',')[0];
|
||||
return normalizePlaceName(head)
|
||||
.split(' ')
|
||||
.filter((word) => word && !STOP_WORDS.has(word));
|
||||
}
|
||||
|
||||
/**
|
||||
* Words naming what a building is, which OSM rarely carries in the name or
|
||||
* type ("Burj Khalifa", building=yes). Enclosures (zoo, garden, arch,
|
||||
* conservatory) are left out: those name a feature inside a park.
|
||||
*/
|
||||
const BUILDING_WORDS = [
|
||||
'building',
|
||||
'tower',
|
||||
'church',
|
||||
'cathedral',
|
||||
'basilica',
|
||||
'chapel',
|
||||
'temple',
|
||||
'mosque',
|
||||
'synagogue',
|
||||
'shrine',
|
||||
'museum',
|
||||
'palace',
|
||||
'castle',
|
||||
'hall',
|
||||
'library',
|
||||
];
|
||||
|
||||
/** How well a result's names answer the ask. */
|
||||
const NAME_MISS = 0;
|
||||
const NAME_OVERLAP = 1;
|
||||
const NAME_WHOLE = 2;
|
||||
const NAME_EXACT = 3;
|
||||
|
||||
/**
|
||||
* Settlements require one complete normalized name or explicit returned alias.
|
||||
* Landmark/building asks retain their word-overlap and unspaced-script
|
||||
* shorthand. An exact name ranks first, then one holding every asked word,
|
||||
* then an overlap.
|
||||
*/
|
||||
function outlineNameMatch(row, query, kind) {
|
||||
const names = new Set();
|
||||
const add = (value) => {
|
||||
const name = normalizePlaceName(value);
|
||||
if (name) names.add(name);
|
||||
};
|
||||
add(row?.name);
|
||||
add(String(row?.display_name || '').split(',')[0]);
|
||||
// OSM name tags may hold several names separated by ";" ("York;Eboracum").
|
||||
if (row?.namedetails && typeof row.namedetails === 'object')
|
||||
for (const value of Object.values(row.namedetails))
|
||||
for (const part of String(value ?? '').split(';')) add(part);
|
||||
if (kind === 'city' || kind === 'admin' || kind === 'neighborhood') {
|
||||
const head = normalizePlaceName(String(query ?? '').split(',')[0]);
|
||||
return head && names.has(head) ? NAME_EXACT : NAME_MISS;
|
||||
}
|
||||
const words = queryWords(query);
|
||||
if (!words.length) return NAME_MISS;
|
||||
const whole = words.join(' ');
|
||||
const unspaced = UNSPACED_SCRIPT_RE.test(whole);
|
||||
// The last word usually names the feature ("Zoo", "Arch"): a name without
|
||||
// it is the place around the feature ("Central Park"), unless the word is
|
||||
// where the result is ("Space Needle Seattle"), what it is ("… Bridge"), or
|
||||
// a building word on a single-structure result ("Burj Khalifa Tower").
|
||||
const context = new Set(
|
||||
normalizePlaceName(
|
||||
[
|
||||
String(row?.display_name || '')
|
||||
.split(',')
|
||||
.slice(1)
|
||||
.join(' '),
|
||||
row?.type,
|
||||
row?.category || row?.class,
|
||||
]
|
||||
.join(' ')
|
||||
.replace(/_/g, ' '),
|
||||
).split(' '),
|
||||
);
|
||||
// Only a single structure, as a building ask would accept: never a campus,
|
||||
// zoo or district whose name merely starts the ask ("Duke University").
|
||||
if (acceptsOutlineClass('building', row?.category || row?.class, row?.type))
|
||||
for (const word of BUILDING_WORDS) context.add(word);
|
||||
let match = NAME_MISS;
|
||||
for (const name of names) {
|
||||
if (
|
||||
name
|
||||
.split(' ')
|
||||
.filter((word) => !STOP_WORDS.has(word))
|
||||
.join(' ') === whole
|
||||
)
|
||||
return NAME_EXACT;
|
||||
// Scripts without spaces compare by characters instead.
|
||||
if (unspaced && name.replace(/ /g, '').includes(whole.replace(/ /g, '')))
|
||||
match = NAME_WHOLE;
|
||||
// Whole words only: "York" never matches "Yorkshire Park".
|
||||
const nameSet = new Set(name.split(' '));
|
||||
const hits = words.filter((word) => nameSet.has(word)).length;
|
||||
if (hits === words.length) match = NAME_WHOLE;
|
||||
const last = words.at(-1);
|
||||
if (
|
||||
words.length > 1 &&
|
||||
hits >= Math.ceil(words.length * 0.6) &&
|
||||
(unspaced || nameSet.has(last) || context.has(last))
|
||||
)
|
||||
match = Math.max(match, NAME_OVERLAP);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
const finiteLon = (value) =>
|
||||
Number.isFinite(value) && value >= -180 && value <= 180;
|
||||
const finiteLat = (value) =>
|
||||
Number.isFinite(value) && value >= -90 && value <= 90;
|
||||
|
||||
/** A valid closed ring of [lon, lat] numbers, or null. */
|
||||
function cleanRing(ring) {
|
||||
if (!Array.isArray(ring)) return null;
|
||||
const out = [];
|
||||
for (const point of ring) {
|
||||
if (!Array.isArray(point)) return null;
|
||||
const [lon, lat] = point;
|
||||
if (!finiteLon(lon) || !finiteLat(lat)) return null;
|
||||
out.push([lon, lat]);
|
||||
}
|
||||
if (out.length < 4) return null;
|
||||
const [first, last] = [out[0], out.at(-1)];
|
||||
if (first[0] !== last[0] || first[1] !== last[1]) out.push([...first]);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Polygon/MultiPolygon GeoJSON → `[[outer, ...holes], …]`, largest first. */
|
||||
function polygonsFromGeoJson(geometry) {
|
||||
const raw =
|
||||
geometry?.type === 'Polygon'
|
||||
? [geometry.coordinates]
|
||||
: geometry?.type === 'MultiPolygon'
|
||||
? geometry.coordinates
|
||||
: null;
|
||||
if (!Array.isArray(raw) || !raw.length || raw.length > OUTLINE_MAX_PARTS)
|
||||
return null;
|
||||
const polygons = [];
|
||||
for (const polygon of raw) {
|
||||
if (
|
||||
!Array.isArray(polygon) ||
|
||||
!polygon.length ||
|
||||
polygon.length - 1 > OUTLINE_MAX_HOLES
|
||||
)
|
||||
return null;
|
||||
const outer = cleanRing(polygon[0]);
|
||||
if (!outer) return null;
|
||||
const holes = polygon.slice(1).map(cleanRing);
|
||||
if (holes.some((hole) => !hole)) return null;
|
||||
polygons.push({ rings: [outer, ...holes], area: ringAreaM2(outer) });
|
||||
}
|
||||
polygons.sort((a, b) => b.area - a.area);
|
||||
return polygons.map((entry) => entry.rings);
|
||||
}
|
||||
|
||||
/** Iterative Douglas–Peucker on a closed ring (keeps it closed, ≥ 4 points). */
|
||||
function simplifyClosedRing(ring, tolerance) {
|
||||
if (ring.length <= 8 || tolerance <= 0) return ring;
|
||||
const out = douglasPeucker(ring, tolerance);
|
||||
return out.length >= 4 ? out : ring;
|
||||
}
|
||||
|
||||
const countVertices = (polygons) =>
|
||||
polygons.reduce(
|
||||
(sum, rings) => sum + rings.reduce((n, ring) => n + ring.length, 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const edgeSide = (p, q, r) =>
|
||||
(q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0]);
|
||||
const onEdge = (p, q, r) =>
|
||||
r[0] >= Math.min(p[0], q[0]) &&
|
||||
r[0] <= Math.max(p[0], q[0]) &&
|
||||
r[1] >= Math.min(p[1], q[1]) &&
|
||||
r[1] <= Math.max(p[1], q[1]);
|
||||
|
||||
/** Intersection, including touching, of two planar closed-ring edges. */
|
||||
function edgesIntersect(a, b, c, d) {
|
||||
if (
|
||||
Math.max(a[0], b[0]) < Math.min(c[0], d[0]) ||
|
||||
Math.max(c[0], d[0]) < Math.min(a[0], b[0]) ||
|
||||
Math.max(a[1], b[1]) < Math.min(c[1], d[1]) ||
|
||||
Math.max(c[1], d[1]) < Math.min(a[1], b[1])
|
||||
)
|
||||
return false;
|
||||
const ac = edgeSide(a, b, c);
|
||||
const ad = edgeSide(a, b, d);
|
||||
const ca = edgeSide(c, d, a);
|
||||
const cb = edgeSide(c, d, b);
|
||||
return (
|
||||
(ac === 0 && onEdge(a, b, c)) ||
|
||||
(ad === 0 && onEdge(a, b, d)) ||
|
||||
(ca === 0 && onEdge(c, d, a)) ||
|
||||
(cb === 0 && onEdge(c, d, b)) ||
|
||||
(ac > 0 !== ad > 0 && ca > 0 !== cb > 0)
|
||||
);
|
||||
}
|
||||
|
||||
function ringsIntersect(a, b) {
|
||||
for (let i = 1; i < a.length; i++)
|
||||
for (let j = 1; j < b.length; j++)
|
||||
if (edgesIntersect(a[i - 1], a[i], b[j - 1], b[j])) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check only generated bounded output, not a new input-geometry policy.
|
||||
* Touching generated boundaries are conservatively unsupported here.
|
||||
*/
|
||||
function simplifiedTopologyFits(polygons) {
|
||||
for (const [outer, ...holes] of polygons) {
|
||||
for (const ring of [outer, ...holes]) {
|
||||
if (ringAreaM2(ring) === 0) return false;
|
||||
for (let i = 1; i < ring.length; i++)
|
||||
for (let j = i + 2; j < ring.length; j++) {
|
||||
if (i === 1 && j === ring.length - 1) continue;
|
||||
if (edgesIntersect(ring[i - 1], ring[i], ring[j - 1], ring[j]))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < holes.length; i++) {
|
||||
const hole = holes[i];
|
||||
if (
|
||||
!hole.every(([lon, lat]) => pointInRing(outer, lat, lon)) ||
|
||||
ringsIntersect(outer, hole)
|
||||
)
|
||||
return false;
|
||||
for (let j = 0; j < i; j++) {
|
||||
const other = holes[j];
|
||||
if (
|
||||
ringsIntersect(hole, other) ||
|
||||
pointInRing(other, hole[0][1], hole[0][0]) ||
|
||||
pointInRing(hole, other[0][1], other[0][0])
|
||||
)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Simplify until the whole outline fits the vertex cap. */
|
||||
export function capOutlineVertices(
|
||||
polygons,
|
||||
maxVertices = OUTLINE_MAX_VERTICES,
|
||||
) {
|
||||
let current = polygons;
|
||||
let tolerance = 0.00002;
|
||||
for (let round = 0; round < 12; round++) {
|
||||
if (countVertices(current) <= maxVertices)
|
||||
return current === polygons || simplifiedTopologyFits(current)
|
||||
? current
|
||||
: null;
|
||||
current = polygons.map((rings) =>
|
||||
rings.map((ring) => simplifyClosedRing(ring, tolerance)),
|
||||
);
|
||||
tolerance *= 2;
|
||||
}
|
||||
// Never turn a resource refusal into an apparently complete partial outline.
|
||||
return countVertices(current) <= maxVertices &&
|
||||
simplifiedTopologyFits(current)
|
||||
? current
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every result within the limit that suits the ask, names the place and
|
||||
* carries a polygon — best name match first, then upstream order — with the reason each other result was passed over. `outline` is
|
||||
* the first of them, or null.
|
||||
*
|
||||
* @param {object[]} rows Nominatim jsonv2 results, in upstream order.
|
||||
* @param {{kind: string, query: string}} ask
|
||||
*/
|
||||
export function selectOutlineResult(rows, { kind, query }) {
|
||||
const skipped = [];
|
||||
const found = [];
|
||||
for (const row of Array.isArray(rows) ? rows : []) {
|
||||
const osmClass = row?.category || row?.class;
|
||||
const osmType = row?.type;
|
||||
const label = `${osmClass}=${osmType}`;
|
||||
if (!acceptsOutlineClass(kind, osmClass, osmType)) {
|
||||
skipped.push(`${label}: wrong type`);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
(String(osmClass).toLowerCase() === 'place' ||
|
||||
(String(osmClass).toLowerCase() === 'boundary' &&
|
||||
String(osmType).toLowerCase() === 'administrative')) &&
|
||||
!adminLevelFits(kind, row)
|
||||
) {
|
||||
skipped.push(`${label}: level`);
|
||||
continue;
|
||||
}
|
||||
const nameMatch = outlineNameMatch(row, query, kind);
|
||||
if (nameMatch === NAME_MISS) {
|
||||
skipped.push(`${label}: name`);
|
||||
continue;
|
||||
}
|
||||
const parsed = polygonsFromGeoJson(row?.geojson);
|
||||
if (!parsed) {
|
||||
skipped.push(`${label}: no polygon`);
|
||||
continue;
|
||||
}
|
||||
// A building ask needs building evidence: the building class, or a
|
||||
// footprint no larger than a large building.
|
||||
if (
|
||||
kind === 'building' &&
|
||||
String(osmClass).toLowerCase() !== 'building' &&
|
||||
ringAreaM2(parsed[0][0]) > BUILDING_MAX_AREA_M2
|
||||
) {
|
||||
skipped.push(`${label}: not a building`);
|
||||
continue;
|
||||
}
|
||||
const polygons = capOutlineVertices(parsed);
|
||||
if (!polygons) {
|
||||
skipped.push(`${label}: too large`);
|
||||
continue;
|
||||
}
|
||||
found.push({
|
||||
nameMatch,
|
||||
outline: {
|
||||
name: String(row.name || '').slice(0, 160) || null,
|
||||
displayName: String(row.display_name || '').slice(0, 300) || null,
|
||||
class: String(osmClass),
|
||||
type: String(osmType),
|
||||
osm:
|
||||
row.osm_type && row.osm_id != null
|
||||
? `${row.osm_type}/${row.osm_id}`
|
||||
: null,
|
||||
polygons,
|
||||
},
|
||||
});
|
||||
}
|
||||
// Stable: equal matches keep upstream order.
|
||||
found.sort((a, b) => b.nameMatch - a.nameMatch);
|
||||
const outlines = found.map((entry) => entry.outline);
|
||||
const ranks = found.map((entry) => entry.nameMatch);
|
||||
return { outline: outlines[0] || null, outlines, ranks, skipped };
|
||||
}
|
||||
|
||||
/**
|
||||
* Among the best-named outlines (`ranks` from selectOutlineResult; all of
|
||||
* them without), the one holding `lat, lon` (the resolved anchor), else the
|
||||
* first. A worse name never wins by containment alone: an exact building
|
||||
* that misses a pin by metres is not replaced by the plaza around it.
|
||||
* Without an anchor the first answers, as before.
|
||||
*/
|
||||
export function outlineAt(outlines, lat, lon, ranks = null) {
|
||||
const list = Array.isArray(outlines) ? outlines : [];
|
||||
if (!finiteLat(lat) || !finiteLon(lon)) return list[0] || null;
|
||||
const best = Array.isArray(ranks) ? ranks[0] : undefined;
|
||||
return (
|
||||
list.find(
|
||||
(outline, index) =>
|
||||
(best === undefined || ranks[index] === best) &&
|
||||
containsAnchor(outline.polygons, lat, lon),
|
||||
) ||
|
||||
list[0] ||
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Coarse view bias: the centre is snapped to a grid (1° for places, 0.05°
|
||||
* for small grounds) so small camera moves reuse the same cache entry, and
|
||||
* the viewbox sent upstream is built from the snapped centre.
|
||||
*/
|
||||
export function outlineBias(kind, lat, lon) {
|
||||
// Missing is missing: Number(null) and Number('') would read as (0, 0).
|
||||
const given = (value) =>
|
||||
value !== null &&
|
||||
value !== undefined &&
|
||||
(typeof value === 'number' || String(value).trim() !== '');
|
||||
if (!given(lat) || !given(lon)) return null;
|
||||
const latitude = Number(lat);
|
||||
const longitude = Number(lon);
|
||||
if (!finiteLat(latitude) || !finiteLon(longitude)) return null;
|
||||
const small = kind === 'landmark' || kind === 'building';
|
||||
const step = small ? 0.05 : 1;
|
||||
const half = small ? 0.25 : 2;
|
||||
const snap = (value) => Math.round(value / step) * step;
|
||||
const cLat = Number(snap(latitude).toFixed(2));
|
||||
const cLon = Number(snap(longitude).toFixed(2));
|
||||
const clampLat = (value) => Math.max(-90, Math.min(90, value));
|
||||
const clampLon = (value) => Math.max(-180, Math.min(180, value));
|
||||
return {
|
||||
key: `${cLat},${cLon}`,
|
||||
viewbox: [
|
||||
clampLon(cLon - half),
|
||||
clampLat(cLat + half),
|
||||
clampLon(cLon + half),
|
||||
clampLat(cLat - half),
|
||||
]
|
||||
.map((value) => Number(value.toFixed(4)))
|
||||
.join(','),
|
||||
};
|
||||
}
|
||||
+451
-196
@@ -1,114 +1,140 @@
|
||||
import path from 'node:path';
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
import { fetchRegionalJson } from './http.js';
|
||||
import { admitSameSite } from '../common/same-site.js';
|
||||
import { defaultSourceRoot } from '../common/source-root.js';
|
||||
import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
|
||||
import {
|
||||
nominatimToGeocodeResult,
|
||||
nominatimViewboxFromBounds,
|
||||
} from '../../../src/nominatimGeocode.js';
|
||||
|
||||
/**
|
||||
* The usage policy for the public Nominatim instance asks for a User-Agent or
|
||||
* Referer that identifies the application, and states that stock library
|
||||
* agents will not do. Both are sent.
|
||||
*/
|
||||
const NOMINATIM_HEADERS = Object.freeze({
|
||||
'User-Agent':
|
||||
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
|
||||
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
|
||||
});
|
||||
|
||||
/**
|
||||
* Minimum spacing between upstream calls. The policy states an absolute
|
||||
* maximum of one request per second; 1.1 s keeps clock jitter from crossing it.
|
||||
*/
|
||||
const NOMINATIM_MIN_SPACING_MS = 1100;
|
||||
|
||||
/**
|
||||
* How many searches may be waiting for their turn.
|
||||
*
|
||||
* One request per second and an unbounded queue are incompatible: a burst of
|
||||
* searches would keep the upstream busy long after everyone who asked has given
|
||||
* up, which is exactly the load the policy asks callers not to create. Past
|
||||
* this depth a search is refused at once instead of being promised a slot
|
||||
* minutes away.
|
||||
*/
|
||||
const NOMINATIM_MAX_PENDING = 4;
|
||||
|
||||
/**
|
||||
* How long a queued search may wait before it is not worth sending. The
|
||||
* browser gives up well before this, so anything reaching the front later than
|
||||
* this is answering nobody.
|
||||
*/
|
||||
const NOMINATIM_MAX_WAIT_MS = 10_000;
|
||||
|
||||
const NOMINATIM_SEARCH_CACHE_MS = 5 * 60_000;
|
||||
|
||||
const NOMINATIM_SEARCH_MAX_CACHE = 80;
|
||||
import {
|
||||
NOMINATIM_MAX_PENDING,
|
||||
isNominatimBusy,
|
||||
sharedNominatimGate,
|
||||
} from './nominatimGate.js';
|
||||
import {
|
||||
NOMINATIM_CACHE_TTL_MS,
|
||||
createNominatimCache,
|
||||
} from './nominatimCache.js';
|
||||
import {
|
||||
OUTLINE_KINDS,
|
||||
OUTLINE_POLYGON_THRESHOLD,
|
||||
outlineAt,
|
||||
outlineBias,
|
||||
selectOutlineResult,
|
||||
} from './outlineSelect.js';
|
||||
|
||||
const NOMINATIM_SEARCH_MAX_QUERY = 200;
|
||||
|
||||
// The pacer is deliberately module state while everything else here is
|
||||
// per-instance. One request per second is a budget for the whole application,
|
||||
// not for each provider object: two instances each pacing themselves would
|
||||
// send two requests a second between them. Only last-resort forward searches
|
||||
// use this queue.
|
||||
let _nominatimQueue = Promise.resolve();
|
||||
/** Results considered per outline ask: enough to skip a node or a wrong type. */
|
||||
const OUTLINE_RESULT_LIMIT = 3;
|
||||
|
||||
let _nominatimLastRequestAt = 0;
|
||||
/** Largest upstream answer read for an outline (polygons are simplified upstream). */
|
||||
const OUTLINE_MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
let _nominatimPending = 0;
|
||||
/** Bump when cached answers change shape or selection authority. */
|
||||
const CACHE_SCHEMA = 'v3';
|
||||
|
||||
/** Raised when the queue is already as deep as it is allowed to get. */
|
||||
function queueFullError() {
|
||||
return Object.assign(new Error('Place search queue is full'), {
|
||||
code: 'NOMINATIM_QUEUE_FULL',
|
||||
/**
|
||||
* The query text actually sent upstream: Unicode-composed, whitespace
|
||||
* collapsed. Nothing else is changed, so two queries share a cache entry only
|
||||
* when they are the same upstream request.
|
||||
*/
|
||||
export function upstreamQuery(query) {
|
||||
return String(query ?? '')
|
||||
.normalize('NFC')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Cache key: endpoint, schema and the exact upstream parameters. */
|
||||
function requestKey(gate, kind, params) {
|
||||
const endpoint = gate.settings?.()?.endpoint || 'none';
|
||||
return `${CACHE_SCHEMA}|${endpoint}|${kind}|${params}`;
|
||||
}
|
||||
|
||||
/** A decimal coordinate query value, or NaN: never "" or " " as 0. */
|
||||
function coordinateParam(value) {
|
||||
return /^-?\d+(\.\d+)?$/.test(value ?? '') ? Number(value) : NaN;
|
||||
}
|
||||
|
||||
function malformed() {
|
||||
return Object.assign(new Error('Place answer was not understood'), {
|
||||
code: 'NOMINATIM_MALFORMED',
|
||||
});
|
||||
}
|
||||
|
||||
/** Raised when a queued search waited so long that nobody is left to answer. */
|
||||
function abandonedError() {
|
||||
function abandoned() {
|
||||
return Object.assign(new Error('Place search was abandoned'), {
|
||||
code: 'NOMINATIM_ABANDONED',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one piece of upstream work, never closer than the policy spacing to the
|
||||
* last one.
|
||||
*
|
||||
* `bounded` applies queue-depth and staleness limits to interactive forward searches.
|
||||
*
|
||||
* @param {() => Promise<unknown>} work
|
||||
* @param {{bounded?: boolean, signal?: AbortSignal}} [options]
|
||||
* Share one upstream call while keeping each caller's cancellation independent.
|
||||
* The shared controller is aborted only after the last subscriber leaves.
|
||||
*/
|
||||
function enqueueNominatim(work, { bounded = false, signal } = {}) {
|
||||
if (bounded && _nominatimPending >= NOMINATIM_MAX_PENDING)
|
||||
return Promise.reject(queueFullError());
|
||||
if (bounded) _nominatimPending += 1;
|
||||
const queuedAt = Date.now();
|
||||
const task = _nominatimQueue.then(async () => {
|
||||
try {
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
NOMINATIM_MIN_SPACING_MS - (Date.now() - _nominatimLastRequestAt),
|
||||
);
|
||||
if (waitMs) await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
// Nobody is waiting for this any more: sending it would spend the one
|
||||
// request per second the policy allows on an answer with no reader.
|
||||
if (bounded && Date.now() - queuedAt > NOMINATIM_MAX_WAIT_MS)
|
||||
throw abandonedError();
|
||||
if (signal?.aborted) throw abandonedError();
|
||||
_nominatimLastRequestAt = Date.now();
|
||||
return await work();
|
||||
} finally {
|
||||
if (bounded) _nominatimPending -= 1;
|
||||
}
|
||||
function subscribeSharedRequest(inFlight, key, signal, create) {
|
||||
if (signal?.aborted) return Promise.reject(abandoned());
|
||||
let flight = inFlight.get(key);
|
||||
if (!flight) {
|
||||
flight = {
|
||||
controller: new AbortController(),
|
||||
users: 0,
|
||||
settled: false,
|
||||
promise: null,
|
||||
};
|
||||
const owned = flight;
|
||||
owned.promise = Promise.resolve()
|
||||
.then(() => create(owned.controller.signal))
|
||||
.finally(() => {
|
||||
owned.settled = true;
|
||||
if (inFlight.get(key) === owned) inFlight.delete(key);
|
||||
});
|
||||
inFlight.set(key, owned);
|
||||
}
|
||||
flight.users += 1;
|
||||
return new Promise((resolve, reject) => {
|
||||
let left = false;
|
||||
const leave = () => {
|
||||
if (left) return;
|
||||
left = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
flight.users -= 1;
|
||||
if (
|
||||
flight.users === 0 &&
|
||||
!flight.settled &&
|
||||
inFlight.get(key) === flight
|
||||
) {
|
||||
inFlight.delete(key);
|
||||
flight.controller.abort();
|
||||
}
|
||||
};
|
||||
const onAbort = () => {
|
||||
leave();
|
||||
reject(abandoned());
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
flight.promise.then(
|
||||
(value) => {
|
||||
leave();
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
leave();
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
_nominatimQueue = task.catch(() => null);
|
||||
return task;
|
||||
}
|
||||
|
||||
/** Where the dev server keeps Nominatim answers and the daily count. */
|
||||
export const NOMINATIM_CACHE_DIR = path.join(
|
||||
defaultSourceRoot,
|
||||
'.gev-cache',
|
||||
'nominatim',
|
||||
);
|
||||
|
||||
/** Construct the offline regional context provider from bundled Natural Earth polygons. */
|
||||
export function createRegionalPlaceProvider() {
|
||||
return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
|
||||
@@ -117,137 +143,366 @@ export function createRegionalPlaceProvider() {
|
||||
export const fetchRegionalPlace = createRegionalPlaceProvider();
|
||||
|
||||
/**
|
||||
* Construct the forward search adapter with a trusted endpoint.
|
||||
* Snap a `"swLat,swLng|neLat,neLng"` view to a coarse grid so small camera
|
||||
* moves reuse one cache entry. The viewbox is a soft preference upstream.
|
||||
*/
|
||||
function coarseBounds(bounds) {
|
||||
// Validate the original hint before snapping or clamping can disguise a
|
||||
// wrapped, inverted or out-of-range box as an unrelated valid viewbox.
|
||||
if (!nominatimViewboxFromBounds(bounds)) return null;
|
||||
const match = String(bounds || '')
|
||||
.trim()
|
||||
.split('|')
|
||||
.map((corner) => corner.split(',').map(Number));
|
||||
if (match.length !== 2 || match.some((c) => c.length !== 2)) return null;
|
||||
const [[swLat, swLng], [neLat, neLng]] = match;
|
||||
if (![swLat, swLng, neLat, neLng].every(Number.isFinite)) return null;
|
||||
const span = Math.max(neLat - swLat, neLng - swLng, 0.01);
|
||||
const step = 2 ** Math.ceil(Math.log2(span));
|
||||
const cLat = Math.round((swLat + neLat) / 2 / (step / 4)) * (step / 4);
|
||||
const cLng = Math.round((swLng + neLng) / 2 / (step / 4)) * (step / 4);
|
||||
const clamp = (v, limit) => Math.max(-limit, Math.min(limit, v));
|
||||
const fix = (v) => Number(v.toFixed(4));
|
||||
return `${fix(clamp(cLat - step / 2, 90))},${fix(clamp(cLng - step / 2, 180))}|${fix(clamp(cLat + step / 2, 90))},${fix(clamp(cLng + step / 2, 180))}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the forward search adapter over the shared gate.
|
||||
*
|
||||
* The policy asks that results be cached, and warns that a client repeating the
|
||||
* same query may be treated as faulty, so an answer is remembered and identical
|
||||
* searches already in flight share one upstream call rather than queueing
|
||||
* behind each other.
|
||||
* The policy asks that results be cached, and warns that a client repeating
|
||||
* the same query may be treated as faulty, so answers are kept (30 days for a
|
||||
* hit, 24 hours for a genuine no-result; a failure is never kept) and
|
||||
* identical searches already in flight share one upstream call.
|
||||
*/
|
||||
export function createNominatimSearchProvider({
|
||||
endpoint = 'https://nominatim.openstreetmap.org/search',
|
||||
requestJson = fetchRegionalJson,
|
||||
gate = () => sharedNominatimGate(),
|
||||
cache = createNominatimCache(),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
|
||||
const trimCache = () => {
|
||||
while (cache.size > NOMINATIM_SEARCH_MAX_CACHE) {
|
||||
const oldest = cache.keys().next().value;
|
||||
if (oldest === undefined) break;
|
||||
cache.delete(oldest);
|
||||
}
|
||||
};
|
||||
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
|
||||
|
||||
return async function fetchNominatimSearch(query, bounds, { signal } = {}) {
|
||||
const cacheKey = `${query.toLowerCase()}|${bounds || ''}`;
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.cachedAt <= NOMINATIM_SEARCH_CACHE_MS) {
|
||||
return { ...cached.payload, cached: true };
|
||||
}
|
||||
const { promise } = coalesceProxyRequest(inFlight, cacheKey, async () => {
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: query,
|
||||
addressdetails: '1',
|
||||
limit: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
const viewbox = nominatimViewboxFromBounds(bounds);
|
||||
if (viewbox) params.set('viewbox', viewbox);
|
||||
const rows = await enqueueNominatim(
|
||||
() =>
|
||||
requestJson(`${endpoint}?${params}`, {
|
||||
headers: NOMINATIM_HEADERS,
|
||||
redirect: 'error',
|
||||
}),
|
||||
{ bounded: true, signal },
|
||||
);
|
||||
const result = nominatimToGeocodeResult(
|
||||
Array.isArray(rows) ? rows[0] : null,
|
||||
);
|
||||
const payload = result
|
||||
? { status: 'OK', results: [result] }
|
||||
: { status: 'ZERO_RESULTS', results: [] };
|
||||
cache.set(cacheKey, { payload, cachedAt: Date.now() });
|
||||
trimCache();
|
||||
return payload;
|
||||
const coarse = coarseBounds(bounds);
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: upstreamQuery(query),
|
||||
addressdetails: '1',
|
||||
limit: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
return await promise;
|
||||
const viewbox = nominatimViewboxFromBounds(coarse);
|
||||
if (viewbox) params.set('viewbox', viewbox);
|
||||
const gateNow = currentGate();
|
||||
const cacheKey = requestKey(gateNow, 'search', params);
|
||||
const cached = await cache.get(cacheKey);
|
||||
if (cached) return { ...cached.payload, cached: true };
|
||||
return await subscribeSharedRequest(
|
||||
inFlight,
|
||||
cacheKey,
|
||||
signal,
|
||||
async (sharedSignal) => {
|
||||
const rows = await gateNow.requestJson(
|
||||
(endpoint) => `${endpoint}?${params}`,
|
||||
{ signal: sharedSignal },
|
||||
);
|
||||
// Only a well-formed empty list is "no such place"; anything else is a
|
||||
// failure and is never cached.
|
||||
if (!Array.isArray(rows)) throw malformed();
|
||||
const result = nominatimToGeocodeResult(rows[0] ?? null);
|
||||
if (rows.length && !result) throw malformed();
|
||||
const payload = result
|
||||
? { status: 'OK', results: [result] }
|
||||
: { status: 'ZERO_RESULTS', results: [] };
|
||||
await cache.set(
|
||||
cacheKey,
|
||||
payload,
|
||||
result
|
||||
? NOMINATIM_CACHE_TTL_MS.place
|
||||
: NOMINATIM_CACHE_TTL_MS.notFound,
|
||||
);
|
||||
return payload;
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct the outline lookup: one guarded search with polygons for one
|
||||
* explicit ask, accepting only results whose class suits the ask kind.
|
||||
* Every accepted result is kept (90 days; genuine no-results 24 hours), and
|
||||
* each ask is answered with the one holding its anchor, so a same-name place
|
||||
* elsewhere neither answers nor repeats from the cache.
|
||||
*/
|
||||
export function createNominatimOutlineProvider({
|
||||
gate = () => sharedNominatimGate(),
|
||||
cache = createNominatimCache(),
|
||||
} = {}) {
|
||||
const inFlight = new Map();
|
||||
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
|
||||
|
||||
const answer = (payload, anchorLat, anchorLon, cached) => ({
|
||||
status: payload.status,
|
||||
outline: outlineAt(payload.outlines, anchorLat, anchorLon, payload.ranks),
|
||||
...(payload.skipped ? { skipped: payload.skipped } : {}),
|
||||
...(cached ? { cached: true } : {}),
|
||||
});
|
||||
|
||||
return async function fetchNominatimOutline(
|
||||
{ query, kind, lat, lon, anchorLat = null, anchorLon = null },
|
||||
{ signal } = {},
|
||||
) {
|
||||
const bias = outlineBias(kind, lat, lon);
|
||||
const sent = upstreamQuery(query);
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
q: sent,
|
||||
limit: String(OUTLINE_RESULT_LIMIT),
|
||||
polygon_geojson: '1',
|
||||
polygon_threshold: String(OUTLINE_POLYGON_THRESHOLD[kind]),
|
||||
namedetails: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
if (bias) params.set('viewbox', bias.viewbox);
|
||||
const gateNow = currentGate();
|
||||
const cacheKey = requestKey(gateNow, `outlines-${kind}`, params);
|
||||
const cached = await cache.get(cacheKey);
|
||||
if (cached) return answer(cached.payload, anchorLat, anchorLon, true);
|
||||
const payload = await subscribeSharedRequest(
|
||||
inFlight,
|
||||
cacheKey,
|
||||
signal,
|
||||
async (sharedSignal) => {
|
||||
const rows = await gateNow.requestJson(
|
||||
(endpoint) => `${endpoint}?${params}`,
|
||||
{ signal: sharedSignal, maxBytes: OUTLINE_MAX_RESPONSE_BYTES },
|
||||
);
|
||||
if (!Array.isArray(rows)) throw malformed();
|
||||
const { outlines, ranks, skipped } = selectOutlineResult(rows, {
|
||||
kind,
|
||||
query: sent,
|
||||
});
|
||||
const found = outlines.length
|
||||
? { status: 'OK', outlines, ranks }
|
||||
: { status: 'ZERO_RESULTS', outlines: [], skipped };
|
||||
await cache.set(
|
||||
cacheKey,
|
||||
found,
|
||||
outlines.length
|
||||
? NOMINATIM_CACHE_TTL_MS.outline
|
||||
: NOMINATIM_CACHE_TTL_MS.notFound,
|
||||
);
|
||||
return found;
|
||||
},
|
||||
);
|
||||
return answer(payload, anchorLat, anchorLon, false);
|
||||
};
|
||||
}
|
||||
|
||||
export const fetchNominatimSearch = createNominatimSearchProvider();
|
||||
|
||||
/** Vite plugin: last-resort place search over the public Nominatim instance. */
|
||||
export function geocodeProxy({ search = fetchNominatimSearch } = {}) {
|
||||
const limiter = makeRateLimiter({
|
||||
function sendJson(res, status, body, headers = {}) {
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...headers,
|
||||
});
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
/** Map a gate refusal to the route's answer. Never "not found". */
|
||||
function sendFailure(res, error, what) {
|
||||
if (error?.code === 'NOMINATIM_DISABLED') {
|
||||
sendJson(res, 503, {
|
||||
error: `${what} is not configured`,
|
||||
code: 'NOMINATIM_DISABLED',
|
||||
retryable: false,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (error?.code === 'NOMINATIM_DAILY_CAP') {
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{
|
||||
error: `${what} daily allowance is used`,
|
||||
code: 'NOMINATIM_DAILY_CAP',
|
||||
retryable: false,
|
||||
},
|
||||
{ 'Retry-After': '3600' },
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (isNominatimBusy(error)) {
|
||||
const seconds = Math.max(5, Math.ceil((error.retryAfterMs || 0) / 1000));
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{ error: `${what} is busy`, code: error.code },
|
||||
{ 'Retry-After': String(seconds) },
|
||||
);
|
||||
return;
|
||||
}
|
||||
sendJson(res, 503, {
|
||||
error: `${what} is temporarily unavailable`,
|
||||
code: 'NOMINATIM_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Vite plugin: last-resort place search (`/api/geocode`) and voice outlines
|
||||
* (`/api/geocode/outline`) over the configured Nominatim endpoint.
|
||||
*
|
||||
* `storageDir` turns on the disk cache and the persisted daily count; the
|
||||
* application composition supplies it, tests and embedders may omit it.
|
||||
*/
|
||||
export function geocodeProxy({
|
||||
search,
|
||||
outline,
|
||||
storageDir = null,
|
||||
gate = null,
|
||||
} = {}) {
|
||||
const currentGate = () => gate || sharedNominatimGate();
|
||||
if (storageDir) {
|
||||
// The shared count and pause live beside, never inside, the answer cache,
|
||||
// so no cache sweep can reset them.
|
||||
currentGate().persistUsage(path.join(storageDir, 'state.json'));
|
||||
const diskCache = createNominatimCache({
|
||||
dir: path.join(storageDir, 'answers'),
|
||||
});
|
||||
void diskCache.sweep();
|
||||
search ||= createNominatimSearchProvider({
|
||||
gate: currentGate,
|
||||
cache: diskCache,
|
||||
});
|
||||
outline ||= createNominatimOutlineProvider({
|
||||
gate: currentGate,
|
||||
cache: diskCache,
|
||||
});
|
||||
}
|
||||
search ||= gate
|
||||
? createNominatimSearchProvider({ gate })
|
||||
: fetchNominatimSearch;
|
||||
outline ||= createNominatimOutlineProvider({ gate: currentGate });
|
||||
|
||||
const searchLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 30,
|
||||
globalMax: 90,
|
||||
});
|
||||
// Outlines are explicit voice asks: a person cannot make many a minute.
|
||||
const outlineLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 12,
|
||||
globalMax: 30,
|
||||
});
|
||||
|
||||
/** Common front door: method, same-site, rate limit, abandonment signal. */
|
||||
function admit(req, res, limiter) {
|
||||
if (admitSameSite(req, res)) return null;
|
||||
if (req.method !== 'GET') {
|
||||
sendJson(res, 405, { error: 'Method Not Allowed' });
|
||||
return null;
|
||||
}
|
||||
if (!limiter(clientKey(req))) {
|
||||
sendJson(
|
||||
res,
|
||||
429,
|
||||
{ error: 'Rate limit exceeded' },
|
||||
{ 'Retry-After': '10' },
|
||||
);
|
||||
return null;
|
||||
}
|
||||
// A browser that gave up is no longer waiting; the queue reads this
|
||||
// before spending its slot.
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
return abandoned.signal;
|
||||
}
|
||||
|
||||
async function handleOutline(req, res, url) {
|
||||
const signal = admit(req, res, outlineLimiter);
|
||||
if (!signal) return;
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
const kind = String(url.searchParams.get('kind') || '');
|
||||
if (
|
||||
!query ||
|
||||
query.length > NOMINATIM_SEARCH_MAX_QUERY ||
|
||||
!OUTLINE_KINDS.includes(kind)
|
||||
) {
|
||||
sendJson(res, 400, {
|
||||
error:
|
||||
'A place query of 1-200 characters and a known kind are required',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const payload = await outline(
|
||||
{
|
||||
query,
|
||||
kind,
|
||||
lat: url.searchParams.get('lat'),
|
||||
lon: url.searchParams.get('lon'),
|
||||
anchorLat: coordinateParam(url.searchParams.get('alat')),
|
||||
anchorLon: coordinateParam(url.searchParams.get('alon')),
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
{ status: payload.status, outline: payload.outline || null },
|
||||
{
|
||||
'Cache-Control': payload.cached ? 'private, max-age=300' : 'no-store',
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
sendFailure(res, error, 'Outline lookup');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSearch(req, res, url) {
|
||||
const signal = admit(req, res, searchLimiter);
|
||||
if (!signal) return;
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
|
||||
sendJson(res, 400, {
|
||||
error: 'A place query of 1-200 characters is required',
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const payload = await search(query, url.searchParams.get('bounds'), {
|
||||
signal,
|
||||
});
|
||||
sendJson(
|
||||
res,
|
||||
200,
|
||||
{ status: payload.status, results: payload.results },
|
||||
{
|
||||
'Cache-Control': payload.cached ? 'private, max-age=60' : 'no-store',
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
sendFailure(res, error, 'Place search');
|
||||
}
|
||||
}
|
||||
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/geocode', async (req, res) => {
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
if (!limiter(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '10',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
middlewares.use('/api/geocode', async (req, res, next) => {
|
||||
const url = new URL(req.url || '', 'http://localhost');
|
||||
const query = String(url.searchParams.get('q') || '').trim();
|
||||
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
|
||||
res.writeHead(400, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'A place query of 1-200 characters is required',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
// A browser that gave up is no longer waiting; the queue reads this
|
||||
// before spending its slot.
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
try {
|
||||
const payload = await search(query, url.searchParams.get('bounds'), {
|
||||
signal: abandoned.signal,
|
||||
// Capability probe, like /api/overpass/status: clients skip lookups a
|
||||
// server without a Nominatim endpoint could only refuse.
|
||||
if (url.pathname === '/status' && req.method === 'GET')
|
||||
return sendJson(res, 200, {
|
||||
configured: Boolean(currentGate().settings()?.endpoint),
|
||||
});
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': payload.cached ? 'public, max-age=60' : 'no-store',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({ status: payload.status, results: payload.results }),
|
||||
);
|
||||
} catch (error) {
|
||||
if (res.writableEnded) return;
|
||||
const busy =
|
||||
error?.code === 'NOMINATIM_QUEUE_FULL' ||
|
||||
error?.code === 'NOMINATIM_ABANDONED';
|
||||
res.writeHead(busy ? 429 : 503, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(busy ? { 'Retry-After': '5' } : {}),
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: busy
|
||||
? 'Place search is busy'
|
||||
: 'Place search is temporarily unavailable',
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (url.pathname === '/outline') return handleOutline(req, res, url);
|
||||
if (url.pathname === '/' || url.pathname === '')
|
||||
return handleSearch(req, res, url);
|
||||
if (typeof next === 'function') return next();
|
||||
sendJson(res, 404, { error: 'Not Found' });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
parseTerrainPoints,
|
||||
resolveTerrainHeightRequest,
|
||||
terrainPointKey,
|
||||
terrainRetryAfterMs,
|
||||
validTerrainResult,
|
||||
} from '../../src/data/terrainHeightsProxy.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
@@ -55,6 +56,15 @@ export function terrainHeightsProxy(options = {}) {
|
||||
// 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
|
||||
// that range.
|
||||
const UPSTREAM_CHUNK = 64;
|
||||
// After a 429 the upstream is left alone for its Retry-After (at most
|
||||
// RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a
|
||||
// restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for
|
||||
// new points (about once a second while the camera moves) otherwise asks
|
||||
// again with its own retries, which keeps the limit in force and logs each
|
||||
// time.
|
||||
const RATE_LIMIT_PAUSE_MS = 60_000;
|
||||
const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000;
|
||||
let rateLimitedUntil = 0;
|
||||
const MAX_POINTS = UPSTREAM_CHUNK;
|
||||
/**
|
||||
* Entry ceiling for the point cache, and with it the file written from it.
|
||||
@@ -199,7 +209,8 @@ export function terrainHeightsProxy(options = {}) {
|
||||
*
|
||||
* The first try keeps its empirically required 30s timeout; network errors,
|
||||
* 429, and 5xx receive up to three jittered retries sharing a 10s added-time
|
||||
* budget, with Retry-After honored within that bound.
|
||||
* budget, with Retry-After honored within that bound; a longer Retry-After
|
||||
* ends the retries at once so the 429 pause below starts without delay.
|
||||
*
|
||||
* A chunk that still fails yields nulls for its own positions rather than
|
||||
* rejecting: with several chunks per request, throwing would discard every
|
||||
@@ -211,15 +222,37 @@ export function terrainHeightsProxy(options = {}) {
|
||||
* @returns {Promise<Array<object>>}
|
||||
*/
|
||||
async function fetchUpstreamAll(points) {
|
||||
if (Date.now() < rateLimitedUntil) {
|
||||
// Rate-limited: asking again only extends the limit. Cached points are
|
||||
// still served; the rest resolve on the page through its geoid.
|
||||
const error = new Error('HTTP 429');
|
||||
error.status = 429;
|
||||
error.coolingDown = true;
|
||||
throw error;
|
||||
}
|
||||
const results = [];
|
||||
let firstError = null;
|
||||
for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
|
||||
const chunk = points.slice(i, i + UPSTREAM_CHUNK);
|
||||
let chunkResults = [];
|
||||
try {
|
||||
chunkResults = await fetchTerrainChunkWithRetry(chunk);
|
||||
} catch (error) {
|
||||
firstError = firstError || error;
|
||||
// Re-checked per chunk for when a request spans several chunks again;
|
||||
// today requests are capped at one chunk (MAX_POINTS).
|
||||
if (Date.now() >= rateLimitedUntil) {
|
||||
try {
|
||||
chunkResults = await fetchTerrainChunkWithRetry(chunk);
|
||||
} catch (error) {
|
||||
firstError = firstError || error;
|
||||
// Concurrent 429s extend the pause, never cut it short.
|
||||
if (error?.status === 429)
|
||||
rateLimitedUntil = Math.max(
|
||||
rateLimitedUntil,
|
||||
Date.now() +
|
||||
Math.min(
|
||||
terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS,
|
||||
RATE_LIMIT_PAUSE_MAX_MS,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
// Keep later chunks aligned even if a malformed upstream response omits
|
||||
// trailing positions. The resolver will reject each null individually.
|
||||
@@ -292,9 +325,17 @@ export function terrainHeightsProxy(options = {}) {
|
||||
evictOverflow();
|
||||
diskDirty = true;
|
||||
}
|
||||
if (outcome.upstreamError) {
|
||||
if (outcome.upstreamError?.coolingDown) {
|
||||
// Logged once, when the pause began.
|
||||
} else if (outcome.upstreamError) {
|
||||
const pause =
|
||||
outcome.upstreamError.status === 429 &&
|
||||
Date.now() < rateLimitedUntil
|
||||
? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)`
|
||||
: '';
|
||||
console.warn(
|
||||
'[terrain-heights-proxy] refresh incomplete' +
|
||||
pause +
|
||||
' — serving stale points when available',
|
||||
);
|
||||
} else if (outcome.absentPoints > 0) {
|
||||
|
||||
@@ -12,7 +12,10 @@ import { unwatchedRuntimeDirsPlugin } from './unwatched-cache.js';
|
||||
const root = fileURLToPath(new URL('../../', import.meta.url));
|
||||
|
||||
/** Load this checkout's configuration and attach its local provider middleware. */
|
||||
export default defineConfig(({ command, mode }) => {
|
||||
export function standaloneConfig(
|
||||
{ command, mode },
|
||||
{ nominatimStorageDir } = {},
|
||||
) {
|
||||
const loaded = loadEnv(mode, root, '');
|
||||
for (const [key, value] of Object.entries(loaded)) {
|
||||
if (process.env[key] === undefined) process.env[key] = value;
|
||||
@@ -20,7 +23,11 @@ export default defineConfig(({ command, mode }) => {
|
||||
const tools = standaloneVoiceTools();
|
||||
return createBrowserViteConfig({
|
||||
plugins: [
|
||||
...localProviderPlugins({ realtime: { tools }, gemini: { tools } }),
|
||||
...localProviderPlugins({
|
||||
realtime: { tools },
|
||||
gemini: { tools },
|
||||
nominatimStorageDir,
|
||||
}),
|
||||
localMcpPlugin(),
|
||||
googleTokenUnavailablePlugin(),
|
||||
apiNotFoundPlugin(),
|
||||
@@ -28,10 +35,13 @@ export default defineConfig(({ command, mode }) => {
|
||||
],
|
||||
googleApiKey: process.env.GOOGLE_MAPS_API_KEY,
|
||||
cesiumToken: process.env.CESIUM_ION_TOKEN,
|
||||
photonUrl: process.env.PHOTON_URL,
|
||||
mapillaryToken: process.env.MAPILLARY_CLIENT_TOKEN,
|
||||
host: process.env.HOST,
|
||||
port: process.env.PORT,
|
||||
allowedHosts: resolveAllowedHosts(process.env.GEV_ALLOWED_HOSTS),
|
||||
command,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export default defineConfig(standaloneConfig);
|
||||
Reference in new issue
Block a user