benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-11 13:52:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream/main 73c14ac74

Browse Source
This commit is contained in:
daikaginza committed 2026-11-04 11:21:01 +07:00
2 parents 73f03771d8 73c14ac749
commit 0e0557e7f5
1594 files changed
+155855 -24143
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/providers/aircraft/opensky.js
+72 -59
View File
@@ -1,3 +1,5 @@
import fs from 'node:fs';
import os from 'node:os';
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
import { encodedBody, gzipBody } from '../common/compress.js';
import {
@@ -144,6 +146,10 @@ function openskyAdaptiveTtlMs(remaining) {
}
/** @type {boolean} Guards duplicate auth-failure warnings in logs. */
let _openskyAuthWarned = false;
/** One warning per process when OPENSKY_CREDENTIALS_FILE cannot be read. */
let _openskyCredentialsFileWarned = false;
/** One warning per process when OPENSKY_AUTH_MODE=basic is requested. */
let _openskyBasicRetiredWarned = false;
/** @type {boolean} Guards duplicate invalid-auth-mode warnings. */
let _openskyAuthModeWarned = false;
/** Default auth mode when OPENSKY_AUTH_MODE env is unset. */
@@ -163,6 +169,48 @@ const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
// current instead of coasting a stale worldwide frame indefinitely.
const OPENSKY_SOURCE_STALE_MS = 120_000;
/**
* The OpenSky API client for OAuth: OPENSKY_CLIENT_ID and
* OPENSKY_CLIENT_SECRET, or else the credentials.json that OpenSky's account
* page downloads, named by OPENSKY_CREDENTIALS_FILE ("~" is the home
* directory). The file is read on each token refresh, so a replaced file is
* picked up without a restart.
*
* @param {Record<string, string|undefined>} [env]
* @param {(path: string) => string} [readFile]
* @returns {{ clientId: string, clientSecret: string } | null}
*/
export function openSkyClientCredentials(
env = process.env,
readFile = (path) => fs.readFileSync(path, 'utf8'),
) {
const clientId = String(env.OPENSKY_CLIENT_ID || '').trim();
const clientSecret = String(env.OPENSKY_CLIENT_SECRET || '').trim();
if (clientId && clientSecret) return { clientId, clientSecret };
const file = String(env.OPENSKY_CREDENTIALS_FILE || '').trim();
if (!file) return null;
const path = file.replace(/^~(?=$|[\\/])/, os.homedir());
let raw;
try {
raw = JSON.parse(readFile(path));
} catch {
if (!_openskyCredentialsFileWarned) {
console.warn(
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
);
_openskyCredentialsFileWarned = true;
}
return null;
}
const fileId = String(raw?.clientId ?? raw?.client_id ?? '').trim();
const fileSecret = String(
raw?.clientSecret ?? raw?.client_secret ?? '',
).trim();
return fileId && fileSecret
? { clientId: fileId, clientSecret: fileSecret }
: null;
}
/**
* Obtain a valid OpenSky OAuth2 bearer token, refreshing if needed.
*
@@ -181,9 +229,9 @@ export async function getOpenSkyToken() {
// return the same promise instead of issuing a duplicate token request
if (_openskyTokenPromise) return _openskyTokenPromise;
const clientId = process.env.OPENSKY_CLIENT_ID;
const clientSecret = process.env.OPENSKY_CLIENT_SECRET;
if (!clientId || !clientSecret) return null;
const credentials = openSkyClientCredentials();
if (!credentials) return null;
const { clientId, clientSecret } = credentials;
// Wrap the async token fetch in a shared promise stored in _openskyTokenPromise
_openskyTokenPromise = (async () => {
@@ -431,8 +479,10 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
*
* Supports four auth modes controlled by OPENSKY_AUTH_MODE env:
* - 'oauth' (default) — client_credentials bearer token
* - 'basic' — HTTP Basic with OPENSKY_USERNAME / OPENSKY_PASSWORD
* - 'auto' — try OAuth first, fall back to Basic, then anon
* - 'basic' — retired: OpenSky stopped accepting HTTP Basic (username and
* password) on 2026-03-18, so this requests anonymously and
* warns once
* - 'auto' — try OAuth first, then anon
* - 'anon' — no credentials
*
* Successful responses are cached for OPENSKY_CACHE_MS (~9 s). On
@@ -543,21 +593,21 @@ export function openSkyProxy() {
_openskySnapshotInFlight,
'global',
async () => {
const basicUser = process.env.OPENSKY_USERNAME || '';
const basicPass = process.env.OPENSKY_PASSWORD || '';
const hasBasicCreds = Boolean(basicUser && basicPass);
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
if (requestedMode === 'basic') {
if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'basic_credentials';
} else {
reason = 'missing_basic_creds';
// OpenSky no longer accepts a Basic header, so sending
// OPENSKY_USERNAME / OPENSKY_PASSWORD would only fail; ask
// anonymously and say once what to configure instead.
if (!_openskyBasicRetiredWarned) {
console.warn(
'[OpenSky] OPENSKY_AUTH_MODE=basic: OpenSky retired username/password sign-in on 2026-03-18, so flights load anonymously (rate-limited). Use OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
);
_openskyBasicRetiredWarned = true;
}
reason = 'basic_retired';
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
@@ -573,31 +623,12 @@ export function openSkyProxy() {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'oauth_unavailable_fallback_basic';
} else {
reason = 'missing_oauth_and_basic_creds';
reason = 'missing_oauth_creds';
}
}
let { upstream, body } = await fetchOpenSkyStates(headers);
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
if (
(upstream.status === 401 || upstream.status === 403) &&
requestedMode === 'auto' &&
usedMode === 'oauth' &&
hasBasicCreds
) {
const retryHeaders = {
Accept: 'application/json',
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
};
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
usedMode = 'basic';
reason = 'oauth_rejected_fallback_basic';
}
const { upstream, body } = await fetchOpenSkyStates(headers);
const sourceEpochMs = upstream.ok
? openSkySourceEpochMs(body)
@@ -605,12 +636,6 @@ export function openSkyProxy() {
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'basic' &&
reason === 'basic_credentials'
) {
reason = 'basic_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
@@ -651,12 +676,11 @@ export function openSkyProxy() {
body,
usedMode,
reason,
hasBasicCreds,
sourceEpochMs,
};
},
);
let { upstream, body, usedMode, reason, hasBasicCreds, sourceEpochMs } =
let { upstream, body, usedMode, reason, sourceEpochMs } =
await refresh.promise;
if (
upstream.ok &&
@@ -713,35 +737,24 @@ export function openSkyProxy() {
}
if (upstream.status === 401 || upstream.status === 403) {
if (requestedMode === 'basic' && !hasBasicCreds) {
if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth missing. Basic mode requires OPENSKY_USERNAME and OPENSKY_PASSWORD.',
});
reason = 'missing_basic_creds';
} else if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET.',
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'oauth_invalid_or_missing';
} else if (usedMode === 'basic') {
body = JSON.stringify({
error: 'OpenSky auth invalid. Username/password were rejected.',
});
reason = 'basic_invalid_credentials';
} else if (usedMode === 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth client credentials were rejected.',
});
reason = 'oauth_invalid_credentials';
} else if (requestedMode === 'auto' && !hasBasicCreds) {
} else if (requestedMode === 'auto') {
body = JSON.stringify({
error:
'OpenSky auth missing. Provide basic credentials or valid OAuth client credentials.',
'OpenSky auth missing. Provide OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'missing_oauth_and_basic_creds';
reason = 'missing_oauth_creds';
} else {
body = JSON.stringify({
error: 'OpenSky auth required.',
server/providers/aircraft/openskyCredentials.test.mjs
+81
View File
@@ -0,0 +1,81 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import os from 'node:os';
import path from 'node:path';
import { openSkyClientCredentials } from './opensky.js';
const file = (contents) => (filePath) => {
file.read = filePath;
if (contents instanceof Error) throw contents;
return contents;
};
test('OpenSky client env vars come first', () => {
const read = () => assert.fail('the file is not read when env vars are set');
assert.deepEqual(
openSkyClientCredentials(
{
OPENSKY_CLIENT_ID: ' id ',
OPENSKY_CLIENT_SECRET: 'secret',
OPENSKY_CREDENTIALS_FILE: 'credentials.json',
},
read,
),
{ clientId: 'id', clientSecret: 'secret' },
);
});
test('OPENSKY_CREDENTIALS_FILE supplies the client when env vars are empty', () => {
const env = {
OPENSKY_CLIENT_ID: '',
OPENSKY_CREDENTIALS_FILE: '~/Downloads/credentials.json',
};
// The file OpenSky's account page downloads uses camelCase keys.
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"clientId":"cid","clientSecret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
assert.equal(
path.normalize(file.read),
path.normalize(path.join(os.homedir(), 'Downloads/credentials.json')),
);
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"client_id":"cid","client_secret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
});
test('a missing, unreadable or incomplete credentials file means no client', () => {
assert.equal(openSkyClientCredentials({}, file('{}')), null);
const env = { OPENSKY_CREDENTIALS_FILE: 'credentials.json' };
const missing = Object.assign(new Error('nope'), { code: 'ENOENT' });
assert.equal(openSkyClientCredentials(env, file(missing)), null);
assert.equal(openSkyClientCredentials(env, file('not json')), null);
assert.equal(openSkyClientCredentials(env, file('{"clientId":"cid"}')), null);
});
test('credential file warnings never expose file contents or read errors', async (t) => {
const warnings = [];
t.mock.method(console, 'warn', (message) => warnings.push(message));
for (const contents of [
'{"clientId":"id","clientSecret":FAKE_SECRET_123}',
new Error('could not read FAKE_SECRET_123'),
]) {
const { openSkyClientCredentials: resolve } = await import(
`./opensky.js?warning=${warnings.length}`
);
const env = { OPENSKY_CREDENTIALS_FILE: '/synthetic/credentials.json' };
assert.equal(resolve(env, file(contents)), null);
assert.equal(resolve(env, file(contents)), null);
}
assert.deepEqual(warnings, [
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
]);
});
server/providers/cctv/catalog.js
+12
View File
@@ -28,6 +28,8 @@ import {
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
loadFloridaSourcesFromOpenData,
loadHoustonSourcesFromOpenData,
loadA22SourcesFromOpenDataHub,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
@@ -165,6 +167,16 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
load: loadTaiwanSources,
},
{
name: 'houston',
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
load: loadHoustonSourcesFromOpenData,
},
{
name: 'a22',
enabled: () => envEnabled('CCTV_A22_ENABLED'),
load: loadA22SourcesFromOpenDataHub,
},
];
/**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
server/providers/cctv/constants.js
+64
View File
@@ -443,6 +443,70 @@ export const NYC_ANCHORS = [
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** Houston TranStar cameras: one keyless JSON list for the metro (~1,000
* cameras with a numeric id). Frames are stills synthesized from that id on
* the city's snapshot host — never taken from the payload. The catalog is
* served as `application/javascript`; asking for `application/json` gets HTTP
* 406, so the loader accepts any media type. */
export const HOUSTON_CAMERAS_URL =
'https://traffic.houstontranstar.org/data/layers/cctvSnapshots_json.js';
/** The only origin Houston camera frames may come from. The catalog's `url`
* is a `showWin(...)` viewer helper and is ignored; the registered path is
* always this origin plus `/snapshots/cctv/<id>.jpg`. */
export const HOUSTON_IMAGE_ORIGIN = 'https://www.houstontranstar.org/';
export const HOUSTON_IMAGE_URL = (id) =>
`${HOUSTON_IMAGE_ORIGIN}snapshots/cctv/${id}.jpg`;
/** Numeric TranStar snapshot ids: 3–5 digits, no leading zero. Id `0` is a
* different (non-snapshot) camera class in the same list. */
export const HOUSTON_CAMERA_ID_RE = /^[1-9]\d{2,4}$/;
export const DEFAULT_HOUSTON_MAX_SOURCES = 250;
export const HOUSTON_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. The TranStar payload carries none, and
* Houston sits a few metres above sea level. */
export const HOUSTON_GROUND_ELEVATION_M = 15;
/** Downtown Houston (Main @ Texas): the primary prioritization anchor. */
export const HOUSTON_DOWNTOWN = { lat: 29.7604, lon: -95.3698 };
export const HOUSTON_ANCHORS = [
{ lat: 29.7604, lon: -95.3698 }, // Downtown
{ lat: 29.739, lon: -95.461 }, // Uptown / Galleria
{ lat: 29.71, lon: -95.398 }, // Texas Medical Center
{ lat: 29.9902, lon: -95.3368 }, // IAH
{ lat: 29.3013, lon: -94.7977 }, // Galveston
];
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
* own camera list with coordinates, one request for the whole motorway
* (13 cameras). Frames are stills on the operator's host, refreshed about
* once a minute. */
export const DEFAULT_A22_CCTV_URL =
'https://tourism.api.opendatahub.com/v1/WebcamInfo?source=a22&pagesize=200&removenullvalues=true';
/** The only origin A22 camera frames may come from: `km<N>.jpg` per camera. */
export const A22_IMAGE_ORIGIN = 'https://www.autobrennero.it/WebCamImg/';
/** Hard ceiling on the catalog body. The whole motorway is ~45 KB. */
export const A22_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Orthometric ground elevation (m) at each camera, keyed by its motorway
* kilometre, from the Copernicus GLO-90 DEM via Open-Meteo's elevation API.
* The feed's coordinates are 2D and the road climbs from the Po valley (~20 m)
* to the Brenner Pass (~1,370 m), so one flat prior would bury the Alpine
* cameras. A camera not listed here uses A22_DEFAULT_GROUND_ELEVATION_M. */
export const A22_GROUND_ELEVATION_BY_KM = Object.freeze({
1: 1365,
68: 375,
77: 301,
100: 219,
129: 206,
138: 204,
159: 175,
205: 188,
228: 72,
240: 38,
272: 16,
285: 19,
313: 40,
});
export const A22_DEFAULT_GROUND_ELEVATION_M = 200;
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
server/providers/cctv/media.js
+2
View File
@@ -1,6 +1,7 @@
import { Readable } from 'node:stream';
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
import {
A22_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
CCTV_FRAME_FETCH_TIMEOUT_MS,
CCTV_FRAME_MAX_BODY_BYTES,
@@ -528,6 +529,7 @@ for (const prefix of [
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
A22_IMAGE_ORIGIN,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
server/providers/cctv/normalize.js
+23
View File
@@ -410,6 +410,18 @@ export function isLikelyHongKongCoordinate(lat, lon) {
);
}
/** Greater Houston (the Woodlands to Galveston, Katy to Baytown). Farther
* TranStar partners (Dallas, Columbus) are out of this city pack. */
export function isLikelyHoustonCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 29.2 &&
lat <= 30.35 &&
lon >= -96.05 &&
lon <= -94.7
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
@@ -421,6 +433,17 @@ export function isLikelyNorwayCoordinate(lat, lon) {
);
}
/** The A22 corridor, Brenner Pass to Modena, with slack either side. */
export function isLikelyA22Coordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 44.5 &&
lat <= 47.1 &&
lon >= 10.5 &&
lon <= 11.8
);
}
export function isLikelyFinlandCoordinate(lat, lon) {
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
server/providers/cctv/sources.js
+293
View File
@@ -122,6 +122,18 @@ import {
NYC_MAX_CATALOG_BYTES,
NYC_GROUND_ELEVATION_M,
NYC_ANCHORS,
DEFAULT_A22_CCTV_URL,
A22_IMAGE_ORIGIN,
A22_MAX_CATALOG_BYTES,
A22_GROUND_ELEVATION_BY_KM,
A22_DEFAULT_GROUND_ELEVATION_M,
HOUSTON_CAMERAS_URL,
HOUSTON_IMAGE_URL,
HOUSTON_CAMERA_ID_RE,
DEFAULT_HOUSTON_MAX_SOURCES,
HOUSTON_MAX_CATALOG_BYTES,
HOUSTON_GROUND_ELEVATION_M,
HOUSTON_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -142,9 +154,11 @@ import {
isLikelyCataloniaCoordinate,
decodeNumericEntities,
isLikelyNycCoordinate,
isLikelyHoustonCoordinate,
isLikelyHongKongCoordinate,
isLikelyNorwayCoordinate,
isLikelyTorontoCoordinate,
isLikelyA22Coordinate,
cameraDisplayCode,
rowArrayToObject,
prioritizeSources,
@@ -3645,3 +3659,282 @@ export async function loadFloridaSourcesFromOpenData() {
return [];
}
}
/**
* One Open Data Hub `WebcamInfo` item (source `a22`) -> one catalog source,
* or null.
*
* The frame must be the operator's own `WebCamImg/km<N>.jpg` for the
* camera's motorway kilometre. The feed names the place and the kilometre but
* no bearing, so the heading uses the shared id-hash fallback at low
* confidence, like Calgary and Vegvesen; the ground elevation comes from the
* per-kilometre DEM table because the road spans ~1,350 m of altitude.
*
* @param {object} item - One element of the API's `Items` array.
* @returns {?object}
*/
export function a22WebcamToSource(item) {
if (!item || typeof item !== 'object') return null;
if (item.Active === false) return null;
if (String(item.Source ?? '').toLowerCase() !== 'a22') return null;
const gps = Array.isArray(item.GpsInfo) ? item.GpsInfo[0] : null;
const lat = toFiniteNumber(gps?.Latitude);
const lon = toFiniteNumber(gps?.Longitude);
if (!isLikelyA22Coordinate(lat, lon)) return null;
const km = String(item.Mapping?.a22?.km ?? '').trim();
if (!/^\d{1,3}$/.test(km)) return null;
const imageUrl = `${A22_IMAGE_ORIGIN}km${Number(km)}.jpg`;
let image;
try {
image = new URL(String(item.Webcamurl ?? ''));
} catch {
return null;
}
if (image.username || image.password || image.href !== imageUrl) return null;
const place =
String(item.Shortname ?? item.Webcamname?.it ?? '')
.replace(/\s+/g, ' ')
.trim() || `Km ${Number(km)}`;
const cameraId = `it-a22-km${Number(km)}`;
const ground = A22_GROUND_ELEVATION_BY_KM[Number(km)];
return {
id: cameraId,
name: `A22 ${place} (km ${Number(km)})`,
// One motorway-wide category in the camera picker; the place is already
// in the name.
city: 'A22 Brennero',
cityId: 'a22',
provider: 'Autostrada del Brennero',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -15,
fovDeg: 50,
rangeM: 160,
mountHeightM: 8,
groundElevationM: Number.isFinite(ground)
? ground
: A22_DEFAULT_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'a22-opendatahub',
license:
'Webcam images © Autostrada del Brennero S.p.A., via the Open Data Hub (NOI Techpark)',
code: cameraDisplayCode(place.toUpperCase()),
};
}
/**
* Fetch the Autostrada del Brennero (A22) webcams from the keyless Open Data
* Hub tourism API. Frames are stills on www.autobrennero.it.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadA22SourcesFromOpenDataHub() {
try {
const endpoint = process.env.CCTV_A22_URL || DEFAULT_A22_CCTV_URL;
const resp = await fetch(endpoint, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn('[CCTV] A22 catalog redirected; redirects are not followed');
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] A22 camera download failed:', resp.status);
return discard();
}
const payload = await readResponseJsonCapped(resp, A22_MAX_CATALOG_BYTES);
const items = Array.isArray(payload?.Items) ? payload.Items : [];
const cameras = [];
const seen = new Set();
for (const item of items) {
const camera = a22WebcamToSource(item);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
// North to south along the motorway, so the picker reads in road order.
cameras.sort((a, b) => b.lat - a.lat);
console.log(`[CCTV] Loaded A22 camera sources: ${cameras.length}`);
return cameras;
} catch (error) {
console.warn('[CCTV] A22 camera download error:', error?.message || error);
return [];
}
}
/**
* Label for one Houston TranStar camera: the interchange an operator
* recognises ("10 EAST @ SAN JACINTO").
*
* @param {object} record - Raw TranStar camera row.
* @param {string} cameraId - Derived stable id.
* @returns {string}
*/
export function houstonCameraName(record, cameraId) {
const name = String(record?.location ?? '').trim();
if (name) return name;
const short = String(cameraId || '').replace(/^houston-/, '');
return short ? `Houston Camera ${short}` : 'Houston Camera';
}
/**
* TranStar `dir` (the camera's side of the freeway) -> the travel direction
* of the nearest lanes, as TranStar's map script `translateCameraDirection`
* reads it. Only these four values are mapped; anything else is unknown.
*/
const HOUSTON_TRAVEL_HEADING_BY_SIDE = Object.freeze({
south: 90,
north: 270,
west: 180,
east: 0,
});
/**
* One Houston TranStar camera row -> one catalog source, or null.
*
* Frame URLs are synthesized from a validated numeric id on the pinned
* snapshot origin. The payload's `url` is a `showWin(...)` viewer helper
* and is ignored so a catalog edit cannot steer the frame proxy off-host.
* TranStar's `dir` is the side of the freeway the camera sits on, not the
* way it faces: TranStar's own map labels it "Traffic closest to the camera
* is traveling …" (South→Eastbound, North→Westbound, West→Southbound,
* East→Northbound). That travel direction is only a low-confidence prior
* for the view, so the heading stays estimated with the low-confidence
* pose; `None`, blank or unknown values fall back to the shared id-hash
* heading.
*
* @param {object} record - Raw TranStar camera row.
* @returns {?object}
*/
export function houstonCameraToSource(record) {
if (!record || typeof record !== 'object') return null;
const lat = toFiniteNumber(record.lat);
const lon = toFiniteNumber(record.lng);
if (!isLikelyHoustonCoordinate(lat, lon)) return null;
const rawId = String(record.id ?? '').trim();
if (!HOUSTON_CAMERA_ID_RE.test(rawId)) return null;
const cameraId = `houston-${rawId}`;
const imageUrl = HOUSTON_IMAGE_URL(rawId);
const name = houstonCameraName(record, cameraId);
const travelHeading =
HOUSTON_TRAVEL_HEADING_BY_SIDE[
String(record.dir ?? '')
.trim()
.toLowerCase()
];
return {
id: cameraId,
name,
city: 'Houston',
cityId: 'houston',
provider: 'Houston TranStar',
lat,
lon,
headingDeg: Number.isFinite(travelHeading)
? travelHeading
: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: HOUSTON_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'houston-transtar',
license:
'TranStar disclaimer — redistribution and external framing need TxDOT written permission',
code: cameraDisplayCode(name.toUpperCase()),
};
}
/**
* Fetch Houston TranStar cameras from the keyless `cctvSnapshots_json.js`
* catalog. Frames are stills on `www.houstontranstar.org`, registered from
* a validated numeric id rather than from the payload URL. The catalog host
* answers `Accept: application/json` with HTTP 406, so this request accepts
* any media type.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadHoustonSourcesFromOpenData() {
try {
const endpoint =
process.env.CCTV_HOUSTON_CAMERAS_URL || HOUSTON_CAMERAS_URL;
const resp = await fetch(endpoint, {
headers: { Accept: '*/*' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Houston catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Houston camera download failed:', resp.status);
return discard();
}
const payload = await readResponseJsonCapped(
resp,
HOUSTON_MAX_CATALOG_BYTES,
);
const rows = Array.isArray(payload?.cameras) ? payload.cameras : [];
const cameras = [];
const seen = new Set();
for (const record of rows) {
const camera = houstonCameraToSource(record);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_HOUSTON_MAX_SOURCES || DEFAULT_HOUSTON_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
: DEFAULT_HOUSTON_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, HOUSTON_ANCHORS);
console.log(
`[CCTV] Loaded Houston camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Houston camera download error:',
error?.message || error,
);
return [];
}
}
server/providers/common/boundedCache.js
+137
View File
@@ -0,0 +1,137 @@
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
/**
* A bounded two-level cache for JSON answers: memory (entry count) and disk
* (entry count, total bytes, age). Every memory insertion goes through one
* function, so a disk hit is bounded like a fresh answer; disk writes are
* followed by a prune that removes expired files first, then the oldest,
* until both quotas hold.
*
* @param {object} options
* @param {string|null} options.dir Disk directory; null keeps memory only.
* @param {number} options.ttlMs Age past which an entry is not served.
* @param {number} [options.maxMemoryEntries]
* @param {number} [options.maxDiskEntries]
* @param {number} [options.maxDiskBytes]
* @param {() => number} [options.now]
*/
export function createBoundedCache({
dir,
ttlMs,
maxMemoryEntries = 60,
maxDiskEntries = 400,
maxDiskBytes = 256 * 1024 * 1024,
now = () => Date.now(),
}) {
const memory = new Map();
let pruning = null;
let prunePending = false;
const fileFor = (key) =>
path.join(dir, `${createHash('sha1').update(key).digest('hex')}.json`);
function remember(key, entry) {
memory.delete(key);
memory.set(key, entry);
while (memory.size > maxMemoryEntries)
memory.delete(memory.keys().next().value);
}
async function prune() {
if (!dir) return;
let names;
try {
names = (await fsp.readdir(dir)).filter((name) => name.endsWith('.json'));
} catch {
return;
}
const files = [];
for (const name of names) {
const file = path.join(dir, name);
try {
const stat = await fsp.stat(file);
files.push({ file, size: stat.size, mtime: stat.mtimeMs });
} catch {
/* removed meanwhile */
}
}
const cutoff = now() - ttlMs;
const keep = [];
for (const entry of files) {
if (entry.mtime < cutoff) await fsp.rm(entry.file, { force: true });
else keep.push(entry);
}
keep.sort((a, b) => a.mtime - b.mtime);
let bytes = keep.reduce((sum, entry) => sum + entry.size, 0);
while (
keep.length &&
(keep.length > maxDiskEntries || bytes > maxDiskBytes)
) {
const oldest = keep.shift();
bytes -= oldest.size;
await fsp.rm(oldest.file, { force: true });
}
}
/** Run one prune at a time; a request during a prune schedules one more. */
function schedulePrune() {
if (pruning) {
prunePending = true;
return pruning;
}
pruning = prune()
.catch(() => {})
.finally(() => {
pruning = null;
if (prunePending) {
prunePending = false;
schedulePrune();
}
});
return pruning;
}
return {
/** The cached value, or undefined. A disk hit is re-inserted into memory. */
async get(key) {
const held = memory.get(key);
if (held && now() - held.cachedAt <= ttlMs) return held.value;
if (held) memory.delete(key);
if (!dir) return undefined;
try {
const entry = JSON.parse(await fsp.readFile(fileFor(key), 'utf8'));
if (!Number.isFinite(entry?.cachedAt) || now() - entry.cachedAt > ttlMs)
return undefined;
remember(key, entry);
return entry.value;
} catch {
return undefined;
}
},
/** Store a value in memory and, when a directory is set, on disk. */
async set(key, value) {
const entry = { value, cachedAt: now() };
remember(key, entry);
if (!dir) return;
try {
await fsp.mkdir(dir, { recursive: true });
await fsp.writeFile(fileFor(key), JSON.stringify(entry));
} catch {
return;
}
// In the background: an answer never waits on a 400-file directory scan.
// `prune()` returns the pending pass for callers that need it done.
schedulePrune();
},
/** Run a prune and wait until no pass is running or queued. */
async prune() {
schedulePrune();
while (pruning) await pruning;
},
get memorySize() {
return memory.size;
},
};
}
server/providers/common/rate-limit.js
+44
View File
@@ -1,3 +1,4 @@
import { isIPv6 } from 'node:net';
import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
@@ -60,3 +61,46 @@ export function makeCostRateLimiter(envValue, defaultPerMin) {
export function clientKey(req) {
return String(req.socket?.remoteAddress || 'local');
}
const IPV4_MAPPED_RE = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i;
/**
* Whether a socket peer address is this machine: IPv4 loopback in any of its
* forms (`127.0.0.0/8`, including the IPv4-mapped `::ffff:127.x` a dual-stack
* listener reports) or the IPv6 loopback. Non-address keys are not loopback.
* @param {string} address - A `clientKey` value.
* @returns {boolean}
*/
export function isLoopbackPeer(address) {
const raw = String(address || '');
const v4 = IPV4_MAPPED_RE.exec(raw)?.[1] ?? raw;
return v4.startsWith('127.') || raw === '::1';
}
/**
* Rate-limit key that survives address rotation, for limiters whose global
* backstop everyone shares. IPv6 privacy extensions hand one host a fresh
* address at will inside its /64, so keying by the full address lets a single
* host mint per-peer quota until the global window closes for everyone else.
* IPv6 peers are keyed by their /64 prefix instead, and an IPv4-mapped
* address folds to its IPv4 form so one host cannot hold two buckets. Any
* other value (IPv4, or the `local` fallback) passes through unchanged.
* @param {import('node:http').IncomingMessage} req
* @returns {string}
*/
export function clientNetworkKey(req) {
const address = clientKey(req);
const mapped = IPV4_MAPPED_RE.exec(address);
if (mapped) return mapped[1];
const bare = address.replace(/%.*$/, ''); // link-local zone id
if (!isIPv6(bare)) return address;
const [head, tail = ''] = bare.split('::');
const headParts = head ? head.split(':') : [];
const tailParts = tail ? tail.split(':') : [];
const missing = Math.max(0, 8 - headParts.length - tailParts.length);
const hextets = [...headParts, ...Array(missing).fill('0'), ...tailParts];
return `${hextets
.slice(0, 4)
.map((h) => h.toLowerCase().replace(/^0+(?=.)/, ''))
.join(':')}::/64`;
}
server/providers/common/rateLimitKeys.test.mjs
+37
View File
@@ -0,0 +1,37 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { clientNetworkKey, isLoopbackPeer } from './rate-limit.js';
const req = (remoteAddress) => ({ socket: { remoteAddress } });
test('clientNetworkKey keys IPv6 peers by /64 and folds IPv4-mapped peers', () => {
assert.equal(clientNetworkKey(req('2001:db8:1:2::1')), '2001:db8:1:2::/64');
assert.equal(
clientNetworkKey(req('2001:0db8:0001:0002:abcd:ef01:2345:6789')),
'2001:db8:1:2::/64',
);
assert.equal(clientNetworkKey(req('2001:DB8::1')), '2001:db8:0:0::/64');
assert.equal(clientNetworkKey(req('fe80::1%en0')), 'fe80:0:0:0::/64');
assert.equal(clientNetworkKey(req('::1')), '0:0:0:0::/64');
assert.equal(clientNetworkKey(req('::ffff:10.0.0.5')), '10.0.0.5');
assert.equal(clientNetworkKey(req('10.0.0.5')), '10.0.0.5');
assert.equal(clientNetworkKey(req(undefined)), 'local');
assert.equal(clientNetworkKey(req('not-an-address')), 'not-an-address');
});
test('isLoopbackPeer recognizes every loopback form and nothing else', () => {
for (const address of ['127.0.0.1', '127.9.9.9', '::1', '::ffff:127.0.0.1']) {
assert.equal(isLoopbackPeer(address), true, address);
}
for (const address of [
'10.0.0.1',
'::ffff:10.0.0.1',
'2001:db8::1',
'::2',
'local',
'',
undefined,
]) {
assert.equal(isLoopbackPeer(address), false, String(address));
}
});
server/providers/common/same-site.js
+1 -1
View File
@@ -58,7 +58,7 @@ function forwardedProtocol(req) {
/**
* Cross-site gate for keyless routes the map needs in order to render
* (`/api/terrain/heights`). It refuses the same cross-site browser shapes as
* (`/api/terrain/heights`, `/api/overpass`). It refuses the same cross-site browser shapes as
* `admitSameSite` (a `Sec-Fetch-Site` other than same-origin/none, a foreign
* or opaque `Origin`) but does not refuse reverse-proxy forwarding headers:
* those protect credentials and spend, which a keyless route has neither of,
server/providers/common/textChat.js
+221
View File
@@ -0,0 +1,221 @@
import { readRequestBody } from './request.js';
import {
clientKey,
makeCostRateLimiter,
resolvePerMinuteCap,
} from './rate-limit.js';
import { waitForSignal } from '../../../src/sources/requestDeadline.js';
import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
import { isGeminiRequestOriginAllowed } from '../gemini/request.js';
import { parseUsage } from './textTools.js';
const MAX_BYTES = 512 * 1024;
const FAILURE = 'Could not connect. Try again.';
// A rejected key is not retried upstream for this long, unless the key changes.
const REJECTED_KEY_COOLDOWN_MS = 60_000;
// What an upstream failure says about the key or the account. Gemini answers
// an invalid key with 400 and an ErrorInfo reason, not 401; OpenAI answers an
// account without credit with 429 insufficient_quota, which waiting won't fix.
// A 403 is the key's project, region or permissions, so it cools down too but
// does not blame the key. Only small error bodies are read.
async function upstreamRefusal(response, signal) {
const { status } = response;
if (status === 401) return 'API_KEY_REJECTED';
if (status === 403) return 'ACCESS_DENIED';
if (![400, 429].includes(status)) return null;
let error;
try {
error = (await readResponseJsonCapped(response, 16 * 1024, signal))?.error;
} catch {
return null;
}
if (
status === 400 &&
Array.isArray(error?.details) &&
error.details.some((detail) => detail?.reason === 'API_KEY_INVALID')
)
return 'API_KEY_REJECTED';
if (status === 429 && error?.code === 'insufficient_quota')
return 'QUOTA_EXCEEDED';
return null;
}
// Pass through only a delay in whole seconds, capped at an hour.
function upstreamRetryAfter(response) {
const value = String(response.headers?.get?.('retry-after') ?? '').trim();
if (!/^\d{1,6}$/.test(value)) return null;
return String(Math.min(Math.max(Number(value), 1), 3600));
}
export function createTextChatHandler({
provider,
prepare,
parse,
fetchImpl = (...args) => fetch(...args),
resolveApiKey,
resolveHost = () => process.env.HOST,
resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
resolveRateLimit = () =>
process.env[`GEV_RATELIMIT_${provider.toUpperCase()}_CHAT_PER_MIN`],
// An unset chat throttle follows the provider's own cost throttle.
resolveProviderRateLimit = () => undefined,
providerPerMin = 30,
// Returns { error, code } when the route must refuse before any key is used.
refuse = () => null,
now = Date.now,
timeoutMs = 60000,
} = {}) {
let limiter;
let rejected = null;
return async (req, res) => {
if (req.aborted || res.destroyed || res.writableEnded) return;
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('X-Content-Type-Options', 'nosniff');
const reply = (status, value, code) => {
if (req.aborted || res.destroyed || res.writableEnded) return;
res.statusCode = status;
if (status >= 400) res.setHeader('Connection', 'close');
res.end(
JSON.stringify(
typeof value === 'string'
? { error: value, ...(code ? { code } : {}) }
: value,
),
);
};
if (req.method !== 'POST') {
res.setHeader('Allow', 'POST');
return reply(405, 'Method not allowed');
}
const lifetime = new AbortController();
const disconnect = () => {
if (!res.writableEnded) lifetime.abort();
};
req.once('aborted', disconnect);
res.once('close', disconnect);
const timer = setTimeout(() => lifetime.abort(), timeoutMs);
timer.unref?.();
let response;
let usage = null;
try {
if (
!isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
)
return reply(403, FAILURE);
const refusal = refuse();
if (refusal) return reply(503, refusal.error, refusal.code);
const apiKey = String(resolveApiKey() || '').trim();
if (!apiKey)
return reply(
503,
'Add an API key in Provider Settings',
'API_KEY_MISSING',
);
if (rejected && (rejected.key !== apiKey || now() >= rejected.until))
rejected = null;
if (rejected) {
res.setHeader(
'Retry-After',
String(Math.max(1, Math.ceil((rejected.until - now()) / 1000))),
);
return reply(502, FAILURE, rejected.code);
}
if (limiter === undefined)
limiter = makeCostRateLimiter(
resolveRateLimit(),
resolvePerMinuteCap(resolveProviderRateLimit(), providerPerMin),
);
if (limiter && !limiter(clientKey(req))) {
res.setHeader('Retry-After', '60');
return reply(429, FAILURE, 'RATE_LIMITED');
}
if (
!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')
)
return reply(415, FAILURE);
if (Number(req.headers['content-length']) > MAX_BYTES)
return reply(413, FAILURE);
const raw = await readRequestBody(req, MAX_BYTES, lifetime.signal);
let input;
try {
input = JSON.parse(raw);
} catch {
return reply(400, FAILURE);
}
const prepared = prepare(input);
if (!prepared) return reply(400, FAILURE);
if (prepared.errorStatus) return reply(prepared.errorStatus, FAILURE);
const { endpoint, body, headers } = prepared;
lifetime.signal.throwIfAborted();
const acquisition = Promise.resolve(
fetchImpl(endpoint, {
method: 'POST',
redirect: 'error',
signal: lifetime.signal,
headers: { 'Content-Type': 'application/json', ...headers(apiKey) },
body: JSON.stringify(body),
}),
);
void acquisition.then(
(late) => {
response = late;
if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
},
() => {},
);
response = await waitForSignal(acquisition, lifetime.signal);
lifetime.signal.throwIfAborted();
if (response.redirected || (response.url && response.url !== endpoint))
throw new Error('Unexpected destination');
if (!response.ok) {
const refusal = await upstreamRefusal(response, lifetime.signal);
if (refusal) {
void response.body?.cancel().catch(() => {});
rejected = {
key: apiKey,
code: refusal,
until: now() + REJECTED_KEY_COOLDOWN_MS,
};
return reply(502, FAILURE, refusal);
}
void response.body?.cancel().catch(() => {});
if (![429, 503].includes(response.status)) return reply(502, FAILURE);
const retryAfter = upstreamRetryAfter(response);
if (retryAfter) res.setHeader('Retry-After', retryAfter);
return reply(
response.status,
FAILURE,
response.status === 429 ? 'RATE_LIMITED' : undefined,
);
}
const data = await readResponseJsonCapped(
response,
MAX_BYTES,
lifetime.signal,
);
lifetime.signal.throwIfAborted();
usage = parseUsage(provider, data);
const result = parse(data);
if (!result || JSON.stringify(result).includes(apiKey))
return reply(502, { error: FAILURE, usage });
reply(200, { ...result, usage });
} catch (error) {
reply(
error?.code === 'BODY_TOO_LARGE'
? 413
: lifetime.signal.aborted
? 504
: 502,
{ error: FAILURE, usage },
);
} finally {
if (lifetime.signal.aborted)
void response?.body?.cancel().catch(() => {});
clearTimeout(timer);
req.off('aborted', disconnect);
res.off('close', disconnect);
}
};
}
server/providers/common/textInstructions.js
+32
View File
@@ -0,0 +1,32 @@
import { realtimeDirectives } from '../openai/instructions.js';
// Voice wording in the shared directives and its written form. A test checks
// each phrase still occurs in the directives, so rewording one there fails
// rather than leaking spoken wording into chat.
export const TEXT_WORDING = Object.freeze([
['without speaking first', 'without writing a preamble'],
['before speaking', 'before confirming'],
['may first say ONE sentence', 'may first write ONE sentence'],
["speak the result's say line", "write the result's say line"],
['on the voice card: speak a caveat', 'in the tool result: include a caveat'],
['as you talk', 'as you write'],
['you can read aloud', 'you can report'],
]);
// Shared-manual adaptation follows Darren Dignam's GEV COMMAND approach (#976).
// Keep our Radio handoff and annotation outcome policies from the shared manual.
// `options.overpass` matches the tools offered, as for voice.
export function textInstructions(annotationGuidance, options) {
return realtimeDirectives(annotationGuidance, options)
.map((line, index) => {
if (index === 0)
return "You are GEV Chat, a concise text assistant for a Cesium geospatial app called God's Eye View.";
if (index === 1)
return 'Have a natural written conversation. This is text-only chat; no microphone or generated audio is used.';
return TEXT_WORDING.reduce(
(text, [spoken, written]) => text.replace(spoken, written),
line,
);
})
.join('\n');
}
server/providers/common/textTools.js
+54
View File
@@ -0,0 +1,54 @@
import { validateValue } from '../../../src/tools/schema.js';
export function invalidCalls(calls, tools) {
const schemas = new Map(tools.map((tool) => [tool.name, tool.parameters]));
return calls.flatMap((call, index) => {
const schema = schemas.get(call.name);
const errors = !schema
? ['Unknown tool']
: call.parseError
? ['Arguments must be valid JSON']
: validateValue(schema, call.args === undefined ? {} : call.args);
return errors.length
? [{ index, error: errors.slice(0, 4).join('; ').slice(0, 1000) }]
: [];
});
}
// Missing usage stays unknown, never an invented zero-cost request.
export function parseUsage(provider, data) {
const raw = provider === 'gemini' ? data?.usageMetadata : data?.usage;
const number = (value) =>
Number.isSafeInteger(value) && value >= 0 ? value : null;
const inputTokens = number(
provider === 'gemini' ? raw?.promptTokenCount : raw?.prompt_tokens,
);
const candidateTokens = number(
provider === 'gemini' ? raw?.candidatesTokenCount : raw?.completion_tokens,
);
const totalTokens = number(
provider === 'gemini' ? raw?.totalTokenCount : raw?.total_tokens,
);
if (inputTokens === null || candidateTokens === null || totalTokens === null)
return null;
const reasoningTokens =
number(
provider === 'gemini'
? raw?.thoughtsTokenCount
: raw?.completion_tokens_details?.reasoning_tokens,
) ?? 0;
const outputTokens =
candidateTokens + (provider === 'gemini' ? reasoningTokens : 0);
return {
inputTokens,
outputTokens,
totalTokens,
reasoningTokens,
cachedInputTokens:
number(
provider === 'gemini'
? raw?.cachedContentTokenCount
: raw?.prompt_tokens_details?.cached_tokens,
) ?? 0,
};
}
server/providers/cyclones.js
+6 -1
View File
@@ -43,6 +43,11 @@ function number(value, min, max) {
const n = Number(value);
return Number.isFinite(n) && n >= min && n <= max ? n : null;
}
// CurrentStorms.json reports movementSpeed in miles per hour (intensity is in
// knots); convert so `speedKt` means what it says.
function knotsFromMph(mph) {
return mph === null ? null : Math.round(mph * 0.868976);
}
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
// up to about half an hour before; allow that, but not a clock far ahead.
const EARLY_ADVISORY_MS = 90 * 60_000;
@@ -120,7 +125,7 @@ export function parseCycloneStatus(payload, now = Date.now()) {
pressureHpa: number(raw.pressure, 800, 1100),
movement: {
directionDegrees: number(raw.movementDir, 0, 360),
speedKt: number(raw.movementSpeed, 0, 200),
speedKt: knotsFromMph(number(raw.movementSpeed, 0, 200)),
},
advisoryUrl: officialLink(forecast.url),
outlookUrl: raw.id.startsWith('al')
server/providers/droneAirspace.js
+157
View File
@@ -0,0 +1,157 @@
import { normalizeDroneAirspaceSnapshot } from '../../src/layers/droneAirspace/records.js';
import { readResponseJsonCapped, coalesceProxyRequest } from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSiteKeyless } from './common/same-site.js';
// FAA Special Use Airspace (public, keyless). TYPE_CODE P (prohibited) and
// R (restricted) only — the small, always-in-force slice of the national
// SUA table. maxAllowableOffset trades ~100 m of boundary fidelity for a
// payload small enough to refresh (~330 KB for the current P+R set).
const API_URL =
'https://services6.arcgis.com/ssFJjBXIUyZDrSYZ/arcgis/rest/services/' +
'Special_Use_Airspace/FeatureServer/0/query?' +
new URLSearchParams({
where: "TYPE_CODE IN ('P','R')",
outFields: [
'NAME',
'TYPE_CODE',
'UPPER_VAL',
'UPPER_UOM',
'UPPER_CODE',
'LOWER_VAL',
'LOWER_UOM',
'LOWER_CODE',
'TIMESOFUSE',
'CONT_AGENT',
'CITY',
'STATE',
'OBJECTID',
].join(','),
maxAllowableOffset: '0.001',
outSR: '4326',
f: 'geojson',
}).toString();
const MAX_PAGES = 5;
const MIB = 1024 * 1024;
const SNAPSHOT_TTL_MS = 3_600_000;
/** Longest a failed refresh keeps serving the last snapshot, marked stale. */
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
const exceededTransferLimit = (payload) =>
payload?.exceededTransferLimit === true ||
payload?.properties?.exceededTransferLimit === true;
/** Fixed-origin, bounded FAA SUA routes for dev and preview. */
export function droneAirspaceProxy({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
} = {}) {
const cache = new Map();
const inFlight = new Map();
const allow = makeRateLimiter({ windowMs: 60_000, max: 60, globalMax: 1200 });
async function upstream(url, cap, timeout) {
const signal = AbortSignal.timeout(timeout);
const response = await fetchImpl(url, {
signal,
redirect: 'error',
});
if (!response.ok) {
await response.body?.cancel();
throw new Error('upstream_unavailable');
}
return readResponseJsonCapped(response, cap, signal);
}
// A snapshot is accepted only once the service says it is complete. A page
// limit reached while still flagged, a flagged page that adds nothing, or
// no valid zone at all is a failed refresh, so the last good snapshot stays.
async function fetchZones() {
const features = [];
let complete = false;
for (let page = 0; page < MAX_PAGES; page++) {
const url =
page === 0 ? API_URL : `${API_URL}&resultOffset=${features.length}`;
const payload = await upstream(url, 8 * MIB, 30_000);
if (!Array.isArray(payload?.features))
throw new Error('invalid_snapshot');
features.push(...payload.features);
if (!exceededTransferLimit(payload)) {
complete = true;
break;
}
if (!payload.features.length) break;
}
if (!complete) throw new Error('incomplete_snapshot');
const rows = normalizeDroneAirspaceSnapshot({ features });
if (!rows?.length) throw new Error('invalid_snapshot');
return { fetchedAt: now(), rows };
}
async function acquire(key, ttl, load) {
const previous = cache.get(key);
if (previous && now() - previous.savedAt < ttl)
return { value: previous.value, stale: false };
if (!inFlight.has(key) && inFlight.size >= 256)
throw Object.assign(new Error('busy'), { status: 429 });
try {
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
const value = await load();
cache.set(key, { value, savedAt: now() });
return value;
});
return { value: await promise, stale: false };
} catch (error) {
// A failed refresh serves the last snapshot, marked stale, for a day.
if (previous && now() - previous.savedAt <= STALE_MAX_MS)
return { value: previous.value, stale: true };
throw error;
}
}
async function handler(req, res) {
const json = (status, value, stale = false) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...(status === 429 ? { 'Retry-After': '60' } : {}),
...(stale ? { 'X-Data-Stale': 'true' } : {}),
});
res.end(JSON.stringify(value));
};
// Keyless and needed to render the layer: refuse cross-site browsers
// before they spend a client's quota or start a refresh, but keep the
// documented reverse-proxy deployment working.
if (admitSameSiteKeyless(req, res)) return;
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
if (path !== '/' && path !== '')
return json(404, { error: 'unknown_route' });
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
try {
const { value, stale } = await acquire(
'zones',
SNAPSHOT_TTL_MS,
fetchZones,
);
json(200, stale ? { ...value, stale: true } : value, stale);
} catch (error) {
json(error.status === 429 ? 429 : 502, {
error: 'drone_airspace_unavailable',
});
}
}
return {
name: 'drone-airspace',
configureServer({ middlewares }) {
middlewares.use('/api/drone-airspace', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/drone-airspace', handler);
},
};
}
server/providers/firePerimeters.js
+10 -1
View File
@@ -39,6 +39,8 @@ const API_URL =
// The service caps a single response at its maxRecordCount (2000); a peak
// season can exceed that, so follow exceededTransferLimit with offset pages.
const MAX_PAGES = 5;
/** Longest a failed refresh keeps serving the last perimeter snapshot, marked stale. */
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
const exceededTransferLimit = (payload) =>
payload?.exceededTransferLimit === true ||
@@ -165,7 +167,14 @@ export function firePerimetersProxy({
});
return { value: await promise, stale: false };
} catch (error) {
if (previous) return { value: previous.value, stale: true };
// A failed refresh serves the last answer, marked stale. A perimeter
// snapshot only for a day: older than that, it is no picture of
// current fires. (InciWeb entries keep their previous behaviour.)
if (
previous &&
(key !== 'perimeters' || now() - previous.savedAt <= STALE_MAX_MS)
)
return { value: previous.value, stale: true };
throw error;
}
}
server/providers/firms.js
+138 -6
View File
@@ -1,9 +1,132 @@
import path from 'node:path';
import https from 'node:https';
import { promises as fsp } from 'node:fs';
import { Readable } from 'node:stream';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import { readResponseTextCapped } from './common/http.js';
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
/**
* The per-address connection errors behind a fetch failure, or null. Node 24
* puts the AggregateError directly on `err.cause`; Node 26's fetch wraps it in
* a ConnectTimeoutError one level deeper.
* @param {unknown} err
* @returns {?Array<unknown>}
*/
function connectAttemptErrors(err) {
let cause = err?.cause;
for (let depth = 0; cause && depth < 3; depth++) {
if (Array.isArray(cause.errors)) return cause.errors;
cause = cause.cause;
}
return null;
}
/**
* Describe a fetch failure with its socket-level cause. Node reports every
* connection failure as `fetch failed`; the cause carries the error code and
* the address that failed (never the URL, which embeds the MAP_KEY).
* @param {unknown} err
* @returns {string}
*/
export function describeFetchError(err) {
const message = err?.message || String(err);
const cause = err?.cause;
if (!cause) return message;
const attempts = connectAttemptErrors(err);
const parts = attempts?.length
? attempts.map((inner) => inner?.message || inner?.code || String(inner))
: [cause.message || cause.code || String(cause)];
return `${message} (${parts.join('; ')})`;
}
/**
* Whether a fetch failed because every address it tried failed to connect.
* Node's network-family auto-selection gives each address a short attempt
* timeout, so a slow IPv4 handshake is abandoned in favour of an unreachable
* IPv6 address and the whole request fails (#68).
* @param {unknown} err
* @returns {boolean}
*/
export function isAddressSelectionFailure(err) {
return err instanceof TypeError && connectAttemptErrors(err)?.length > 0;
}
/**
* GET over IPv4 only, with one connection attempt bounded by the caller's
* signal rather than by the auto-selection attempt timeout.
* @param {string} url
* @param {{signal?: AbortSignal}} [options]
* @returns {Promise<Response>}
*/
export function fetchIpv4(url, { signal } = {}) {
return new Promise((resolve, reject) => {
const request = https.request(
url,
{
method: 'GET',
signal,
family: 4,
autoSelectFamily: false,
agent: false,
},
(response) => {
// Nothing thrown here may escape: this callback runs outside the
// promise, so an exception would be uncaught and crash the server.
try {
const status = response.statusCode || 500;
// A `Response` only represents statuses up to 599; Node never hands
// this callback a 1xx.
if (status > 599)
throw new Error(`FIRMS returned unsupported status ${status}`);
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value))
value.forEach((item) => headers.append(name, item));
else if (value !== undefined) headers.set(name, String(value));
}
const nullBody = NULL_BODY_STATUSES.has(status);
const upstream = new Response(
nullBody ? null : Readable.toWeb(response),
{ status, statusText: response.statusMessage || '', headers },
);
if (nullBody) response.resume();
resolve(upstream);
} catch (error) {
response.destroy();
reject(error);
}
},
);
request.on('error', reject);
request.end();
});
}
/**
* Fetch a FIRMS URL, retrying once over IPv4 when address selection fails.
* Other failures (HTTP status, timeout, DNS) are returned or thrown as is.
* @param {string} url
* @param {{signal?: AbortSignal}} options
* @param {{fetchImpl?: typeof fetch, fetchIpv4Impl?: typeof fetchIpv4, warn?: Function}} [deps]
* @returns {Promise<Response>}
*/
export async function fetchFirms(
url,
options,
{ fetchImpl = fetch, fetchIpv4Impl = fetchIpv4, warn = console.warn } = {},
) {
try {
return await fetchImpl(url, options);
} catch (err) {
if (!isAddressSelectionFailure(err) || options?.signal?.aborted) throw err;
warn(`[firms-proxy] ${describeFetchError(err)} — retrying over IPv4`);
return fetchIpv4Impl(url, options);
}
}
/**
* NASA FIRMS live active-fire proxy with a memory + disk cache.
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
@@ -91,8 +214,12 @@ export function firmsProxy() {
async function fetchSource(key, source) {
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
const signal = AbortSignal.timeout(60_000);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const res = await fetchFirms(url, { signal });
if (!res.ok) {
// Release the connection now rather than when the deadline fires.
await res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const records = parseFirmsCsv(
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
);
@@ -121,7 +248,7 @@ export function firmsProxy() {
} catch (err) {
console.warn(
`[firms-proxy] ${source} fetch failed:`,
err?.message || err,
describeFetchError(err),
);
sources.push({ source, count: 0, ok: false });
}
@@ -156,8 +283,13 @@ export function firmsProxy() {
statusInflight = (async () => {
try {
const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const res = await fetchFirms(url, {
signal: AbortSignal.timeout(10_000),
});
if (!res.ok) {
await res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await res.json();
const used = Number(body?.current_transactions);
const limit = Number(body?.transaction_limit);
@@ -167,7 +299,7 @@ export function firmsProxy() {
} catch (err) {
console.warn(
'[firms-proxy] mapkey status failed:',
err?.message || err,
describeFetchError(err),
);
return null;
}
server/providers/gemini.js
+5
View File
@@ -1,8 +1,13 @@
import { createGeminiChatHandler } from './gemini/chat.js';
import { createGeminiTokenHandler } from './gemini/realtime.js';
/** Mint constrained Live API credentials; the permanent key stays on Node. */
function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
function install(middlewares) {
middlewares.use(
'/api/gemini/chat',
createGeminiChatHandler({ ...realtime, annotationGuidance }),
);
middlewares.use(
'/api/gemini/token',
createGeminiTokenHandler({ ...realtime, annotationGuidance }),
server/providers/gemini/chat.js
+188
View File
@@ -0,0 +1,188 @@
import { createTextChatHandler } from '../common/textChat.js';
import { invalidCalls } from '../common/textTools.js';
import { textInstructions } from '../common/textInstructions.js';
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
import { normalizeGeminiModel } from './config.js';
// The Gemini token route's throttle default (GEV_RATELIMIT_GEMINI_PER_MIN).
export const GEMINI_DEFAULT_PER_MIN = 30;
const object = (value) =>
value && typeof value === 'object' && !Array.isArray(value);
const keys = (value, allowed) =>
Object.keys(value).every((key) => allowed.includes(key));
// Only text and function exchanges may cross this route, never media or config.
export function validChatContents(contents, names) {
return (
Array.isArray(contents) &&
contents.length > 0 &&
contents.length <= 96 &&
contents.every(
(content) =>
object(content) &&
keys(content, ['role', 'parts']) &&
['user', 'model'].includes(content.role) &&
Array.isArray(content.parts) &&
content.parts.length > 0 &&
content.parts.length <= 32 &&
content.parts.every((part) => {
if (
!object(part) ||
!keys(part, [
'text',
'functionCall',
'functionResponse',
'thoughtSignature',
])
)
return false;
if (
part.thoughtSignature !== undefined &&
(typeof part.thoughtSignature !== 'string' ||
part.thoughtSignature.length > 32000 ||
content.role !== 'model')
)
return false;
if (Object.hasOwn(part, 'text'))
return (
keys(part, ['text', 'thoughtSignature']) &&
typeof part.text === 'string'
);
const call = part.functionCall ?? part.functionResponse;
if (
!object(call) ||
typeof call.name !== 'string' ||
call.name.length > 128 ||
(names && !names.has(call.name)) ||
(call.id !== undefined &&
(typeof call.id !== 'string' || call.id.length > 1024))
)
return false;
if (part.functionCall)
return (
content.role === 'model' &&
keys(part, ['functionCall', 'thoughtSignature']) &&
keys(call, ['name', 'args', 'id']) &&
(call.args === null || object(call.args ?? {}))
);
return (
content.role === 'user' &&
keys(part, ['functionResponse']) &&
keys(call, ['name', 'response', 'id']) &&
object(call.response)
);
}),
)
);
}
export function createGeminiChatHandler({
annotationGuidance,
tools = GEV_REALTIME_TOOLS,
// osm_query is offered, and the prompt names it, only with Overpass, as
// for voice.
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
resolveChatModel = () =>
process.env.GEMINI_CHAT_MODEL || 'gemini-3.5-flash-lite',
resolveApiKey = () => process.env.GEMINI_API_KEY,
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN,
...options
} = {}) {
return createTextChatHandler({
...options,
provider: 'gemini',
resolveApiKey,
resolveProviderRateLimit,
providerPerMin: GEMINI_DEFAULT_PER_MIN,
prepare(input) {
const model = normalizeGeminiModel(resolveChatModel());
if (!model || /live|audio|tts/i.test(model)) return { errorStatus: 503 };
if (
!object(input) ||
!keys(input, ['contents']) ||
!validChatContents(input.contents, null)
)
return null;
if (!pairedContents(input.contents)) return null;
const overpass = overpassConfigured();
return {
endpoint: `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
headers: (key) => ({ 'x-goog-api-key': key }),
body: {
contents: input.contents,
systemInstruction: {
parts: [
{ text: textInstructions(annotationGuidance, { overpass }) },
],
},
tools: [
{
functionDeclarations: offeredTools(tools, { overpass }).map(
({ name, description, parameters }) => ({
name,
description,
parametersJsonSchema: parameters,
}),
),
},
],
generationConfig: {
responseModalities: ['TEXT'],
maxOutputTokens: 4096,
thinkingConfig: { thinkingLevel: 'LOW' },
},
},
};
},
parse(data) {
const candidate = data?.candidates?.[0];
const content = candidate?.content;
if (
!content ||
!validChatContents([content], null) ||
content.role !== 'model'
)
return null;
const calls = content.parts
.filter((part) => part.functionCall)
.map((part) => part.functionCall);
return {
content,
finishReason: candidate.finishReason,
invalidCalls: invalidCalls(
calls,
offeredTools(tools, { overpass: overpassConfigured() }),
),
};
},
});
}
// Never forward an orphan result or an unfinished batch in browser history.
function pairedContents(contents) {
let pending = [];
for (const content of contents) {
const calls = content.parts
.filter((part) => part.functionCall)
.map((part) => part.functionCall);
const results = content.parts
.filter((part) => part.functionResponse)
.map((part) => part.functionResponse);
if (pending.length) {
if (
calls.length ||
results.length !== pending.length ||
results.some(
(result, i) =>
result.name !== pending[i].name || result.id !== pending[i].id,
)
)
return false;
pending = [];
} else if (results.length) return false;
if (calls.length) pending = calls;
}
return !pending.length;
}
server/providers/gemini/config.js
+8 -4
View File
@@ -1,5 +1,5 @@
import { realtimeInstructions } from '../openai/instructions.js';
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
export const GEMINI_TOKEN_ENDPOINT =
@@ -14,22 +14,26 @@ export function normalizeGeminiModel(value) {
return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
}
/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
/**
* REST BidiGenerateContentSetup fields, excluding its model resource name.
* `overpass` says an operator configured Overpass, which `osm_query` needs.
*/
export function createGeminiLiveConfig(
annotationGuidance,
inputMode = 'open-mic',
tools = GEV_REALTIME_TOOLS,
{ overpass = false } = {},
) {
if (!['open-mic', 'push-to-talk'].includes(inputMode))
throw new Error('Invalid Gemini input mode');
const config = {
generationConfig: { responseModalities: ['AUDIO'] },
systemInstruction: {
parts: [{ text: realtimeInstructions(annotationGuidance) }],
parts: [{ text: realtimeInstructions(annotationGuidance, { overpass }) }],
},
tools: [
{
functionDeclarations: tools.map(
functionDeclarations: offeredTools(tools, { overpass }).map(
({ name, description, parameters }) => ({
name,
description,
server/providers/gemini/realtime.js
+4
View File
@@ -12,6 +12,7 @@ import {
isGeminiRequestOriginAllowed,
readGeminiTokenRequest,
} from './request.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
@@ -19,6 +20,8 @@ const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
export function createGeminiTokenHandler({
annotationGuidance,
tools,
// osm_query is offered only when an operator configured Overpass.
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
fetchImpl = (...args) => fetch(...args),
resolveApiKey = () => process.env.GEMINI_API_KEY,
resolveModel = () =>
@@ -104,6 +107,7 @@ export function createGeminiTokenHandler({
annotationGuidance,
inputMode,
tools,
{ overpass: overpassConfigured() },
);
const issuedAt = now();
// REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
server/providers/ghostMaps.js
+437
View File
@@ -0,0 +1,437 @@
import fsp from 'node:fs/promises';
import path from 'node:path';
import {
GHOSTMAPS_MAX_FILE_BYTES,
GHOSTMAPS_REPO,
buildGhostMapsCatalog,
ghostMapsRawUrl,
parseGitRefAdvertisement,
} from '../../src/layers/ghostMaps/catalog.js';
import {
readResponseBytesCapped,
readResponseJsonCapped,
readResponseTextCapped,
} from './common/http.js';
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
// The repository is not ours, so there is no webhook: the watcher reads the
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
// REST API rate limit) every few seconds, and only a new commit costs REST
// calls (tree + commit). Browsers hear about a new commit over server-sent
// events and fetch only the products whose blob changed. Files are fetched by
// commit-pinned raw URLs and kept on disk by blob sha.
const DEFAULT_POLL_MS = 15_000;
const MIN_POLL_MS = 5_000;
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
const TIMEOUT_MS = 15_000;
const FILE_TIMEOUT_MS = 120_000;
const HEARTBEAT_MS = 25_000;
const MAX_SUBSCRIBERS = 64;
const MAX_REF_BYTES = 64 * 1024;
const MAX_TREE_BYTES = 16 * 1024 * 1024;
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
const seconds = Number(raw);
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
}
/**
* The repository watcher and file cache, independent of HTTP routing.
* @returns {{start: Function, stop: Function, catalog: Function,
* checkNow: Function, file: Function, subscribe: Function}}
*/
export function createGhostMapsWatcher({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
pollMs = ghostMapsPollMs(),
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
log = (...args) => console.log(...args),
warn = (...args) => console.warn(...args),
} = {}) {
let state = {
commit: null,
committedAt: null,
message: null,
products: [],
checkedAt: null,
changedAt: null,
error: null,
};
let timer = null;
let running = false;
let checking = null;
let failures = 0;
let diskLoaded = false;
const listeners = new Set();
const downloads = new Map();
const apiHeaders = () => ({
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
});
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
const response = await fetchImpl(url, {
headers,
redirect: 'follow',
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
status: response.status,
});
}
return response;
}
async function readHead() {
const response = await get(
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
{ 'User-Agent': USER_AGENT },
);
const sha = parseGitRefAdvertisement(
await readResponseTextCapped(response, MAX_REF_BYTES),
);
if (!sha) throw new Error('GhostMaps branch head not advertised');
return sha;
}
async function readCommit(sha) {
const api = `https://api.github.com/repos/${REPO_PATH}`;
const tree = await readResponseJsonCapped(
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
MAX_TREE_BYTES,
);
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
let committedAt = null;
let message = null;
try {
const commit = await readResponseJsonCapped(
await get(`${api}/commits/${sha}`, apiHeaders()),
MAX_COMMIT_BYTES,
);
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
message = String(commit?.commit?.message || '')
.split('\n')[0]
.slice(0, 200);
} catch {
// The commit date is decoration; the tree is what matters.
}
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
}
async function loadDisk() {
if (diskLoaded) return;
diskLoaded = true;
try {
const saved = JSON.parse(
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
);
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
state = { ...state, ...saved, error: null };
} catch {
/* first run */
}
}
async function saveDisk() {
try {
await fsp.mkdir(cacheDir, { recursive: true });
const { commit, committedAt, message, products, changedAt } = state;
await fsp.writeFile(
path.join(cacheDir, 'catalog.json'),
JSON.stringify({ commit, committedAt, message, products, changedAt }),
);
} catch (error) {
warn('[ghostmaps] could not save catalog:', error.message);
}
}
function emit() {
const event = publicCatalog();
for (const listener of listeners) {
try {
listener(event);
} catch {
/* a broken subscriber must not stop the rest */
}
}
}
async function check() {
await loadDisk();
const head = await readHead();
state.checkedAt = now();
if (head === state.commit) {
if (state.error !== null) {
state.error = null;
emit();
}
return false;
}
const next = await readCommit(head);
const previous = new Map(state.products.map((p) => [p.id, p.version]));
const changed = next.products
.filter((p) => previous.get(p.id) !== p.version)
.map((p) => p.id);
state = {
...state,
...next,
commit: head,
changedAt: now(),
error: null,
};
log(
`[ghostmaps] commit ${head.slice(0, 8)}${
next.message ? ` "${next.message}"` : ''
}; updated: ${changed.join(', ') || 'none'}`,
);
await saveDisk();
emit();
// Warm the default products so the first browser load is local.
for (const product of state.products)
if (product.defaultOn) file(product.id).catch(() => {});
return true;
}
/** Run one check now, coalescing with one already in flight. */
function checkNow() {
if (!checking)
checking = check()
.then(
(changed) => {
failures = 0;
return changed;
},
(error) => {
failures += 1;
state.error = error.message;
state.checkedAt = now();
warn('[ghostmaps] check failed:', error.message);
emit();
return false;
},
)
.finally(() => {
checking = null;
});
return checking;
}
function schedule() {
clearTimeout(timer);
if (!running) return;
const delay = failures
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
: pollMs;
timer = setTimeout(async () => {
await checkNow();
schedule();
}, delay);
timer.unref?.();
}
/** Start watching (idempotent); the first check runs immediately. */
function start() {
if (running) return checking || Promise.resolve(false);
running = true;
const first = checkNow();
first.then(schedule);
return first;
}
function stop() {
running = false;
clearTimeout(timer);
timer = null;
}
function publicCatalog() {
return {
repo: GHOSTMAPS_REPO.url,
branch: GHOSTMAPS_REPO.branch,
commit: state.commit,
committedAt: state.committedAt,
message: state.message,
checkedAt: state.checkedAt,
changedAt: state.changedAt,
pollMs,
error: state.error,
products: state.products.map(
({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}) => ({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}),
),
};
}
/**
* The bytes of one product's current edition, from disk or a commit-pinned
* download. Only catalog paths are ever fetched.
*/
async function file(id) {
const product = state.products.find((p) => p.id === id);
if (!product) return null;
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
try {
return { product, bytes: await fsp.readFile(target) };
} catch {
/* not cached yet */
}
const commit = state.commit;
let pending = downloads.get(product.version);
if (!pending) {
pending = (async () => {
const response = await get(
ghostMapsRawUrl(commit, product.path),
{ 'User-Agent': USER_AGENT },
FILE_TIMEOUT_MS,
);
const bytes = Buffer.from(
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
);
await fsp.mkdir(path.dirname(target), { recursive: true });
const temp = `${target}.${process.pid}.tmp`;
await fsp.writeFile(temp, bytes);
await fsp.rename(temp, target);
await pruneFiles();
return bytes;
})().finally(() => downloads.delete(product.version));
downloads.set(product.version, pending);
}
return { product, bytes: await pending };
}
/** Keep only files that a current product still points to. */
async function pruneFiles() {
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
const dir = path.join(cacheDir, 'files');
for (const name of await fsp.readdir(dir).catch(() => [])) {
if (!keep.has(name) && !name.endsWith('.tmp'))
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
}
}
function subscribe(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
}
return {
start,
stop,
checkNow,
catalog: publicCatalog,
file,
subscribe,
subscribers: () => listeners.size,
};
}
/** Same-origin GhostMaps routes for dev and preview. */
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
const json = (res, status, value) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
});
res.end(JSON.stringify(value));
};
function events(req, res) {
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
return json(res, 429, { error: 'too_many_subscribers' });
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-store',
Connection: 'keep-alive',
'X-Accel-Buffering': 'no',
});
const send = (catalog) =>
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
res.write('retry: 5000\n\n');
send(watcher.catalog());
const unsubscribe = watcher.subscribe(send);
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
heartbeat.unref?.();
const close = () => {
clearInterval(heartbeat);
unsubscribe();
};
req.on('close', close);
res.on('error', close);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const url = new URL(req.url || '/', 'http://local');
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
if (!['/catalog', '/events'].includes(url.pathname) && !match)
return json(res, 404, { error: 'unknown_route' });
// Watching starts with the first request, so an install that never turns
// the layer on never contacts GitHub.
const first = watcher.start();
if (url.pathname === '/catalog') {
if (!watcher.catalog().commit) await first;
return json(res, 200, watcher.catalog());
}
if (url.pathname === '/events') return events(req, res);
try {
if (!watcher.catalog().commit) await first;
const found = await watcher.file(match[1]);
if (!found) return json(res, 404, { error: 'unknown_product' });
const pinned = url.searchParams.get('v') === found.product.version;
res.writeHead(200, {
'Content-Type': 'application/vnd.google-earth.kmz',
'Content-Length': found.bytes.length,
'X-GhostMaps-Version': found.product.version,
// A versioned URL names one immutable blob.
'Cache-Control': pinned
? 'private, max-age=31536000, immutable'
: 'no-store',
});
return res.end(found.bytes);
} catch {
return json(res, 502, { error: 'ghostmaps_unavailable' });
}
}
return {
name: 'ghostmaps',
configureServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
};
}
server/providers/ghostMaps.test.mjs
+294
View File
@@ -0,0 +1,294 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdtemp, readdir, rm } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
createGhostMapsWatcher,
ghostMapsPollMs,
ghostMapsProxy,
} from './ghostMaps.js';
const MASTER =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
const KINETIC =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
const sha = (c) => c.repeat(40);
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
function fakeGitHub() {
const repo = { head: sha('1'), trees: new Map(), calls: [] };
repo.trees.set(sha('1'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('a'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
const body = (text, status = 200) =>
new Response(text, {
status,
headers: { 'content-length': String(Buffer.byteLength(text)) },
});
const fetchImpl = async (url) => {
repo.calls.push(url);
if (url.endsWith('/info/refs?service=git-upload-pack'))
return body(
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
);
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
if (/\/commits\//.test(url))
return body(
JSON.stringify({
commit: {
message: 'Oct 18 Update\nmore',
committer: { date: '2026-10-18T00:00:00Z' },
},
}),
);
if (url.startsWith('https://raw.githubusercontent.com/'))
return body('KMZ!');
return body('nope', 404);
};
return { repo, fetchImpl };
}
async function withCache(fn) {
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
try {
await fn(dir);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
// checkNow announces the catalog before background warm-up finishes. Join the
// same file reads before changing editions or removing the temporary cache.
async function warmDefaults(watcher) {
await Promise.all(
watcher
.catalog()
.products.filter((product) => product.defaultOn)
.map((product) => watcher.file(product.id)),
);
}
test('a new commit is announced once, naming only the products that changed', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
const first = watcher.catalog();
assert.equal(first.commit, sha('1'));
assert.equal(first.message, 'Oct 18 Update');
assert.deepEqual(
first.products.map((p) => p.id),
['cip', 'kinetic-30d'],
);
assert.equal(events.length, 1);
// Same head: one ref read, no REST calls, no event.
repo.calls.length = 0;
assert.equal(await watcher.checkNow(), false);
assert.equal(repo.calls.length, 1);
assert.equal(events.length, 1);
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
repo.head = sha('2');
repo.trees.set(sha('2'), [
...repo.trees.get(sha('1')),
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
]);
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
assert.equal(events.length, 2);
const cip = events[1].products.find((p) => p.id === 'cip');
assert.equal(cip.version, sha('c'));
assert.match(cip.fileName, /October_18/);
assert.equal(
events[1].products.find((p) => p.id === 'kinetic-30d').version,
sha('b'),
);
// Repository paths stay on the server.
assert.equal('path' in cip, false);
watcher.stop();
}));
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
await watcher.checkNow();
const found = await watcher.file('kinetic-30d');
assert.equal(found.bytes.toString(), 'KMZ!');
const raw = repo.calls.filter((url) =>
url.includes('raw.githubusercontent'),
);
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
assert.ok(
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
);
const before = repo.calls.length;
await watcher.file('kinetic-30d');
assert.equal(repo.calls.length, before, 'second read comes from disk');
assert.equal(await watcher.file('not-a-product'), null);
// When the edition changes, the old blob leaves the disk cache.
await watcher.file('cip');
repo.head = sha('2');
repo.trees.set(sha('2'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
await watcher.checkNow();
await watcher.file('cip');
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
watcher.stop();
}));
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
const quiet = { log: () => {}, warn: () => {} };
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
await online.checkNow();
await warmDefaults(online);
const offline = createGhostMapsWatcher({
fetchImpl: async () => new Response('down', { status: 503 }),
cacheDir,
...quiet,
});
assert.equal(await offline.checkNow(), false);
const catalog = offline.catalog();
assert.equal(catalog.commit, sha('1'));
assert.equal(catalog.products.length, 2);
assert.match(catalog.error, /HTTP 503/);
}));
test('an unchanged head clears an upstream outage and announces recovery once', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
let unavailable = false;
let checkedAt = 1000;
const requests = [];
const watcher = createGhostMapsWatcher({
cacheDir,
now: () => checkedAt,
fetchImpl: async (url, options) => {
requests.push(url);
if (unavailable) return new Response('down', { status: 503 });
return fetchImpl(url, options);
},
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
await watcher.checkNow();
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
const healthy = watcher.catalog();
assert.equal(events.length, 1);
unavailable = true;
checkedAt = 2000;
assert.equal(await watcher.checkNow(), false);
assert.match(watcher.catalog().error, /HTTP 503/);
assert.equal(events.length, 2);
unavailable = false;
checkedAt = 3000;
requests.length = 0;
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
const recovered = watcher.catalog();
assert.equal(recovered.error, null);
assert.equal(recovered.checkedAt, checkedAt);
assert.equal(recovered.commit, healthy.commit);
assert.equal(recovered.changedAt, healthy.changedAt);
assert.deepEqual(recovered.products, healthy.products);
assert.equal(requests.length, 1, 'recovery needs only the ref read');
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
assert.equal(events[2].error, null);
assert.equal(events[2].checkedAt, checkedAt);
checkedAt = 4000;
assert.equal(await watcher.checkNow(), false);
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
watcher.stop();
}));
test('poll interval is configurable with a 5 s floor', () => {
assert.equal(ghostMapsPollMs(undefined), 15000);
assert.equal(ghostMapsPollMs('30'), 30000);
assert.equal(ghostMapsPollMs('1'), 5000);
assert.equal(ghostMapsPollMs('soon'), 15000);
});
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
let started = 0;
const watcher = {
start: () => {
started += 1;
return Promise.resolve(false);
},
catalog: () => ({ commit: sha('1'), products: [] }),
file: async () => null,
subscribe: () => () => {},
subscribers: () => 0,
};
let handler;
ghostMapsProxy({ watcher }).configureServer({
middlewares: { use: (_route, fn) => (handler = fn) },
});
const call = async (method, url) => {
const res = {
destroyed: false,
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end(body) {
this.body = String(body ?? '');
},
};
await handler({ method, url }, res);
return res;
};
assert.equal((await call('GET', '/nope')).status, 404);
assert.equal((await call('POST', '/catalog')).status, 405);
assert.equal((await call('GET', '/file/../../etc')).status, 404);
assert.equal(started, 0);
assert.equal((await call('GET', '/file/cip')).status, 404);
const catalog = await call('GET', '/catalog');
assert.equal(catalog.status, 200);
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
assert.equal(started, 2);
});
server/providers/local.js
+20 -3
View File
@@ -1,3 +1,4 @@
import path from 'node:path';
import { openSkyProxy } from './aircraft/opensky.js';
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
import { tomtomProxy } from './traffic.js';
@@ -7,7 +8,7 @@ import { adsbdbProxy } from './aircraft/enrichment.js';
import { overpassProxy } from './overpass.js';
import { militaryInstallationsProxy } from './military-installations.js';
import { regionalBriefProxy } from './regional/briefing.js';
import { geocodeProxy } from './regional/place.js';
import { geocodeProxy, NOMINATIM_CACHE_DIR } from './regional/place.js';
import { weatherEffectsProxy } from './regional/weather-effects.js';
import { cctvProxy } from './cctv.js';
import { defaultSourceRoot } from './common/source-root.js';
@@ -26,14 +27,28 @@ import { keySetupEndpoint } from '../standalone/key-setup.js';
import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { ghostMapsProxy } from './ghostMaps.js';
import { droneAirspaceProxy } from './droneAirspace.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
/**
* Construct the local provider plugins in their established order.
* `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live.
* `nominatimStorageDir` binds an owned absolute directory before construction;
* omission retains the durable per-source cache and policy state.
*/
function localProviderPlugins({ realtime, gemini } = {}) {
function localProviderPlugins({
realtime,
gemini,
nominatimStorageDir = NOMINATIM_CACHE_DIR,
} = {}) {
if (
typeof nominatimStorageDir !== 'string' ||
!nominatimStorageDir.trim() ||
!path.isAbsolute(nominatimStorageDir)
)
throw new TypeError('nominatimStorageDir must be an absolute directory');
return [
openSkyProxy(),
celestrakProxy(),
@@ -45,7 +60,7 @@ function localProviderPlugins({ realtime, gemini } = {}) {
overpassProxy(),
militaryInstallationsProxy(),
regionalBriefProxy(),
geocodeProxy(),
geocodeProxy({ storageDir: nominatimStorageDir }),
weatherEffectsProxy(),
cctvProxy({ sourceRoot: defaultSourceRoot }),
radioBrowserProxy(),
@@ -64,6 +79,8 @@ function localProviderPlugins({ realtime, gemini } = {}) {
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
ghostMapsProxy(),
droneAirspaceProxy(),
keySetupEndpoint(),
];
}
server/providers/military-installations.js
+2
View File
@@ -1,4 +1,5 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSite } from './common/same-site.js';
import {
MILITARY_INSTALLATION_ELEMENT_CAP,
MILITARY_INSTALLATION_MAX_RESPONSE_BYTES,
@@ -97,6 +98,7 @@ function militaryInstallationsProxy() {
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
if (admitSameSite(req, res)) return;
const url = new URL(req.url, 'http://localhost');
const requested = validMilitaryInstallationBox(url.searchParams);
if (!requested) {
server/providers/openai.js
+5
View File
@@ -1,3 +1,4 @@
import { createOpenAiChatHandler } from './openai/chat.js';
import { defaultSourceRoot } from './common/source-root.js';
import { handleHudSummary } from './openai/hud-summary.js';
import { createDebugLogHandler } from './openai/debug-log.js';
@@ -22,6 +23,10 @@ function openAiRealtimeProxy({
} = {}) {
const oauthLogin = createCodexOAuthLogin(realtime);
function install(middlewares) {
middlewares.use(
'/api/openai/chat',
createOpenAiChatHandler({ ...realtime, annotationGuidance }),
);
// Cost-bearing and log endpoints refuse cross-site browser requests
// (see server/providers/common/same-site.js and SECURITY.md).
middlewares.use('/api/openai/hud-summary', sameSiteGated(handleHudSummary));
server/providers/openai/chat.js
+169
View File
@@ -0,0 +1,169 @@
import { createTextChatHandler } from '../common/textChat.js';
import { preferCodexOAuth } from './codex-auth.js';
import { OPENAI_DEFAULT_PER_MIN } from './rate-limit.js';
import { invalidCalls } from '../common/textTools.js';
import { textInstructions } from '../common/textInstructions.js';
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
const object = (value) =>
value && typeof value === 'object' && !Array.isArray(value);
const keys = (value, allowed) =>
Object.keys(value).every((key) => allowed.includes(key));
const callShape = (call) =>
object(call) &&
keys(call, ['id', 'type', 'function']) &&
typeof call.id === 'string' &&
call.id.length > 0 &&
call.id.length <= 1024 &&
call.type === 'function' &&
object(call.function) &&
keys(call.function, ['name', 'arguments']) &&
typeof call.function.name === 'string' &&
call.function.name.length <= 128 &&
typeof call.function.arguments === 'string';
function assistantShape(message) {
return (
object(message) &&
keys(message, ['role', 'content', 'tool_calls']) &&
message.role === 'assistant' &&
(message.content == null || typeof message.content === 'string') &&
(message.tool_calls === undefined ||
(Array.isArray(message.tool_calls) &&
message.tool_calls.length > 0 &&
message.tool_calls.length <= 32 &&
message.tool_calls.every(callShape) &&
new Set(message.tool_calls.map((call) => call.id)).size ===
message.tool_calls.length))
);
}
function validMessages(messages) {
if (!Array.isArray(messages) || !messages.length || messages.length > 96)
return false;
let pending = [];
for (const message of messages) {
if (!object(message)) return false;
if (pending.length) {
if (
!keys(message, ['role', 'content', 'tool_call_id']) ||
message.role !== 'tool' ||
typeof message.content !== 'string' ||
message.tool_call_id !== pending.shift()
)
return false;
} else if (message.role === 'user') {
if (
!keys(message, ['role', 'content']) ||
typeof message.content !== 'string'
)
return false;
} else if (assistantShape(message)) {
pending = (message.tool_calls || []).map((call) => call.id);
} else return false;
}
return !pending.length;
}
export function createOpenAiChatHandler({
annotationGuidance,
tools = GEV_REALTIME_TOOLS,
// osm_query is offered, and the prompt names it, only with Overpass, as
// for voice.
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
resolveApiKey = () => process.env.OPENAI_API_KEY,
resolveChatModel = () => process.env.OPENAI_CHAT_MODEL || 'gpt-4.1-mini',
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
environment = process.env,
...options
} = {}) {
return createTextChatHandler({
...options,
provider: 'openai',
resolveApiKey,
resolveProviderRateLimit,
providerPerMin: OPENAI_DEFAULT_PER_MIN,
// Chat can only bill the API key, which this operator lock forbids.
refuse() {
let error =
'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH, and typed OpenAI chat needs an API key.';
try {
if (!preferCodexOAuth(environment)) return null;
} catch (invalid) {
error = invalid.message;
}
return { error, code: 'CODEX_OAUTH_REQUIRED' };
},
prepare(input) {
const model = String(resolveChatModel()).trim();
if (
!/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,119}$/.test(model) ||
/realtime|audio|tts/i.test(model)
)
return { errorStatus: 503 };
if (
!object(input) ||
!keys(input, ['messages']) ||
!validMessages(input.messages)
)
return null;
const overpass = overpassConfigured();
return {
endpoint: 'https://api.openai.com/v1/chat/completions',
headers: (key) => ({ Authorization: `Bearer ${key}` }),
body: {
model,
n: 1,
stream: false,
store: false,
max_completion_tokens: 4096,
messages: [
{
role: 'system',
content: textInstructions(annotationGuidance, { overpass }),
},
...input.messages,
],
tools: offeredTools(tools, { overpass }).map(
({ name, description, parameters }) => ({
type: 'function',
function: { name, description, parameters, strict: false },
}),
),
},
};
},
parse(data) {
const choice = data?.choices?.[0];
const raw = choice?.message;
if (!raw) return null;
const content = {
role: raw.role,
content: raw.content ?? null,
...(raw.tool_calls?.length ? { tool_calls: raw.tool_calls } : {}),
};
if (
!assistantShape(content) ||
(!content.content && !content.tool_calls?.length)
)
return null;
const calls = (content.tool_calls || []).map((call) => {
try {
return {
name: call.function.name,
args: JSON.parse(call.function.arguments),
};
} catch {
return { name: call.function.name, parseError: true };
}
});
return {
content,
finishReason: choice.finish_reason,
invalidCalls: invalidCalls(
calls,
offeredTools(tools, { overpass: overpassConfigured() }),
),
};
},
});
}
server/providers/openai/hud-summary.js
+82 -13
View File
@@ -1,5 +1,12 @@
import { SNAPSHOT_FEED_STATES } from '../../../src/data/layerSnapshot.js';
import {
HUD_SUMMARY_INSTRUCTIONS,
HUD_SUMMARY_MAX_CHARS,
HUD_SUMMARY_MAX_UNSPACED_CHARS,
HUD_SUMMARY_RESPONSE_FORMAT,
hudSummaryInput,
hudSummaryInstructions,
hudSummaryIsUnspaced,
hudSummaryLocale,
keylessHudSummaryResponse,
} from '../../../src/hudSummaryResponse.js';
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
@@ -18,17 +25,74 @@ function extractOpenAiResponseText(data) {
.trim();
}
function toFiveWordHudSummary(value) {
return String(value || '')
.replace(/[^\p{L}\p{N}\s-]/gu, ' ')
.trim()
/** Parse the summary and normalize the canonical feed state. */
function parseHudSummaryReply(text, locale) {
let reply = null;
try {
reply = JSON.parse(text);
} catch {
/* not JSON: no summary */
}
const feedState = SNAPSHOT_FEED_STATES.includes(reply?.feedState)
? reply.feedState
: null;
return {
summary: toHudSummary(reply?.summary, locale),
feedState,
};
}
function toHudSummary(value, requested) {
const locale = hudSummaryLocale(requested);
const words = String(value || '')
.normalize('NFC')
// Marks, and apostrophes and dots inside a word, belong to it: ar vowel
// signs, fr d'Orléans, ja サンフランシスコ・ベイエリア.
.replace(/[^\p{L}\p{M}\p{N}\s'’·・-]/gu, ' ')
.split(/\s+/)
.filter(Boolean)
.slice(0, 5)
.join(' ');
.map((word) => word.replace(/^['’·・-]+|['’·・-]+$/gu, ''))
.filter(Boolean);
// Characters, not words: the model counts Rue de Rivoli as one word.
const max = hudSummaryIsUnspaced(locale)
? HUD_SUMMARY_MAX_UNSPACED_CHARS
: HUD_SUMMARY_MAX_CHARS;
return capAtWord(words.join(' '), max, locale);
}
/**
* Cut `text` to `max` characters at a word boundary; a first word longer
* than that is cut where the cap falls.
*/
function capAtWord(text, max, locale) {
const graphemes = [
...new Intl.Segmenter(locale, { granularity: 'grapheme' }).segment(text),
];
if (graphemes.length <= max) return text;
const cut = graphemes[max].index;
let end = 0;
for (const { index, segment } of new Intl.Segmenter(locale, {
granularity: 'word',
}).segment(text)) {
if (index + segment.length > cut) break;
end = index + segment.length;
}
return text.slice(0, end).trim() || text.slice(0, cut);
}
async function handleHudSummary(req, res) {
// Capability probe: lets the HUD skip its paid place lookups when there is
// no summary to feed them to.
if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
res.statusCode = 200;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'no-store');
res.end(
JSON.stringify({
configured: !keylessHudSummaryResponse(process.env.OPENAI_API_KEY),
}),
);
return;
}
if (req.method !== 'POST') {
res.statusCode = 405;
res.setHeader('Content-Type', 'application/json');
@@ -64,14 +128,18 @@ async function handleHudSummary(req, res) {
model:
process.env.OPENAI_HUD_SUMMARY_MODEL ||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
instructions: HUD_SUMMARY_INSTRUCTIONS,
input: JSON.stringify(context),
instructions: hudSummaryInstructions(context?.locale),
input: hudSummaryInput(context),
text: { format: HUD_SUMMARY_RESPONSE_FORMAT },
reasoning: { effort: 'minimal' },
max_output_tokens: 100,
max_output_tokens: 200,
}),
});
const data = await response.json().catch(() => ({}));
const summary = toFiveWordHudSummary(extractOpenAiResponseText(data));
const { summary, feedState } = parseHudSummaryReply(
extractOpenAiResponseText(data),
context?.locale,
);
res.statusCode = response.ok && summary ? 200 : response.status || 502;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'no-store');
@@ -80,6 +148,7 @@ async function handleHudSummary(req, res) {
res.end(
JSON.stringify({
summary: summary || null,
feedState,
// Never relay `data.error.message`: that is OpenAI's own wording, and
// it carries request ids, organization hints and quota phrasing.
error: response.ok ? null : 'OpenAI HUD summary request failed',
@@ -97,4 +166,4 @@ async function handleHudSummary(req, res) {
}
}
export { handleHudSummary };
export { handleHudSummary, parseHudSummaryReply };
server/providers/openai/instructions.js
+27 -14
View File
@@ -1,8 +1,14 @@
const DEFAULT_ANNOTATION_GUIDANCE =
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
function realtimeInstructions(
/**
* The Realtime session directives, one per line. `overpass` says an operator
* configured Overpass, which is what offers osm_query (see tools.js); without
* it the prompt never mentions the tool.
*/
function realtimeDirectives(
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
{ overpass = false } = {},
) {
return [
"You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
@@ -10,7 +16,7 @@ function realtimeInstructions(
'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
'Only control the app by calling the provided tools. Never invent tool names or arguments.',
'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
@@ -18,20 +24,23 @@ function realtimeInstructions(
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names, or an areaId you already have; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
'AREAS: resolve_area turns a named boundary (country, state, province, county, city, district, natural region) into an areaId; to outline it pass draw=true; annotate_map is for marks, not boundaries. Count inside it → analyst_query kind=area with the areaId (or kind=region); "the area I drew" → kind=drawn; "the area you outlined" → kind=annotation. Imagery of a place → resolve_area, then find_imagery. Ambiguous name: call resolve_area; on needsClarification ask which, never guess.' +
(overpass
? ' "All hospitals/schools/… in <place>" → resolve_area, then osm_query; in view → osm_query alone; follow-ups → analyst_query layers [osm-places], scope anywhere.'
: ''),
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
'Do not mention disabled layers or stale selections.',
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target, find_imagery area. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers, approximate outlines and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
//
@@ -47,25 +56,29 @@ function realtimeInstructions(
// string is the whole rollback.
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'HUD requests ("hud on/off", "switch to minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
'Confirmations echo the RESULTING state, never the request: "HUD minimal layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus or district, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park or district footprint; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
annotationGuidance,
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
].join('\n');
];
}
export { realtimeInstructions };
function realtimeInstructions(annotationGuidance, options) {
return realtimeDirectives(annotationGuidance, options).join('\n');
}
export { realtimeInstructions, realtimeDirectives };
server/providers/openai/realtime.js
+7 -3
View File
@@ -13,7 +13,8 @@ import {
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
} from './constants.js';
import { realtimeInstructions } from './instructions.js';
import { GEV_REALTIME_TOOLS } from './tools.js';
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
import { resolveOverpassUpstreams } from '../overpass/constants.js';
import {
preferCodexOAuth,
readCodexOAuthAccessToken,
@@ -136,6 +137,8 @@ function createRealtimeTokenHandler({
environment = process.env,
models = {},
tools = GEV_REALTIME_TOOLS,
// osm_query is offered only when an operator configured Overpass.
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
} = {}) {
return async (req, res) => {
res.setHeader('Cache-Control', 'no-store');
@@ -273,6 +276,7 @@ function createRealtimeTokenHandler({
transcribeModel.toLowerCase() === 'off'
? {}
: { transcription: { model: transcribeModel } };
const overpass = overpassConfigured();
const sessionConfig = {
session: {
type: 'realtime',
@@ -298,8 +302,8 @@ function createRealtimeTokenHandler({
},
output: { voice },
},
instructions: realtimeInstructions(annotationGuidance),
tools,
instructions: realtimeInstructions(annotationGuidance, { overpass }),
tools: offeredTools(tools, { overpass }),
tool_choice: 'auto',
},
};
server/providers/openai/toolDescriptions.js
+55 -10
View File
@@ -44,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
},
select_nearest_aircraft: {
description:
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
$position: 1,
parameters: {
properties: {
@@ -111,12 +111,18 @@ export const ACTION_DESCRIPTIONS = {
},
},
set_panel_open: {
description: 'Open or close a GEV UI panel/dropdown.',
description:
'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
$position: 1,
},
set_voice_inactivity_timeout: {
description:
'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
$position: 1,
},
set_context_mode: {
description:
'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
$position: 1,
parameters: {
properties: {
@@ -129,23 +135,23 @@ export const ACTION_DESCRIPTIONS = {
},
control_cockpit: {
description:
'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
'Explicit Cockpit requests only: enter from a selected/tracked aircraft (establishes Contacts itself; no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
$position: 1,
parameters: {
properties: {
action: {
description:
'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
'previous/next (or prev) cycles nearby Cockpit contacts.',
$position: 2,
},
targetLayer: {
description:
'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
'Optional next/previous layer filter (e.g. military-only).',
$position: 2,
},
aircraftClass: {
description:
'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
'Optional next/previous aircraft class filter (e.g. helicopter).',
$position: 1,
},
},
@@ -399,7 +405,7 @@ export const ACTION_DESCRIPTIONS = {
},
annotate_map: {
description:
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
$position: 1,
parameters: {
properties: {
@@ -541,9 +547,48 @@ export const ACTION_DESCRIPTIONS = {
},
},
},
resolve_area: {
description:
'Named place → reusable areaId for analyst_query scope {kind:area} and other area tools. draw=true outlines it ("outline Bagmati province"); around=true is "an area around <landmark>": its enclosing mapped area, else a buffer marked approximate. needsClarification: ask which, then call again with its candidateId. AREA_TIMEOUT: retry once.',
$position: 1,
parameters: {
properties: {
query: {
description: 'The place as said, with any kind word.',
$position: 1,
},
level: {
description: 'Only when the user names the kind.',
$position: 1,
},
within: {
description: 'Country or state the user added.',
$position: 1,
},
},
},
},
find_imagery: {
description:
'Recent satellite imagery (Sentinel-2/Landsat 30 m, VIIRS daily; last 30 days): pins the best acquisition in the Recent Imagery panel. areaId from resolve_area; area=pointer for "this spot"; omit for the view. BOX_REFUSED: offer to zoom in. DISPLACED: leave it.',
$position: 1,
parameters: {
properties: {
maxCloudPct: {
description: '"cloud-free" → 10.',
$position: 1,
},
},
},
},
osm_query: {
description:
'OpenStreetMap places of one kind ("hospitals in Kathmandu"). areaId from resolve_area; omit for the view. Shows them in OSM Places with numbered referents; they become the last answer. countScope bounding-box means an approximate count. OSM_TIMEOUT: retry once.',
$position: 1,
},
analyst_query: {
description:
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), NEEDS_CLARIFICATION (ask which), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
$position: 1,
parameters: {
properties: {
@@ -554,7 +599,7 @@ export const ACTION_DESCRIPTIONS = {
},
scope: {
description:
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"); area + areaId from resolve_area; drawn = "the area I drew"; annotation = "the area you outlined". Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates.',
$position: 2,
properties: {
name: {
server/providers/openai/tools.js
+25
View File
@@ -1,6 +1,7 @@
import { createActionTools } from '../../../src/voice/actionSchemas.js';
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
/** Every voice tool, as the Realtime session describes them. */
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
/**
@@ -14,3 +15,27 @@ export function realtimeSessionTools(additional = []) {
...additional.filter((tool) => !names.has(tool.name)),
];
}
/** Tools that need an operator-configured Overpass (OVERPASS_UPSTREAMS). */
export const OVERPASS_ONLY_TOOLS = Object.freeze(['osm_query']);
/**
* The tools one session is offered. Bulk OpenStreetMap place search runs
* only on an operator's Overpass, so without one `osm_query` is left out
* rather than offered and refused.
* @param {Array<{name: string}>} tools
* @param {{overpass?: boolean}} [options]
*/
export function offeredTools(tools, { overpass = false } = {}) {
return overpass
? tools
: tools.filter((tool) => !OVERPASS_ONLY_TOOLS.includes(tool.name));
}
/**
* The app actions one session is offered.
* @param {{overpass?: boolean}} [options]
*/
export function realtimeTools({ overpass = false } = {}) {
return offeredTools(GEV_REALTIME_TOOLS, { overpass });
}
server/providers/overpass.js
+60 -9
View File
@@ -1,6 +1,12 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import {
makeRateLimiter,
clientKey,
clientNetworkKey,
isLoopbackPeer,
} from './common/rate-limit.js';
import { hasProxySignals } from '../../src/localRequestGate.mjs';
import { readRequestBodyCapped } from './common/request.js';
import { admitSameSite } from './common/same-site.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import {
OVERPASS_MAX_BODY_BYTES,
OVERPASS_MAX_CONCURRENT,
@@ -22,6 +28,7 @@ import {
overpassNotConfigured,
} from './overpass/transport.js';
import { installRouteMiddleware } from './places/routes.js';
import { installOsmFeaturesRoute } from './overpass/osmFeaturesRoute.js';
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
const _overpassInFlight = new Map();
@@ -29,10 +36,21 @@ const _overpassInFlight = new Map();
let _overpassConcurrent = 0;
// Bound body-reading and validation work, including malformed requests and
// cache hits. Keep the tighter upstream budget below independent of this gate.
const _overpassRequestLimiter = makeRateLimiter({
windowMs: 60_000,
max: 300,
// cache hits, before a byte is read. Remote peers share one global window so
// address rotation cannot mint admission without bound (IPv6 peers are keyed
// by /64 for the same reason). A direct loopback request — the owner's own
// browser and the repo QA harnesses, with no forwarding headers — has its own
// per-peer window outside that global, so busy LAN peers cannot lock the owner
// out of the map. A request relayed by a reverse proxy on this machine also
// arrives from loopback, but it is remote traffic: it is admitted from the
// remote pool under its own socket-keyed bucket, never from the owner's. The
// forwarded client address is not trusted as identity. Only admission is
// isolated: the upstream budget below (90/min per key, 300/min globally) is
// shared, so an owner request that misses cache can still be refused there.
const OVERPASS_ADMISSION_WINDOW = { windowMs: 60_000, max: 300 };
const _overpassLoopbackAdmission = makeRateLimiter(OVERPASS_ADMISSION_WINDOW);
const _overpassPeerAdmission = makeRateLimiter({
...OVERPASS_ADMISSION_WINDOW,
globalMax: 900,
});
@@ -42,6 +60,33 @@ const _overpassRateLimiter = makeRateLimiter({
globalMax: 300,
});
/**
* Classify a request for admission. Direct loopback — a loopback socket with
* no forwarding header — is the owner's own pool; everything else, including a
* loopback socket that carries forwarding headers (a local reverse proxy), is
* remote. A proxied request keeps its socket identity under a `proxied:`
* prefix, so it never shares a bucket with the owner's direct requests.
* @param {import('http').IncomingMessage} req
* @returns {{direct: boolean, key: string}}
*/
function overpassAdmissionClass(req) {
const peer = clientNetworkKey(req);
if (!isLoopbackPeer(clientKey(req))) return { direct: false, key: peer };
if (hasProxySignals(req.headers || {}))
return { direct: false, key: `proxied:${peer}` };
return { direct: true, key: peer };
}
/**
* Spend one admission slot for this request; false when its bucket is over its
* window (or, for remote traffic, when the shared global window is full).
* @param {{direct: boolean, key: string}} admission - `overpassAdmissionClass(req)`.
* @returns {boolean}
*/
function admitOverpassRequest({ direct, key }) {
return (direct ? _overpassLoopbackAdmission : _overpassPeerAdmission)(key);
}
/**
* Write a completed Overpass payload to the HTTP response.
*
@@ -103,8 +148,13 @@ function overpassProxy({ routing = {} } = {}) {
return;
}
if (admitSameSite(req, res)) return;
if (!_overpassRequestLimiter(clientKey(req))) {
// Keyless gate: refuses cross-site browser requests but, unlike the
// cost-bearing routes, admits the forwarding headers a reverse proxy
// adds — the layers that depend on Overpass must keep working for an
// instance deployed the way the README documents.
if (admitSameSiteKeyless(req, res)) return;
const admission = overpassAdmissionClass(req);
if (!admitOverpassRequest(admission)) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
@@ -163,7 +213,7 @@ function overpassProxy({ routing = {} } = {}) {
// requests; boundary-class queries keep their month-long TTL.
readDisk: () =>
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
allowUpstream: () => _overpassRateLimiter(admission.key),
loadUpstream: async () => {
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
return {
@@ -225,6 +275,7 @@ function overpassProxy({ routing = {} } = {}) {
});
installRouteMiddleware(server.middlewares, routing);
installOsmFeaturesRoute(server.middlewares);
};
return {
name: 'overpass-proxy',
server/providers/overpass/osmFeatures.js
+205
View File
@@ -0,0 +1,205 @@
import { OSM_PRESETS } from '../../../src/data/osmPresets.js';
/**
* Bounded, typed OpenStreetMap feature search: one preset from the curated
* table inside a bounding box.
*
* The request carries a preset id, a box, a mode and a limit — never query
* text. The server writes the Overpass query itself and bounds it: box size
* per mode (a count may cover more ground than a fetch), result count,
* Overpass timeout and memory, response bytes, and one query in flight. It
* runs only against the operator's `OVERPASS_UPSTREAMS`; the generic
* `/api/overpass` guard is untouched.
*
* There is deliberately no boundary (`area`) filter: Overpass servers differ
* in whether they hold area data, and one without it answers zero rather than
* failing. The browser filters the box's features by the exact area geometry
* instead, so a count inside a province is exact and a box count is labelled
* as a box count.
*/
export const OSM_FEATURE_LIMITS = Object.freeze({
/** Largest box a fetch may cover, in square degrees (~200 × 200 km at the equator). */
maxFetchDeg2: 4,
/** Largest box a count may cover. */
maxCountDeg2: 36,
/** Most features one fetch returns. */
maxLimit: 1000,
/** Overpass server-side timeout, seconds: it gives up before we stop waiting. */
timeoutS: 20,
/** Overpass server-side memory cap, bytes. */
maxsize: 64 * 1024 * 1024,
/** Response bytes read from the upstream. */
maxResponseBytes: 16 * 1024 * 1024,
/** Per-upstream wait: the Overpass timeout plus transfer time. */
upstreamTimeoutMs: 22_000,
/**
* The whole search, across every configured endpoint in turn. It ends
* before the voice answer's own deadline (`OSM_QUERY_LIMITS.budgetMs` in
* `src/voice/osmActions.js`, 25 s): a browser that gives up first closes
* the request, which cancels the upstream query, so nothing would be
* cached for the retry.
*/
upstreamBudgetMs: 22_000,
});
/** Tags a returned feature keeps, besides the preset's own. */
const KEPT_TAGS = [
'name',
'name:en',
'operator',
'brand',
'opening_hours',
'wheelchair',
'emergency',
'healthcare',
'addr:street',
'addr:city',
];
const quote = (text) =>
`"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
/**
* Parse and bound a request. Returns the compiled query or a refusal.
* @param {{preset: string, bbox: string|number[], mode?: string, limit?: string|number}} request
* @returns {{ok: true, ql: string, mode: string, preset: object, limit: number, boxes: number[][]}|{ok: false, code: string, error: string}}
*/
export function compileOsmFeatureQuery(request = {}) {
const refuse = (code, error) => ({ ok: false, code, error });
const presetId = String(request.preset || '');
const preset = Object.hasOwn(OSM_PRESETS, presetId)
? OSM_PRESETS[presetId]
: null;
if (!preset) return refuse('UNKNOWN_PRESET', 'Unknown feature kind.');
const mode = request.mode === 'count' ? 'count' : 'features';
const parts = (
Array.isArray(request.bbox)
? request.bbox
: String(request.bbox || '').split(',')
).map(Number);
if (parts.length !== 4 || !parts.every(Number.isFinite))
return refuse('BAD_BOX', 'bbox needs west,south,east,north.');
const [west, south, east, north] = parts;
if (
Math.abs(west) > 180 ||
Math.abs(east) > 180 ||
south < -90 ||
north > 90 ||
south >= north ||
west === east
)
return refuse('BAD_BOX', 'bbox edges are out of range.');
// A box crossing the antimeridian is searched as its two halves.
const boxes =
west < east
? [[south, west, north, east]]
: [
[south, west, north, 180],
[south, -180, north, east],
];
const deg2 = boxes.reduce((sum, [s, w, n, e]) => sum + (n - s) * (e - w), 0);
const cap =
mode === 'count'
? OSM_FEATURE_LIMITS.maxCountDeg2
: OSM_FEATURE_LIMITS.maxFetchDeg2;
if (deg2 > cap)
return refuse(
'AREA_TOO_LARGE',
mode === 'count'
? 'That area is too large to search; narrow it down.'
: 'That area is too large to list; count first or narrow it down.',
);
// An absent or empty limit is the default, not Number(null) = 0.
const requested =
request.limit === null ||
request.limit === undefined ||
String(request.limit).trim() === ''
? NaN
: Number(request.limit);
const limit = Math.max(
1,
Math.min(
OSM_FEATURE_LIMITS.maxLimit,
Number.isFinite(requested) ? Math.trunc(requested) : 500,
),
);
const selectors = [];
for (const [key, value] of preset.tags)
for (const [s, w, n, e] of boxes)
selectors.push(
`nwr[${quote(key)}=${quote(value)}](${s},${w},${n},${e});`,
);
const ql =
`[out:json][timeout:${OSM_FEATURE_LIMITS.timeoutS}][maxsize:${OSM_FEATURE_LIMITS.maxsize}];` +
`(${selectors.join('')});` +
(mode === 'count' ? 'out count;' : `out center tags ${limit};`);
return { ok: true, ql, mode, preset, limit, boxes };
}
/**
* Overpass JSON → the compact answer the browser gets.
* @param {object} payload Parsed Overpass JSON.
* @param {{mode: string, preset: object, limit: number}} compiled
*/
export function summarizeOsmFeatures(payload, compiled) {
if (!Array.isArray(payload?.elements))
throw new TypeError('Malformed Overpass feature response');
const elements = payload.elements;
if (compiled.mode === 'count') {
const tags = elements.find((el) => el?.type === 'count')?.tags;
const rawTotal = tags?.total;
const total = Number(rawTotal);
if (
!tags ||
!Object.hasOwn(tags, 'total') ||
!['string', 'number'].includes(typeof rawTotal) ||
String(rawTotal).trim() === '' ||
!Number.isSafeInteger(total) ||
total < 0
)
throw new TypeError('Malformed Overpass count response');
return {
mode: 'count',
preset: compiled.preset.id,
count: total,
nodes: Number(tags.nodes) || 0,
ways: Number(tags.ways) || 0,
relations: Number(tags.relations) || 0,
};
}
const features = [];
let unplaced = 0;
for (const el of elements) {
if (
!['node', 'way', 'relation'].includes(el?.type) ||
(!Number.isFinite(Number(el?.id)) && typeof el?.id !== 'string')
) {
unplaced += 1;
continue;
}
const lat = Number.isFinite(el?.lat) ? el.lat : el?.center?.lat;
const lon = Number.isFinite(el?.lon) ? el.lon : el?.center?.lon;
if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
unplaced += 1;
continue;
}
const tags = {};
for (const key of [...KEPT_TAGS, ...compiled.preset.tags.map(([k]) => k)])
if (typeof el.tags?.[key] === 'string')
tags[key] = el.tags[key].slice(0, 120);
features.push({ id: `${el.type}/${el.id}`, lat, lon, tags });
}
// A genuinely empty `elements` array is authoritative. A non-empty answer
// whose every row is unusable is a schema failure, not "zero places"; do
// not let the route cache that as current state for a day.
if (elements.length && features.length === 0)
throw new TypeError('Overpass feature response has no usable rows');
return {
mode: 'features',
preset: compiled.preset.id,
count: features.length,
features,
truncated: elements.length >= compiled.limit,
...(unplaced ? { unplaced } : {}),
};
}
server/providers/overpass/osmFeaturesRoute.js
+252
View File
@@ -0,0 +1,252 @@
import path from 'node:path';
import { makeRateLimiter, clientNetworkKey } from '../common/rate-limit.js';
import { admitSameSiteKeyless } from '../common/same-site.js';
import { createBoundedCache } from '../common/boundedCache.js';
import {
compileOsmFeatureQuery,
OSM_FEATURE_LIMITS,
summarizeOsmFeatures,
} from './osmFeatures.js';
import {
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
} from './constants.js';
import { fetchOverpassPayload, overpassPayloadIsData } from './transport.js';
/** Places of interest change slowly: answers are kept a day. */
const OSM_FEATURES_TTL_MS = 86_400_000;
/** Upstream searches at once, across all clients (one per client). */
const OSM_MAX_RUNNING = 2;
/** Cache-key version; nothing cached under another version is reused. */
const CACHE_VERSION = 'v2';
/** Never simplify: these answers are points and counts, not drawings. */
const keepBody = (body) => body;
const jsonError = (status, error, extra = {}) => ({
status,
body: JSON.stringify({ error, ...extra }),
});
/**
* One upstream Overpass query at a time, with identical requests sharing it
* and answers in a bounded memory + disk cache.
*/
function createOverpassAnswerer({
fetchPayload,
cache,
maxResponseBytes,
upstreamTimeoutMs,
upstreamBudgetMs,
maxRunning = OSM_MAX_RUNNING,
}) {
const inFlight = new Map();
// One search per client, a few in all: one client can't hold the route.
const runningBy = new Map();
let running = 0;
return async function answer(key, ql, summarize, signal = null, owner = '') {
signal?.throwIfAborted?.();
const held = await cache.get(key);
signal?.throwIfAborted?.();
if (typeof held === 'string')
return { status: 200, body: held, cache: 'HIT' };
let operation = inFlight.get(key);
if (operation?.controller.signal.aborted) {
inFlight.delete(key);
operation = null;
}
if (!operation) {
if (runningBy.get(owner) || running >= maxRunning)
return {
...jsonError(
503,
runningBy.get(owner)
? 'Your previous map search is still running — try again shortly.'
: 'Another map search is running — try again shortly.',
),
retryAfter: '3',
};
running += 1;
runningBy.set(owner, (runningBy.get(owner) || 0) + 1);
const controller = new AbortController();
// The whole search, every configured endpoint in turn, gets one budget.
const budget = AbortSignal.timeout(upstreamBudgetMs);
operation = { controller, waiters: 0, settled: false, promise: null };
const owned = operation;
owned.promise = (async () => {
try {
const payload = await fetchPayload(
`data=${encodeURIComponent(ql)}`,
maxResponseBytes,
{
simplify: keepBody,
timeoutMs: upstreamTimeoutMs,
signal: AbortSignal.any([controller.signal, budget]),
},
);
if (!overpassPayloadIsData(payload))
return {
...jsonError(
payload?.rateLimited ? 429 : 503,
'OpenStreetMap search is unavailable right now.',
),
retryAfter: '10',
};
let parsed;
try {
parsed = JSON.parse(payload.body);
} catch {
return jsonError(
502,
'OpenStreetMap returned an unreadable answer.',
);
}
const body = JSON.stringify(summarize(parsed));
controller.signal.throwIfAborted();
await cache.set(key, body);
controller.signal.throwIfAborted();
return { status: 200, body, cache: 'MISS' };
} catch {
return budget.aborted && !controller.signal.aborted
? jsonError(504, 'OpenStreetMap took too long to answer.')
: jsonError(502, 'OpenStreetMap search failed.');
} finally {
owned.settled = true;
running -= 1;
const left = (runningBy.get(owner) || 1) - 1;
if (left > 0) runningBy.set(owner, left);
else runningBy.delete(owner);
if (inFlight.get(key) === owned) inFlight.delete(key);
}
})();
inFlight.set(key, owned);
}
operation.waiters += 1;
let abort = null;
const abandoned = signal
? new Promise((_, reject) => {
abort = () => {
operation.waiters -= 1;
if (operation.waiters === 0 && !operation.settled)
operation.controller.abort();
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
};
signal.addEventListener('abort', abort, { once: true });
if (signal.aborted) abort();
})
: null;
try {
return await (abandoned
? Promise.race([operation.promise, abandoned])
: operation.promise);
} finally {
if (signal && abort) signal.removeEventListener('abort', abort);
if (!signal?.aborted) operation.waiters -= 1;
}
};
}
function sender(res) {
return ({ status, body, cache, retryAfter }) => {
if (res.writableEnded) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(cache ? { 'X-OSM-Cache': cache } : {}),
...(retryAfter ? { 'Retry-After': retryAfter } : {}),
});
res.end(body);
};
}
/**
* Install `GET /api/osm/features?preset=&bbox=w,s,e,n[&mode=count][&limit=]`:
* one curated preset inside a box, answered by the operator's Overpass
* (`OVERPASS_UPSTREAMS`). With none configured the route answers
* `OVERPASS_NOT_CONFIGURED` and fetches nothing; public Nominatim is never a
* substitute for bulk place search.
*
* @param {{use: Function}} middlewares
* @param {{fetchPayload?: Function, featuresCache?: object, cacheRoot?: string|null, configured?: () => boolean, upstreamTag?: () => string}} [deps]
*/
export function installOsmFeaturesRoute(
middlewares,
{
fetchPayload = fetchOverpassPayload,
configured = () => resolveOverpassUpstreams().length > 0,
// Keys each answer to the instance that gave it, as the other Overpass
// caches do, so repointing OVERPASS_UPSTREAMS never serves the old one's.
upstreamTag = resolveOverpassUpstreamTag,
upstreamBudgetMs = OSM_FEATURE_LIMITS.upstreamBudgetMs,
cacheRoot = path.join(process.cwd(), '.gev-cache'),
featuresCache = createBoundedCache({
dir: cacheRoot ? path.join(cacheRoot, 'osm-features') : null,
ttlMs: OSM_FEATURES_TTL_MS,
maxMemoryEntries: 60,
maxDiskEntries: 400,
maxDiskBytes: 256 * 1024 * 1024,
}),
} = {},
) {
const limiter = makeRateLimiter({ windowMs: 60_000, max: 12, globalMax: 30 });
const features = createOverpassAnswerer({
fetchPayload,
cache: featuresCache,
maxResponseBytes: OSM_FEATURE_LIMITS.maxResponseBytes,
upstreamTimeoutMs: OSM_FEATURE_LIMITS.upstreamTimeoutMs,
upstreamBudgetMs,
});
middlewares.use('/api/osm/features', async (req, res) => {
const send = sender(res);
if (req.method !== 'GET') return send(jsonError(405, 'Method Not Allowed'));
// A page elsewhere must not spend the operator's Overpass. The keyless
// gate, as on /api/overpass: voice works behind a reverse proxy too.
if (admitSameSiteKeyless(req, res)) return undefined;
if (!configured())
return send(
jsonError(503, 'Place search needs a configured Overpass server.', {
code: 'OVERPASS_NOT_CONFIGURED',
retryable: false,
}),
);
const url = new URL(req.url || '', 'http://localhost');
const compiled = compileOsmFeatureQuery({
preset: url.searchParams.get('preset'),
bbox: url.searchParams.get('bbox'),
mode: url.searchParams.get('mode'),
limit: url.searchParams.get('limit'),
});
if (!compiled.ok)
return send(jsonError(400, compiled.error, { code: compiled.code }));
// IPv6 peers by /64, as /api/overpass does: rotating addresses inside
// one prefix earns neither a second rate bucket nor a second slot.
const client = clientNetworkKey(req);
if (!limiter(client))
return send({
...jsonError(429, 'Rate limit exceeded'),
retryAfter: '10',
});
const controller = new AbortController();
const abort = () => controller.abort();
req.once?.('aborted', abort);
res.once?.('close', abort);
try {
const answer = await features(
`${CACHE_VERSION}|features|${upstreamTag()}|${compiled.ql}`,
compiled.ql,
(parsed) => summarizeOsmFeatures(parsed, compiled),
controller.signal,
client,
);
if (!controller.signal.aborted) send(answer);
} catch {
if (!controller.signal.aborted)
send(jsonError(502, 'OpenStreetMap search failed.'));
} finally {
req.removeListener?.('aborted', abort);
res.removeListener?.('close', abort);
}
});
}
server/providers/overpass/query.js
+10 -6
View File
@@ -122,12 +122,14 @@ function sanitizeOverpassBody(rawBody) {
const boundedSets = new Set();
for (let stmt of withoutTags.split(';')) {
stmt = stmt.trim();
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
// The settings block (`[out:json][timeout:25]`) was blanked together with
// the tag filters, so it arrives here as an empty statement.
if (!stmt || /^out\b/.test(stmt)) continue;
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
// misread as a spatial bound. Bounds live in (...) / function calls / set
// refs, never inside [...], so the probe loses nothing real.
// Strip output-set assignments (NOT input bounds). With the tag filters
// already gone, a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot
// be misread as a spatial bound; bounds live in (...) / function calls /
// set refs, never inside [...], so nothing real was lost.
const outSets = [];
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
outSets.push(name);
@@ -166,8 +168,10 @@ function sanitizeOverpassBody(rawBody) {
if (bounded) for (const name of outSets) boundedSets.add(name);
}
// Tolerate whitespace around the directive so `[ timeout : N ]` is clamped
// rather than passed through as written.
const clamped = data.replace(
/\[timeout:\s*(\d+)\s*\]/gi,
/\[\s*timeout\s*:\s*(\d+)\s*\]/gi,
(_, n) =>
`[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
);
server/providers/overpass/requestAdmission.test.mjs
+255 -5
View File
@@ -3,10 +3,24 @@ import assert from 'node:assert/strict';
import { Readable } from 'node:stream';
import { spawnSync } from 'node:child_process';
import { overpassProxy } from '../overpass.js';
import { _overpassCache } from './cache.js';
import { sanitizeOverpassBody } from './query.js';
const form = (query) => `data=${encodeURIComponent(query)}`;
// The limiters are module-level, so every test that spends admission mocks a
// clock a full day past the previous one in execution order: no test can see
// another's window, whatever order (or randomization) they run in. A new
// test that reaches admission must call it too: earlier tests leave
// timestamps days ahead of the real clock, and those still count.
let clockBase = Date.now() + 86_400_000;
function freshClock(t) {
clockBase += 86_400_000;
const clock = { now: clockBase };
t.mock.method(Date, 'now', () => clock.now);
return clock;
}
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
for (const query of [
'node[',
@@ -37,6 +51,15 @@ test('bounded queries preserve quoted brackets, comments, multiple tags and time
query.replace('[timeout:999]', '[timeout:30]'),
);
}
// Whitespace inside the directive is clamped too, not passed through.
const spaced = sanitizeOverpassBody(
form('[out:json][ timeout : 999 ];node(around:20,1,1);out;'),
);
assert.equal(spaced.ok, true);
assert.equal(
new URLSearchParams(spaced.body).get('data'),
'[out:json][timeout:30];node(around:20,1,1);out;',
);
});
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
@@ -126,7 +149,13 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
{ 'sec-fetch-site': 'cross-site' },
{ 'sec-fetch-site': 'same-site' },
{ origin: 'null' },
{ 'x-forwarded-for': '127.0.0.1' },
// Forwarding headers do not launder a cross-site request.
{ 'x-forwarded-for': '10.0.0.9', 'sec-fetch-site': 'cross-site' },
{
'x-forwarded-proto': 'https',
origin: 'http://localhost:4173',
'sec-fetch-site': 'same-origin',
},
]) {
const response = await call(route, { headers, forbidRead: true });
assert.equal(response.statusCode, 403);
@@ -136,12 +165,24 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
});
}
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
test('same-origin, LAN, proxied and non-browser requests still reach validation; GET status stays available', async (t) => {
freshClock(t);
const route = handler();
for (const headers of [
{},
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
// The README's reverse-proxy deployment: Overpass is keyless, so the
// forwarding headers the proxy adds are admitted (as for terrain heights).
{
host: 'globe.example.lan',
origin: 'https://globe.example.lan',
'sec-fetch-site': 'same-origin',
'x-forwarded-for': '203.0.113.7',
'x-forwarded-proto': 'https',
'x-forwarded-host': 'globe.example.lan',
},
{ 'x-forwarded-for': '127.0.0.1' },
]) {
const response = await call(route, { headers });
assert.equal(response.statusCode, 400);
@@ -158,8 +199,7 @@ test('same-origin, LAN and non-browser requests still reach validation; GET stat
});
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
let now = Date.now() + 120000;
t.mock.method(Date, 'now', () => now);
const clock = freshClock(t);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
@@ -173,6 +213,216 @@ test('invalid requests spend admission quota before parsing, including on an unc
assert.equal(denied.reads, 0);
assert.equal(denied.headers['retry-after'], '60');
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
now += 60001;
clock.now += 60_001;
assert.equal((await call(route, { peer })).statusCode, 400);
});
test('the 900/min global window spans non-loopback peers only; loopback keeps its own window', async (t) => {
const clock = freshClock(t);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler();
for (const peer of ['10.0.0.1', '10.0.0.2', '10.0.0.3']) {
for (let i = 0; i < 300; i++) {
assert.equal((await call(route, { peer })).statusCode, 400, peer);
}
}
const starved = await call(route, { peer: '10.0.0.4', forbidRead: true });
assert.equal(starved.statusCode, 429, 'global window is full for LAN peers');
assert.equal(starved.reads, 0);
for (const peer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
const owner = await call(route, { peer });
assert.equal(owner.statusCode, 400, `${peer} is admitted past the global`);
assert.equal(owner.reads, 1);
}
clock.now += 60_001;
assert.equal((await call(route, { peer: '10.0.0.4' })).statusCode, 400);
});
test('IPv6 peers share a /64 admission bucket and IPv4-mapped peers fold to IPv4', async (t) => {
const clock = freshClock(t);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler();
for (let i = 0; i < 300; i++) {
assert.equal(
(await call(route, { peer: `2001:db8:1:2::${(i % 200) + 1}` }))
.statusCode,
400,
);
}
const rotated = await call(route, {
peer: '2001:db8:1:2:ffff:ffff:ffff:9',
forbidRead: true,
});
assert.equal(rotated.statusCode, 429, 'a rotating host stays in one bucket');
assert.equal(
(await call(route, { peer: '2001:db8:1:3::1' })).statusCode,
400,
);
for (let i = 0; i < 300; i++) {
assert.equal(
(await call(route, { peer: '::ffff:10.1.1.1' })).statusCode,
400,
);
}
const unmapped = await call(route, { peer: '10.1.1.1', forbidRead: true });
assert.equal(unmapped.statusCode, 429, 'mapped and bare IPv4 share a bucket');
clock.now += 60_001;
assert.equal((await call(route, { peer: '10.1.1.1' })).statusCode, 400);
});
test('cache hits spend admission budget but not the 90/min upstream budget', async (t) => {
const clock = freshClock(t);
const prior = process.env.OVERPASS_UPSTREAMS;
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
t.after(() => {
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
else process.env.OVERPASS_UPSTREAMS = prior;
_overpassCache.delete(hitKey);
});
let fetches = 0;
t.mock.method(globalThis, 'fetch', async () => {
fetches++;
// Not data: nothing is cached or written to disk by this test.
return new Response('upstream unavailable', { status: 503 });
});
const hit = 'node(around:20,1,1);out;';
const hitKey = `data=${encodeURIComponent(hit)}`;
_overpassCache.set(hitKey, {
status: 200,
body: '{"elements":[]}',
contentType: 'application/json',
endpoint: 'https://overpass.example/api/interpreter',
cachedAt: clock.now,
});
const route = handler();
const peer = '10.7.7.7';
// More cache hits than the upstream limiter would allow per peer.
for (let i = 0; i < 100; i++) {
const response = await call(route, { peer, body: form(hit) });
assert.equal(response.statusCode, 200);
assert.equal(response.headers['x-overpass-cache'], 'HIT');
}
assert.equal(fetches, 0);
const miss = await call(route, {
peer,
body: form('node(around:20,2,2);out;'),
});
assert.equal(fetches, 1, 'the first miss still reaches upstream');
assert.notEqual(miss.statusCode, 429);
for (let i = 0; i < 199; i++) {
assert.equal(
(await call(route, { peer, body: form(hit) })).statusCode,
200,
);
}
const exhausted = await call(route, {
peer,
body: form(hit),
forbidRead: true,
});
assert.equal(exhausted.statusCode, 429, 'hits spent the admission window');
assert.equal(exhausted.reads, 0);
});
test('a local reverse proxy cannot spend the direct-loopback reserve; owner cache hits survive it', async (t) => {
const clock = freshClock(t);
const prior = process.env.OVERPASS_UPSTREAMS;
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
const hit = 'node(around:20,3,3);out;';
const hitKey = `data=${encodeURIComponent(hit)}`;
t.after(() => {
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
else process.env.OVERPASS_UPSTREAMS = prior;
_overpassCache.delete(hitKey);
});
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
_overpassCache.set(hitKey, {
status: 200,
body: '{"elements":[]}',
contentType: 'application/json',
endpoint: 'https://overpass.example/api/interpreter',
cachedAt: clock.now,
});
const route = handler();
// The proxy connects over loopback; a rotating forwarded client address
// must not mint fresh buckets, so all 300 land in one socket-keyed bucket.
for (const proxyPeer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
const proxied = (i) => ({
peer: proxyPeer,
headers: { 'x-forwarded-for': `203.0.113.${i % 250}` },
body: form(hit),
});
for (let i = 0; i < 300; i++) {
assert.equal((await call(route, proxied(i))).statusCode, 200, proxyPeer);
}
const refused = await call(route, { ...proxied(300), forbidRead: true });
assert.equal(refused.statusCode, 429, `${proxyPeer} proxy bucket is full`);
assert.equal(refused.reads, 0);
for (const peer of ['127.0.0.1', '::1']) {
const owner = await call(route, { peer, body: form(hit) });
assert.equal(owner.statusCode, 200, `direct ${peer} is still admitted`);
assert.equal(owner.headers['x-overpass-cache'], 'HIT');
}
clock.now += 60_001;
assert.equal((await call(route, proxied(0))).statusCode, 200);
}
});
test('proxied loopback traffic draws on the shared remote global window', async (t) => {
const clock = freshClock(t);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler();
for (const peer of ['10.0.0.1', '10.0.0.2']) {
for (let i = 0; i < 300; i++) {
assert.equal((await call(route, { peer })).statusCode, 400, peer);
}
}
for (let i = 0; i < 300; i++) {
assert.equal(
(await call(route, { headers: { via: '1.1 proxy' } })).statusCode,
400,
);
}
const starved = await call(route, { peer: '10.0.0.3', forbidRead: true });
assert.equal(starved.statusCode, 429, 'proxied requests counted globally');
assert.equal((await call(route, {})).statusCode, 400, 'owner unaffected');
});
test("a local proxy has its own upstream bucket, so its misses cannot spend the owner's", async (t) => {
freshClock(t);
const prior = process.env.OVERPASS_UPSTREAMS;
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
t.after(() => {
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
else process.env.OVERPASS_UPSTREAMS = prior;
});
// Not data: nothing is cached, so every request below is a miss.
t.mock.method(
globalThis,
'fetch',
async () => new Response('upstream unavailable', { status: 503 }),
);
const route = handler();
const miss = form('node(around:20,4,4);out;');
// The upstream limiter's own refusal is the only miss without this header.
const passedLimiter = (response) =>
response.headers['x-overpass-cache'] !== undefined;
const proxied = { peer: '127.0.0.1', headers: { via: '1.1 proxy' } };
for (let i = 0; i < 90; i++) {
const response = await call(route, { ...proxied, body: miss });
assert.equal(passedLimiter(response), true, `proxied miss ${i}`);
}
const spent = await call(route, { ...proxied, body: miss });
assert.equal(spent.statusCode, 429);
assert.equal(passedLimiter(spent), false, 'the proxy bucket is spent');
const owner = await call(route, { peer: '127.0.0.1', body: miss });
assert.equal(passedLimiter(owner), true, 'the owner keeps its own bucket');
});
server/providers/overpass/transport.js
+13 -3
View File
@@ -124,6 +124,8 @@ async function fetchOverpassPayload(
readBody = readResponseTextCapped,
simplify = simplifyOverpassPayloadBody,
now = Date.now,
timeoutMs = OVERPASS_TIMEOUT_MS,
signal = null,
} = {},
) {
if (!endpoints.length) return overpassNotConfigured();
@@ -135,7 +137,10 @@ async function fetchOverpassPayload(
continue;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
const requestSignal = signal
? AbortSignal.any([signal, controller.signal])
: controller.signal;
try {
const requestUrl = new URL(endpoint);
const authorization =
@@ -156,7 +161,7 @@ async function fetchOverpassPayload(
...(authorization ? { Authorization: authorization } : {}),
},
body,
signal: controller.signal,
signal: requestSignal,
});
const responseBody = await readBody(upstream, maxResponseBytes);
const parsed = parseJson(responseBody);
@@ -207,7 +212,12 @@ async function fetchOverpassPayload(
upstreams: resolveOverpassUpstreamTag(),
rateLimited: false,
};
} catch {
} catch (error) {
// A route whose last requester disconnected owns this cancellation. It
// is not evidence that the configured upstream is unhealthy, so do not
// put the endpoint on cooldown or translate the abort into a response
// that a now-gone client could cache.
if (signal?.aborted) throw signal.reason ?? error;
const failures = (previous?.failures || 0) + 1;
const delay = retryDelay(null, failures - 1, now());
cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
server/providers/radio.js
+6 -2
View File
@@ -7,9 +7,13 @@ export {
publicRadioHttpsUrl,
} from './radio/stations.js';
export function radioBrowserProxy() {
const middleware = createRadioProxyMiddleware();
let middleware;
const initialize = () =>
(middleware ||= createRadioProxyMiddleware({
priorityCountries: process.env.RADIO_PRIORITY_COUNTRIES || '',
}));
const install = (server) => {
server.middlewares.use('/api/radio', middleware);
server.middlewares.use('/api/radio', initialize());
};
return {
name: 'radio-browser-proxy',
server/providers/radio/catalog.js
+106 -15
View File
@@ -1,3 +1,4 @@
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js';
import { lookup as lookupDns } from 'node:dns/promises';
import { randomUUID } from 'node:crypto';
@@ -27,6 +28,34 @@ import {
RADIO_FALLBACK_MIRRORS,
RADIO_REJECTED_STATION_COUNT_MAX,
} from './constants.js';
const PRIORITY_COUNTRY_LIMIT = 8;
const COUNTRY_QUERY_LIMIT = 200;
const COUNTRY_RESERVED_STATIONS = 40;
const REFRESH_RETRY_MS = 60_000;
const MAX_MIRROR_ATTEMPTS = 3;
/** Validate an operator-owned list; malformed configuration must not silently widen queries. */
export function parseRadioPriorityCountries(value = '') {
const entries = Array.isArray(value) ? value : String(value).split(',');
const countries = [];
for (const entry of entries) {
const input = String(entry).trim().toUpperCase();
if (!input) continue;
const normalized = normalizeRadioCountryInput(input);
if (!/^[A-Z]{2}$/.test(input) || !normalized.valid)
throw new TypeError(
'RADIO_PRIORITY_COUNTRIES requires ISO alpha-2 country codes',
);
const code = normalized.code;
if (!countries.includes(code)) countries.push(code);
}
if (countries.length > PRIORITY_COUNTRY_LIMIT)
throw new RangeError(
'RADIO_PRIORITY_COUNTRIES supports at most eight countries',
);
return countries;
}
async function mapRadioConcurrent(values, concurrency, mapper) {
const results = new Array(values.length);
let cursor = 0;
@@ -49,7 +78,9 @@ export function createRadioProxyMiddleware({
fetchImpl = null,
lookupImpl = lookupDns,
now = Date.now,
priorityCountries = [],
} = {}) {
const countries = parseRadioPriorityCountries(priorityCountries);
let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 };
let mirrorPromise = null;
let catalogCache = null;
@@ -61,6 +92,9 @@ export function createRadioProxyMiddleware({
const catalogInstance = randomUUID();
let servedStationIds = new Set();
let refreshPromise = null;
let retryAfter = 0;
let partialCatalog = null;
let refreshError = null;
async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) {
const destination = radioProxyDestination(url);
@@ -139,7 +173,7 @@ export function createRadioProxyMiddleware({
async function fetchPath(pathname) {
let lastError = null;
for (const origin of await mirrors()) {
for (const origin of (await mirrors()).slice(0, MAX_MIRROR_ATTEMPTS)) {
try {
return await fetchJson(`${origin}${pathname}`);
} catch (error) {
@@ -160,20 +194,27 @@ export function createRadioProxyMiddleware({
'aviation',
'marine',
'traffic',
];
].map((tag) => ({ tag }));
const baseQueryCount = queries.length;
queries.push(...countries.map((country) => ({ country })));
const outcomes = await mapRadioConcurrent(
queries,
3,
async (tag, index) => {
async ({ tag, country }, index) => {
const params = new URLSearchParams({
has_geo_info: 'true',
is_https: 'true',
hidebroken: 'true',
order: 'clickcount',
reverse: 'true',
limit: index === 0 ? '1800' : '220',
limit: country
? String(COUNTRY_QUERY_LIMIT)
: index === 0
? '1800'
: '220',
});
if (tag) params.set('tag', tag);
if (country) params.set('countrycode', country);
try {
const rows = await fetchPath(`/json/stations/search?${params}`);
if (!Array.isArray(rows))
@@ -195,7 +236,12 @@ export function createRadioProxyMiddleware({
const rejectedRows = rows.length - structuralRows.length;
const stations = structuralRows
.map(normalizeRadioBrowserStation)
.filter(Boolean);
.filter(
(station) =>
station && (!country || station.countryCode === country),
);
if (country && rows.length > COUNTRY_QUERY_LIMIT)
throw new Error('Radio Browser exceeded the country query limit');
const requestedTag = normalizeRadioTag(tag);
const requestedTagCovered =
!requestedTag ||
@@ -207,10 +253,12 @@ export function createRadioProxyMiddleware({
),
);
return {
// Query coverage is based on accepted rows, not merely a payload that
// happens to match the upstream schema. Specialist responses must
// also contain an accepted station tagged for the requested category.
succeeded: stations.length > 0 && requestedTagCovered,
// A valid country response can contain no playable stations. Country
// coverage is advisory; only broad/specialist queries gate health.
// Specialist responses need an accepted station with matching tags.
succeeded: country
? rows.length === 0 || structuralRows.length > 0
: stations.length > 0 && requestedTagCovered,
stations,
rejectedRows,
};
@@ -233,9 +281,24 @@ export function createRadioProxyMiddleware({
seen.add(station.id);
selected.push(station);
};
// Reserve country slots first; low-click regional stations cannot lose all
// representation to the global charts. Keep room for all eight specialist
// seeds (8 * 45) and global fill even at the eight-country limit (8 * 40).
for (const rows of resultSets.slice(baseQueryCount)) {
const ranked = [...rows].sort(
(a, b) => b.clickCount - a.clickCount || a.id.localeCompare(b.id),
);
let added = 0;
for (const station of ranked) {
if (seen.has(station.id)) continue;
take(station);
if (++added >= COUNTRY_RESERVED_STATIONS) break;
}
}
// Seed specialist station-tag queries before popularity fill so operational
// categories remain represented even when global click charts skew musical.
for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take);
for (const rows of resultSets.slice(1, baseQueryCount))
rows.slice(0, 45).forEach(take);
resultSets
.flat()
.sort(
@@ -243,9 +306,9 @@ export function createRadioProxyMiddleware({
)
.forEach(take);
const timestamp = now();
const successfulQueries = outcomes.filter(
(outcome) => outcome.succeeded,
).length;
const successfulQueries = outcomes
.slice(0, baseQueryCount)
.filter((outcome) => outcome.succeeded).length;
const broadQueryHealthy =
outcomes[0].succeeded && outcomes[0].stations.length > 0;
const healthReasons = [];
@@ -261,7 +324,18 @@ export function createRadioProxyMiddleware({
);
const coverage = {
successfulQueries,
totalQueries: queries.length,
totalQueries: baseQueryCount,
...(countries.length
? {
priorityCountries: countries.map((country, index) => ({
countryCode: country,
succeeded: outcomes[baseQueryCount + index].succeeded,
stationCount: selected.filter(
(station) => station.countryCode === country,
).length,
})),
}
: {}),
stationCount: selected.length,
healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS,
};
@@ -315,7 +389,24 @@ export function createRadioProxyMiddleware({
return { ...catalogCache, stale: false };
}
if (!refreshPromise) {
refreshPromise = refreshCatalog().finally(() => {
refreshPromise = (async () => {
if (now() < retryAfter) {
if (refreshError) throw refreshError;
return partialCatalog;
}
try {
const catalog = await refreshCatalog();
refreshError = null;
partialCatalog = catalog.degraded ? catalog : null;
retryAfter = catalog.degraded ? now() + REFRESH_RETRY_MS : 0;
return catalog;
} catch (error) {
refreshError = error;
partialCatalog = null;
retryAfter = now() + REFRESH_RETRY_MS;
throw error;
}
})().finally(() => {
refreshPromise = null;
});
}
server/providers/regional/nominatimCache.js
+174
View File
@@ -0,0 +1,174 @@
import crypto from 'node:crypto';
import fs from 'node:fs/promises';
import path from 'node:path';
import { normalizePlaceName as foldName } from '../../../src/data/placeNames.js';
const DAY_MS = 86_400_000;
/** Entry files this cache writes: a 40-hex-digit key hash. */
const CACHE_FILE_RE = /^[0-9a-f]{40}\.json$/;
/** How long an answer is kept, by what it was. */
export const NOMINATIM_CACHE_TTL_MS = Object.freeze({
outline: 90 * DAY_MS,
place: 30 * DAY_MS,
notFound: DAY_MS,
});
/**
* Comparable form of a place name for matching results to the ask: the
* shared place-name key (see `normalizePlaceName` in placeNames.js, so
* "St. Paul" and "Saint Paul" agree), bounded in length.
*
* @param {string} value
* @returns {string}
*/
export function normalizePlaceName(value) {
return foldName(value).slice(0, 200);
}
/**
* Memory + disk cache for Nominatim answers.
*
* Every entry carries its own expiry. Memory is an LRU bounded by entries;
* disk is bounded by entries and bytes, with expired files removed during a
* periodic sweep. A null `dir` keeps everything in memory.
*/
export function createNominatimCache({
dir = null,
maxMemoryEntries = 256,
maxDiskEntries = 4000,
maxDiskBytes = 64 * 1024 * 1024,
maxEntryBytes = 2 * 1024 * 1024,
sweepEvery = 50,
now = Date.now,
} = {}) {
const memory = new Map();
let writesSinceSweep = 0;
let sweeping = null;
const digestFor = (key) =>
crypto.createHash('sha256').update(key).digest('hex');
const fileFor = (key) =>
path.join(dir, `${digestFor(key).slice(0, 40)}.json`);
function remember(key, entry) {
memory.delete(key);
memory.set(key, entry);
while (memory.size > maxMemoryEntries)
memory.delete(memory.keys().next().value);
}
async function get(key) {
const hit = memory.get(key);
if (hit) {
if (hit.expiresAt > now()) {
remember(key, hit);
return hit;
}
memory.delete(key);
}
if (!dir) return null;
try {
const entry = JSON.parse(await fs.readFile(fileFor(key), 'utf8'));
if (entry?.keyDigest !== digestFor(key) || !(entry.expiresAt > now()))
return null;
remember(key, entry);
return entry;
} catch {
return null;
}
}
async function set(key, payload, ttlMs) {
const entry = {
keyDigest: digestFor(key),
payload,
storedAt: now(),
expiresAt: now() + ttlMs,
};
remember(key, entry);
if (!dir) return;
const body = JSON.stringify(entry);
if (Buffer.byteLength(body) > maxEntryBytes) return;
let temp = null;
try {
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
await fs.chmod(dir, 0o700);
const file = fileFor(key);
temp = `${file}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`;
await fs.writeFile(temp, body, { mode: 0o600, flag: 'wx' });
await fs.rename(temp, file);
temp = null;
} catch {
return;
} finally {
if (temp) await fs.rm(temp, { force: true }).catch(() => {});
}
writesSinceSweep += 1;
if (writesSinceSweep >= sweepEvery) {
writesSinceSweep = 0;
await sweep();
}
}
/** Remove expired files, then the oldest until both disk bounds hold. */
async function sweep() {
if (!dir) return;
if (sweeping) return sweeping;
sweeping = (async () => {
let names;
try {
names = await fs.readdir(dir);
} catch {
return;
}
const files = [];
for (const name of names) {
// Only this cache's own entry files; anything else in the directory
// is left alone.
if (!CACHE_FILE_RE.test(name)) continue;
const file = path.join(dir, name);
let stat;
let text;
try {
[stat, text] = await Promise.all([
fs.stat(file),
fs.readFile(file, 'utf8'),
]);
} catch {
// Unreadable right now (gone, busy, too many open files): an error
// reading is not evidence the entry is bad, so it stays.
continue;
}
let expiresAt;
try {
expiresAt = JSON.parse(text)?.expiresAt;
} catch {
expiresAt = null; // corrupt: removed below
}
if (!(expiresAt > now())) {
await fs.rm(file, { force: true }).catch(() => {});
continue;
}
files.push({ file, size: stat.size, at: stat.mtimeMs });
}
files.sort((a, b) => a.at - b.at);
let bytes = files.reduce((sum, entry) => sum + entry.size, 0);
while (
files.length &&
(files.length > maxDiskEntries || bytes > maxDiskBytes)
) {
const oldest = files.shift();
bytes -= oldest.size;
await fs.rm(oldest.file, { force: true }).catch(() => {});
}
})().finally(() => {
sweeping = null;
});
return sweeping;
}
return { get, set, sweep };
}
server/providers/regional/nominatimGate.js
+955
View File
@@ -0,0 +1,955 @@
import fs from 'node:fs';
import path from 'node:path';
import { readResponseJsonCapped } from '../common/http.js';
/**
* One gate for every request this server sends to a Nominatim instance.
*
* Place search and voice outlines share it. The service is off until an operator
* configures an endpoint. Explicit public requests coordinate fetch invocation
* across processes sharing this installation's state file, at least 1.1 s apart.
* Independent installations do not share an app-wide limiter; these local guards
* do not establish compliance for an application's combined public traffic.
*/
/** The public instance. Only this host is subject to the public-use limits. */
export const PUBLIC_NOMINATIM_SEARCH =
'https://nominatim.openstreetmap.org/search';
const PUBLIC_NOMINATIM_HOST = 'nominatim.openstreetmap.org';
/**
* The public usage policy asks for a User-Agent or Referer that identifies the
* application. The identity is stable: it is never rotated or varied.
*/
const NOMINATIM_HEADERS = Object.freeze({
'User-Agent':
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
});
/** Absolute policy maximum is one request per second; 1.1 s absorbs jitter. */
const NOMINATIM_MIN_SPACING_MS = 1100;
/** Requests allowed to wait for their turn; past this, refuse at once. */
export const NOMINATIM_MAX_PENDING = 4;
/** A queued request older than this has no reader left; drop it unsent. */
const NOMINATIM_MAX_WAIT_MS = 10_000;
/** Default public requests per install per UTC day, outlines and search together. */
const NOMINATIM_DEFAULT_DAILY_CAP = 50;
/**
* Pause after a refusal without Retry-After. A Retry-After from the server is
* honoured in full, however long: no local maximum may cause an earlier retry.
*/
const DEFAULT_BACKOFF_MS = 60_000;
/** A 403 from the public instance means this identity is blocked: stop longer. */
const BLOCKED_BACKOFF_MS = 3_600_000;
/** Pause after a server error or a failed retry. */
const OUTAGE_BACKOFF_MS = 30_000;
/** How old an unfinished dispatch must be before the next lock owner reclaims it. */
const DISPATCH_RECLAIM_MS = 10 * 60_000;
const DISABLED_VALUES = new Set(['', 'off', 'none', 'disabled', '0']);
/**
* node:sqlite, loaded on first public-instance use. Node 24 still flags it
* experimental and prints that once per process; only that one warning is
* dropped, and only while the module loads.
*/
let sqlite = null;
function sqliteModule() {
if (sqlite) return sqlite;
const emitWarning = process.emitWarning;
process.emitWarning = function (warning, ...rest) {
const type = typeof rest[0] === 'string' ? rest[0] : rest[0]?.type;
if (type === 'ExperimentalWarning' && /SQLite/.test(String(warning)))
return;
return emitWarning.call(this, warning, ...rest);
};
try {
sqlite = process.getBuiltinModule('node:sqlite');
} finally {
process.emitWarning = emitWarning;
}
return sqlite;
}
/**
* Read the search endpoint setting.
*
* Unset, empty or `off` disables every Nominatim request. An explicit endpoint
* must be an http(s) URL
* without credentials, query or fragment; a bare host gets `/search`. A value
* that is not usable disables the service rather than falling back to the
* public instance.
*
* @param {Record<string, string|undefined>} [env]
* @returns {{endpoint: string|null, isPublic: boolean, dailyCap: number, invalid?: boolean}}
*/
export function resolveNominatimSettings(env = process.env) {
const dailyCap = parseDailyCap(env.NOMINATIM_DAILY_CAP);
const raw = env.NOMINATIM_URL;
if (raw === undefined) return { endpoint: null, isPublic: false, dailyCap };
const value = String(raw).trim();
if (DISABLED_VALUES.has(value.toLowerCase()))
return { endpoint: null, isPublic: false, dailyCap };
let url;
try {
url = new URL(value);
} catch {
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
}
if (
!['http:', 'https:'].includes(url.protocol) ||
!url.hostname ||
url.username ||
url.password ||
url.search ||
url.hash
)
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
const publicHost =
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST;
if (publicHost && url.protocol !== 'https:')
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
if (url.pathname === '/' || url.pathname === '') url.pathname = '/search';
return {
endpoint: url.href.replace(/\/$/, ''),
// A trailing DNS dot names the same host.
isPublic: publicHost,
dailyCap,
};
}
function parseDailyCap(raw) {
if (raw === undefined || String(raw).trim() === '')
return NOMINATIM_DEFAULT_DAILY_CAP;
const value = Number(raw);
return Number.isFinite(value) && value >= 0
? Math.floor(value)
: NOMINATIM_DEFAULT_DAILY_CAP;
}
function gateError(code, message, extra = {}) {
return Object.assign(new Error(message), { code, ...extra });
}
const HTTP_MONTHS = Object.freeze([
'Jan',
'Feb',
'Mar',
'Apr',
'May',
'Jun',
'Jul',
'Aug',
'Sep',
'Oct',
'Nov',
'Dec',
]);
const HTTP_WEEKDAYS = Object.freeze([
'Sun',
'Mon',
'Tue',
'Wed',
'Thu',
'Fri',
'Sat',
]);
const HTTP_WEEKDAYS_LONG = Object.freeze([
'Sunday',
'Monday',
'Tuesday',
'Wednesday',
'Thursday',
'Friday',
'Saturday',
]);
/** Strict RFC 9110 HTTP-date, including the two obsolete recipient formats. */
function parseHttpDate(raw, now) {
let match = raw.match(
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), (\d{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{4}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
);
let weekday, day, month, year, hour, minute, second;
if (match) {
[, weekday, day, month, year, hour, minute, second] = match;
weekday = HTTP_WEEKDAYS.indexOf(weekday);
} else {
match = raw.match(
/^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), (\d{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{2}) (\d{2}):(\d{2}):(\d{2}) GMT$/,
);
if (match) {
[, weekday, day, month, year, hour, minute, second] = match;
weekday = HTTP_WEEKDAYS_LONG.indexOf(weekday);
const currentYear = new Date(now).getUTCFullYear();
if (!Number.isFinite(currentYear)) return null;
year = Math.floor(currentYear / 100) * 100 + Number(year);
if (year > currentYear + 50) year -= 100;
} else {
match = raw.match(
/^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([ \d]\d) (\d{2}):(\d{2}):(\d{2}) (\d{4})$/,
);
if (!match) return null;
[, weekday, month, day, hour, minute, second, year] = match;
weekday = HTTP_WEEKDAYS.indexOf(weekday);
}
}
day = Number(day);
month = HTTP_MONTHS.indexOf(month);
year = Number(year);
hour = Number(hour);
minute = Number(minute);
second = Number(second);
if (
weekday < 0 ||
month < 0 ||
year < 1601 ||
hour > 23 ||
minute > 59 ||
second > 59
)
return null;
const at = Date.UTC(year, month, day, hour, minute, second);
const date = new Date(at);
return date.getUTCFullYear() === year &&
date.getUTCMonth() === month &&
date.getUTCDate() === day &&
date.getUTCHours() === hour &&
date.getUTCMinutes() === minute &&
date.getUTCSeconds() === second &&
date.getUTCDay() === weekday
? at
: null;
}
/** Seconds or an HTTP date → milliseconds from now, or null. */
export function parseRetryAfter(value, now = Date.now()) {
if (value == null) return null;
const raw = String(value).trim();
if (!raw) return null;
if (/^\d+$/.test(raw)) {
const seconds = Number(raw);
const milliseconds = seconds * 1000;
return Number.isSafeInteger(milliseconds) ? milliseconds : null;
}
// Numeric-looking but invalid delay-seconds must not be reinterpreted as a date.
if (/^[+-]?(?:\d|\.\d)/.test(raw)) return null;
const at = parseHttpDate(raw, now);
return at === null ? null : Math.max(0, at - now);
}
/** UTC calendar day, the unit of the daily cap. */
function utcDay(ms) {
return new Date(ms).toISOString().slice(0, 10);
}
/**
* Public-service state shared by every process using the same file: today's
* request count, the next reserved start, and any pause the server asked for.
*
* Every change is a read-modify-write under an OS-owned SQLite transaction,
* and the JSON file is replaced atomically, so two dev servers sharing a
* checkout add to one count instead of overwriting each other. A process exit
* releases only that process's transaction; there is no stale path to delete
* and therefore no stale reclaimer that can remove a successor's live lock.
* Memory keeps the highest values this process has seen, so a failed or
* unreadable file never lowers the count or shortens a pause; a persistence
* failure is reported once and in `status()`.
*/
export function createGateStateStore({
file = null,
lockWaitMs = 500,
onError = (error) =>
console.warn(
`[Nominatim] could not persist usage state: ${error?.message || error}`,
),
} = {}) {
let memory = { day: '', count: 0, pausedUntil: 0, nextStartAt: 0 };
let lastError = null;
const report = (error) => {
if (!lastError) onError(error);
lastError = error;
};
const pauseMarkerPrefix = file ? `${path.basename(file)}.pause.` : null;
const dispatchMarkerPrefix = file ? `${path.basename(file)}.dispatch.` : null;
// A crash or persistence failure during dispatch must never reopen admission
// using an older timestamp. A dispatch removes its own marker after
// persisting the post-invocation deadline. A marker is only ever seen under
// the lock after its owner left the lock, so one that names a start older
// than DISPATCH_RECLAIM_MS (far beyond any request's timeout) is reclaimed
// by the next lock owner, which restarts spacing from its own clock.
function pendingDispatches({ strict = false } = {}) {
if (!file) return [];
try {
return fs
.readdirSync(path.dirname(file))
.filter((name) => name.startsWith(dispatchMarkerPrefix));
} catch (error) {
if (error?.code === 'ENOENT') return [];
report(error);
if (strict) throw error;
return [];
}
}
/**
* Under the lock: remove dispatch markers whose start is older than
* DISPATCH_RECLAIM_MS and report whether any were. Any other marker (newer,
* or naming no start) still refuses admission.
*/
function reclaimDispatches(at) {
const names = pendingDispatches({ strict: true });
for (const name of names) {
const since = /^since-(\d+)$/.exec(
name.slice(dispatchMarkerPrefix.length).split('.', 1)[0],
);
const started = since ? Number(since[1]) : NaN;
if (!Number.isSafeInteger(started) || at - started < DISPATCH_RECLAIM_MS)
throw new Error('unfinished durable Nominatim dispatch');
}
for (const name of names)
fs.unlinkSync(path.join(path.dirname(file), name));
return names.length > 0;
}
/**
* A refusal must remain visible even while another process owns the SQLite
* transaction. Immutable marker names make that write atomic and monotonic:
* concurrent writers cannot shorten one another's pause, and the next lock
* owner folds every marker into the normal JSON state.
*/
function pendingPauses({ strict = false } = {}) {
if (!file) return [];
let names;
try {
names = fs.readdirSync(path.dirname(file));
} catch (error) {
if (error?.code === 'ENOENT') return [];
report(error);
if (strict) throw error;
return [];
}
const pauses = [];
for (const name of names) {
if (!name.startsWith(pauseMarkerPrefix)) continue;
const encoded = name.slice(pauseMarkerPrefix.length).split('.', 1)[0];
if (!/^\d+$/.test(encoded)) continue;
const pausedUntil = Number(encoded);
if (!Number.isSafeInteger(pausedUntil)) {
const error = new Error('invalid durable Nominatim pause marker');
report(error);
if (strict) throw error;
continue;
}
pauses.push({
path: path.join(path.dirname(file), name),
pausedUntil,
});
}
return pauses;
}
function markPause(at) {
fs.mkdirSync(path.dirname(file), { recursive: true });
const pausedUntil = Math.ceil(at);
const marker = `${file}.pause.${pausedUntil}.${process.pid}.${process.hrtime.bigint()}`;
fs.writeFileSync(marker, '', { flag: 'wx' });
return pausedUntil;
}
function clearPersistedPauses(pausedUntil) {
for (const marker of pendingPauses()) {
if (marker.pausedUntil > pausedUntil) continue;
try {
fs.unlinkSync(marker.path);
} catch (error) {
if (error?.code !== 'ENOENT') report(error);
}
}
}
const merge = (a, b) => ({
day: a.day >= b.day ? a.day : b.day,
count:
a.day === b.day
? Math.max(a.count, b.count)
: a.day > b.day
? a.count
: b.count,
pausedUntil: Math.max(a.pausedUntil || 0, b.pausedUntil || 0),
nextStartAt: Math.max(a.nextStartAt || 0, b.nextStartAt || 0),
});
const parseState = (parsed) => {
const validDay =
parsed?.day === '' ||
(typeof parsed?.day === 'string' &&
/^\d{4}-\d{2}-\d{2}$/.test(parsed.day) &&
new Date(`${parsed.day}T00:00:00.000Z`)
.toISOString()
.startsWith(parsed.day));
const validCount =
Number.isInteger(parsed?.count) &&
parsed.count >= 0 &&
(parsed.day !== '' || parsed.count === 0);
const validPausedUntil =
Number.isFinite(parsed?.pausedUntil) && parsed.pausedUntil >= 0;
// nextStartAt was added after the original state format. Its absence is
// the one supported migration; a present value must still be trustworthy.
const validNextStartAt =
parsed?.nextStartAt === undefined ||
(Number.isFinite(parsed.nextStartAt) && parsed.nextStartAt >= 0);
if (
!parsed ||
Array.isArray(parsed) ||
typeof parsed !== 'object' ||
!validDay ||
!validCount ||
!validPausedUntil ||
!validNextStartAt
)
throw new Error('invalid durable Nominatim usage state');
return {
day: parsed.day,
count: parsed.count,
pausedUntil: parsed.pausedUntil,
nextStartAt: parsed.nextStartAt ?? 0,
};
};
function readDisk({ strict = false } = {}) {
if (!file) {
if (strict)
throw new Error('no durable Nominatim usage state is configured');
return null;
}
try {
return parseState(JSON.parse(fs.readFileSync(file, 'utf8')));
} catch (error) {
if (error?.code === 'ENOENT') return null;
report(error);
if (strict) throw error;
return null;
}
}
function current() {
const disk = readDisk();
if (disk) memory = merge(memory, disk);
const pending = pendingPauses();
if (pending.length)
memory = merge(memory, {
...memory,
pausedUntil: Math.max(...pending.map((item) => item.pausedUntil)),
});
return memory;
}
function withLock(work) {
fs.mkdirSync(path.dirname(file), { recursive: true });
const waitMs = Number.isFinite(lockWaitMs)
? Math.max(0, Math.floor(lockWaitMs))
: 500;
const database = new (sqliteModule().DatabaseSync)(`${file}.lock.sqlite`, {
timeout: waitMs,
});
let transactionOpen = false;
try {
database.exec('BEGIN IMMEDIATE');
transactionOpen = true;
const result = work();
database.exec('COMMIT');
transactionOpen = false;
return result;
} catch (error) {
if (transactionOpen) {
try {
database.exec('ROLLBACK');
} catch {
// Closing the connection below still releases this owner's lock.
}
}
throw error;
} finally {
try {
database.close();
} catch {
// The transaction has already been committed or rolled back. Closing
// is best effort during exceptional teardown and never removes a lock.
}
}
}
function writeDisk(next) {
const temp = `${file}.${process.pid}.${process.hrtime.bigint()}.tmp`;
try {
fs.writeFileSync(temp, JSON.stringify(next));
fs.renameSync(temp, file);
} finally {
try {
fs.unlinkSync(temp);
} catch (error) {
if (error?.code !== 'ENOENT') report(error);
}
}
}
/** Apply `change` to the freshest state and persist it atomically. */
function update(change) {
if (!file) {
memory = merge(memory, change(memory));
return memory;
}
try {
withLock(() => {
const next = merge(memory, change(current()));
writeDisk(next);
memory = next;
clearPersistedPauses(next.pausedUntil);
lastError = null;
});
} catch (error) {
report(error);
memory = merge(memory, change(memory));
}
return memory;
}
return {
count(day) {
const state = current();
return state.day === day ? state.count : 0;
},
increment(day) {
return update((state) => ({
day,
count: state.day === day ? state.count + 1 : 1,
pausedUntil: state.pausedUntil,
nextStartAt: state.nextStartAt,
})).count;
},
dispatch(
day,
{ clock, dailyCap, minSpacingMs, latestStartAt = Infinity },
start,
) {
if (!file) {
const error = new Error(
'no durable Nominatim usage state is configured',
);
report(error);
return { status: 'unavailable', error };
}
let value;
try {
let result;
withLock(() => {
const reclaimed = reclaimDispatches(clock());
const disk = readDisk({ strict: true });
if (disk) memory = merge(memory, disk);
const pauses = pendingPauses({ strict: true });
const state = merge(memory, {
...memory,
pausedUntil: Math.max(
memory.pausedUntil,
...pauses.map((p) => p.pausedUntil),
),
});
const at = clock();
if (reclaimed) {
state.nextStartAt = Math.max(state.nextStartAt, at + minSpacingMs);
writeDisk(state);
memory = state;
}
if (at > latestStartAt) {
result = { status: 'abandoned' };
return;
}
if (state.pausedUntil > at) {
result = { status: 'paused', retryAfterMs: state.pausedUntil - at };
return;
}
const count = state.day === day ? state.count : 0;
if (count >= dailyCap) {
result = { status: 'cap' };
return;
}
if (state.nextStartAt > at) {
result =
state.nextStartAt > latestStartAt
? { status: 'abandoned' }
: { status: 'wait', waitMs: state.nextStartAt - at };
return;
}
const marker = `${file}.dispatch.since-${at}.${process.pid}.${process.hrtime.bigint()}`;
fs.writeFileSync(marker, '', { flag: 'wx' });
const next = {
day,
count: count + 1,
pausedUntil: state.pausedUntil,
nextStartAt: at + minSpacingMs,
};
writeDisk(next);
memory = next;
const latePause = Math.max(
next.pausedUntil,
...pendingPauses({ strict: true }).map((p) => p.pausedUntil),
);
if (clock() > latestStartAt) result = { status: 'abandoned' };
else if (latePause > clock())
result = { status: 'paused', retryAfterMs: latePause - clock() };
else {
// Invoke, but never await, the transport while still owning the lock.
try {
value = start();
} catch (error) {
value = Promise.reject(error);
}
result = { status: 'started', value };
}
const after = {
...next,
pausedUntil: latePause,
nextStartAt: Math.max(next.nextStartAt, clock() + minSpacingMs),
};
writeDisk(after);
memory = after;
fs.unlinkSync(marker);
clearPersistedPauses(after.pausedUntil);
lastError = null;
});
return result;
} catch (error) {
// The caller aborts its transport on admission failure. Observe a started
// promise even when persistence failed, without masking the store error.
if (value?.then) Promise.resolve(value).catch(() => {});
report(error);
return { status: 'unavailable', error };
}
},
pausedUntil() {
return current().pausedUntil || 0;
},
pauseUntil(at) {
if (file) {
try {
at = markPause(at);
} catch (error) {
report(error);
}
}
update((state) => ({ ...state, pausedUntil: at }));
},
status: () => {
const dispatchPending = pendingDispatches().length > 0;
return {
file,
durable: Boolean(file),
persisted: Boolean(file) && !lastError && !dispatchPending,
dispatchPending,
error: lastError
? String(lastError.message || lastError)
: dispatchPending
? 'unfinished durable Nominatim dispatch'
: null,
};
},
};
}
/**
* Construct a gate. `settings` may be a function so the environment is read
* after the dev server loads `.env`.
*
* @param {object} [options]
* @param {(() => object)|object} [options.settings]
* @param {typeof fetch} [options.fetchImpl]
* @param {() => number} [options.now]
* @param {(ms: number) => Promise<void>} [options.sleep]
* @param {ReturnType<typeof createGateStateStore>} [options.usage]
*/
export function createNominatimGate({
settings = () => resolveNominatimSettings(),
fetchImpl = (...args) => globalThis.fetch(...args),
now = Date.now,
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
usage = createGateStateStore(),
minSpacingMs = NOMINATIM_MIN_SPACING_MS,
maxPending = NOMINATIM_MAX_PENDING,
maxWaitMs = NOMINATIM_MAX_WAIT_MS,
timeoutMs = 9000,
} = {}) {
let queue = Promise.resolve();
let pending = 0;
const pausedUntilByEndpoint = new Map();
let upstreamRequests = 0;
let usageStore = usage;
const current = () =>
typeof settings === 'function' ? settings() : settings;
/**
* Pause until `ms` from now. The public service's pause is also written to
* the shared state so a restart or a second process honours it.
*/
function pause(ms, config) {
const until = now() + Math.max(0, ms);
const key = String(config?.endpoint || '');
if (until > (pausedUntilByEndpoint.get(key) || 0))
pausedUntilByEndpoint.set(key, until);
if (config?.isPublic) usageStore.pauseUntil(until);
}
function pauseEnd(config) {
const endpointPause =
pausedUntilByEndpoint.get(String(config?.endpoint || '')) || 0;
return config?.isPublic
? Math.max(endpointPause, usageStore.pausedUntil())
: endpointPause;
}
function pausedError(config) {
return gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
});
}
function admissionError(config) {
if (!config.endpoint)
return gateError('NOMINATIM_DISABLED', 'Place service is not configured');
if (now() < pauseEnd(config)) return pausedError(config);
return null;
}
/** One upstream request; classifies refusals and pauses the gate. */
async function send(url, config, maxBytes, signal, queuedAt) {
const controller = new AbortController();
const onAbort = () => controller.abort();
signal?.addEventListener('abort', onAbort, { once: true });
// One deadline covers the headers AND the body: a server that answers its
// headers and then stalls must not hold the shared queue.
let timer;
const start = () => {
upstreamRequests += 1;
timer = setTimeout(() => controller.abort(), timeoutMs);
return fetchImpl(url, {
headers: NOMINATIM_HEADERS,
redirect: 'error',
signal: controller.signal,
});
};
try {
return await exchange(
config,
maxBytes,
signal,
controller.signal,
queuedAt,
start,
);
} catch (error) {
controller.abort();
throw error;
} finally {
clearTimeout(timer);
signal?.removeEventListener('abort', onAbort);
}
}
async function exchange(
config,
maxBytes,
signal,
requestSignal,
queuedAt,
start,
) {
const turn = await takeTurn(config, queuedAt, signal, start);
let response;
try {
response = await turn.value;
} catch (cause) {
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
throw gateError('NOMINATIM_TRANSPORT', 'Place service unreachable', {
cause,
transport: true,
});
}
if (!response.ok) {
void response.body?.cancel?.().catch(() => {});
const status = response.status;
const retryAfter = parseRetryAfter(
response.headers?.get?.('retry-after'),
now(),
);
if (status === 429 || status === 503) {
pause(retryAfter ?? DEFAULT_BACKOFF_MS, config);
throw gateError('NOMINATIM_REFUSED', 'Place service asked to wait', {
status,
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
});
}
if (status === 403 || status === 418) {
pause(retryAfter ?? BLOCKED_BACKOFF_MS, config);
throw gateError('NOMINATIM_REFUSED', 'Place service refused', {
status,
retryAfterMs: Math.max(0, pauseEnd(config) - now()),
});
}
if (status >= 500) pause(retryAfter ?? OUTAGE_BACKOFF_MS, config);
throw gateError('NOMINATIM_UPSTREAM', `Upstream returned ${status}`, {
status,
});
}
try {
return await readResponseJsonCapped(response, maxBytes, requestSignal);
} catch (cause) {
if (cause?.code === 'RESPONSE_TOO_LARGE')
throw gateError('NOMINATIM_TOO_LARGE', 'Place answer too large');
if (signal?.aborted) throw gateError('NOMINATIM_ABANDONED', 'Abandoned');
throw gateError('NOMINATIM_TRANSPORT', 'Place answer unreadable', {
cause,
transport: true,
});
}
}
/** Revalidate public dispatch under shared ownership after every wake. */
async function takeTurn(config, queuedAt, signal, start) {
while (true) {
if (signal?.aborted || now() - queuedAt > maxWaitMs)
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
const refused = admissionError(config);
if (refused) throw refused;
// An operator-selected endpoint is not governed by the public instance's
// shared budget, pacing, or durable-state requirement.
if (!config.isPublic) {
try {
return { value: start() };
} catch (error) {
return { value: Promise.reject(error) };
}
}
const reserved = usageStore.dispatch(
utcDay(now()),
{
clock: now,
dailyCap: config.dailyCap,
minSpacingMs,
latestStartAt: queuedAt + maxWaitMs,
},
start,
);
if (reserved.status === 'unavailable')
throw gateError(
'NOMINATIM_STATE_UNAVAILABLE',
'Public place service usage state is unavailable',
{ cause: reserved.error },
);
if (reserved.status === 'cap')
throw gateError(
'NOMINATIM_DAILY_CAP',
'Daily place lookup allowance is used',
);
if (reserved.status === 'paused')
throw gateError('NOMINATIM_PAUSED', 'Place service is pausing', {
retryAfterMs: reserved.retryAfterMs,
});
if (reserved.status === 'abandoned')
throw gateError('NOMINATIM_ABANDONED', 'Place search was abandoned');
if (reserved.status === 'started') return reserved;
if (reserved.status === 'wait') await sleep(reserved.waitMs);
}
}
/**
* Queue one JSON request for `buildUrl(endpoint)`. At most one retry, and
* only after a transport failure; a failed retry pauses the gate.
*
* @param {(endpoint: string) => string} buildUrl
* @param {{signal?: AbortSignal, maxBytes?: number}} [options]
*/
function requestJson(buildUrl, { signal, maxBytes = 2 * 1024 * 1024 } = {}) {
const config = current();
const refused = admissionError(config);
if (refused) return Promise.reject(refused);
if (signal?.aborted)
return Promise.reject(
gateError('NOMINATIM_ABANDONED', 'Place search was abandoned'),
);
if (pending >= maxPending)
return Promise.reject(
gateError('NOMINATIM_QUEUE_FULL', 'Place search queue is full'),
);
pending += 1;
const queuedAt = now();
const url = buildUrl(config.endpoint);
const task = queue.then(async () => {
try {
try {
return await send(url, config, maxBytes, signal, queuedAt);
} catch (error) {
if (!error?.transport) throw error;
try {
return await send(url, config, maxBytes, signal, now());
} catch (retryError) {
if (retryError?.transport) pause(OUTAGE_BACKOFF_MS, config);
throw retryError;
}
}
} finally {
pending -= 1;
}
});
queue = task.catch(() => null);
return task;
}
return {
requestJson,
settings: current,
/** Share public admission, daily count and pauses through one install's file. */
persistUsage(file) {
usageStore = createGateStateStore({ file });
},
stats: () => ({
upstreamRequests,
pending,
pausedForMs: Math.max(0, pauseEnd(current()) - now()),
usedToday: usageStore.count(utcDay(now())),
persistence: usageStore.status?.() || null,
}),
};
}
let sharedGate = createNominatimGate();
/** The process-wide gate: search and outlines share one budget and pacer. */
export function sharedNominatimGate() {
return sharedGate;
}
/** Replace the process-wide gate (tests, embedders with their own transport). */
export function setSharedNominatimGate(gate) {
sharedGate = gate || createNominatimGate();
return sharedGate;
}
/** Codes that mean "not now" rather than "no such place". */
export function isNominatimBusy(error) {
return [
'NOMINATIM_QUEUE_FULL',
'NOMINATIM_ABANDONED',
'NOMINATIM_PAUSED',
'NOMINATIM_REFUSED',
'NOMINATIM_DAILY_CAP',
].includes(error?.code);
}
server/providers/regional/outlineSelect.js
+746
View File
@@ -0,0 +1,746 @@
import { normalizePlaceName } from './nominatimCache.js';
import {
containsAnchor,
pointInRing,
} from '../../../src/data/adminBoundaries.js';
import {
douglasPeucker,
ringAreaM2,
} from '../../../src/sources/featureGeometry.js';
/**
* Which Nominatim results may answer which voice ask, and how their polygons
* are bounded. Pure: no I/O.
*
* A search returns whatever matches the words best. "Mission District" can
* come back as a car-charging station and "Bandra" as a railway station, so a
* result is accepted only when its OSM class (and, for broad classes, its
* type) suits the kind of thing that was asked for, it carries a polygon, and
* its name actually answers the words.
*/
/** Ask kinds the outline route answers. */
export const OUTLINE_KINDS = Object.freeze([
'city',
'admin',
'neighborhood',
'landmark',
'building',
]);
/** Simplification tolerance (degrees) sent upstream, per ask kind. */
export const OUTLINE_POLYGON_THRESHOLD = Object.freeze({
city: 0.0005,
admin: 0.0005,
neighborhood: 0.0005,
landmark: 0.00005,
building: 0.00005,
});
/**
* Size limits for one outline. Vertices beyond the budget are simplified away;
* more parts, or more holes in one part, than these limits refuse the outline
* whole, because a truncated multipart place would misrepresent it.
*/
const OUTLINE_MAX_VERTICES = 6000;
const OUTLINE_MAX_PARTS = 24;
const OUTLINE_MAX_HOLES = 16;
const ANY = '*';
const SETTLEMENT_PLACES = [
'city',
'town',
'village',
'municipality',
'borough',
'hamlet',
];
const CIVIC_AMENITIES = [
'university',
'college',
'school',
'kindergarten',
'hospital',
'clinic',
'place_of_worship',
'monastery',
'townhall',
'courthouse',
'library',
'theatre',
'cinema',
'arts_centre',
'community_centre',
'conference_centre',
'events_venue',
'exhibition_centre',
'music_venue',
'planetarium',
'marketplace',
'prison',
'grave_yard',
'fire_station',
'police',
'embassy',
'public_building',
];
const LANDMARK_TOURISM = [
'attraction',
'theme_park',
'zoo',
'aquarium',
'museum',
'gallery',
'viewpoint',
'camp_site',
'hotel',
'resort',
];
const MAN_MADE_STRUCTURES = [
'tower',
'lighthouse',
'pier',
'bridge',
'works',
'dam',
'observatory',
'wastewater_plant',
'water_works',
'breakwater',
];
/**
* Accepted `class → types` per ask kind. `*` accepts every type of that
* class. Anything absent is rejected: roads, railways, stations, shops,
* guideposts and single facilities never answer an area ask.
*/
const OUTLINE_ACCEPT = Object.freeze({
city: {
boundary: ['administrative'],
place: SETTLEMENT_PLACES,
},
admin: {
boundary: ['administrative'],
place: ['country', 'state', 'province', 'region', 'county', 'district'],
},
neighborhood: {
boundary: ['administrative'],
place: ['suburb', 'quarter', 'neighbourhood', 'borough', 'city_block'],
},
landmark: {
leisure: ANY,
landuse: ANY,
amenity: CIVIC_AMENITIES,
tourism: LANDMARK_TOURISM,
building: ANY,
natural: ANY,
water: ANY,
waterway: ['riverbank', 'dock'],
historic: ANY,
man_made: MAN_MADE_STRUCTURES,
office: ['government', 'diplomatic'],
aeroway: ['aerodrome', 'heliport', 'terminal'],
military: ANY,
boundary: ['national_park', 'protected_area'],
place: ['island', 'islet', 'square'],
},
// Single structures only: no campuses, grounds, parks or camp sites.
building: {
building: ANY,
amenity: [
'place_of_worship',
'townhall',
'courthouse',
'library',
'theatre',
'cinema',
'arts_centre',
'community_centre',
'conference_centre',
'exhibition_centre',
'music_venue',
'planetarium',
'fire_station',
'police',
'embassy',
'public_building',
],
tourism: ['museum', 'gallery', 'hotel', 'attraction'],
historic: ['building', 'castle', 'church', 'fort', 'palace', 'monastery'],
man_made: ['tower', 'lighthouse', 'observatory'],
office: ANY,
},
});
/**
* Administrative evidence per ask kind: a recognized address type is required.
* A supplied numeric place_rank must also fit the ask's broad band (country 4,
* state 8, county 10-12, city 12-16, suburb 18-20, neighbourhood 20-22).
*/
const ADMIN_EVIDENCE = Object.freeze({
city: {
types: ['city', 'town', 'village', 'municipality', 'borough', 'hamlet'],
ranks: [12, 18],
},
admin: {
types: [
'country',
'state',
'province',
'region',
'county',
'district',
'state_district',
],
ranks: [2, 14],
},
neighborhood: {
types: ['suburb', 'quarter', 'neighbourhood', 'city_district', 'borough'],
ranks: [17, 24],
},
});
/**
* Nominatim address types whose meaning is stable enough for the area consumer's
* country/admin1/admin2 contract. `region`, `district` and numeric admin levels
* vary by country, so they remain unverified instead of being guessed.
*/
const VERIFIED_ADMIN_LEVEL = Object.freeze({
country: 'country',
state: 'admin1',
province: 'admin1',
county: 'admin2',
state_district: 'admin2',
});
const EXPLICIT_NAME_LEVEL = Object.freeze({
country: 'country',
nation: 'country',
province: 'admin1',
state: 'admin1',
region: 'admin1',
oblast: 'admin1',
prefecture: 'admin1',
canton: 'admin1',
governorate: 'admin1',
county: 'admin2',
district: 'admin2',
department: 'admin2',
zone: 'retired',
});
function explicitNameEvidence(value) {
const normalized = String(value || '')
.normalize('NFKC')
.toLocaleLowerCase()
.replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
.trim();
const prefix = normalized.match(
/^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/,
)?.[1];
const suffix = normalized.match(
/\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/,
)?.[1];
const levels = new Set(
[prefix, suffix]
.filter(Boolean)
.map((type) => EXPLICIT_NAME_LEVEL[type])
.filter(Boolean),
);
if (levels.size > 1) return { contradictory: true, level: null };
return { contradictory: false, level: levels.values().next().value || null };
}
/**
* Verified administrative identity carried by the selected result itself.
* Never infer it from the request, desired kind, rank, or a separate geocoder
* anchor. A retired name is preserved without a level so downstream policy can
* reject it explicitly; every other contradiction stays wholly unverified.
*/
function selectedAdminIdentity(kind, row) {
if (kind !== 'admin') return null;
const adminLevel =
VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()];
const authoritativeAdminArea = String(row?.name || '').trim();
if (!adminLevel || !authoritativeAdminArea) return null;
// Validate the complete authoritative name. `adminArea` is truncated only
// for display after every prefix/suffix signal has been considered.
const { contradictory, level: nameLevel } = explicitNameEvidence(
authoritativeAdminArea,
);
if (contradictory) return null;
const adminArea = authoritativeAdminArea.slice(0, 160);
if (nameLevel === 'retired') {
// Nepal replaced its zones in 2015, but zones are current units elsewhere
// (Ethiopia). Without a country the name stays retired.
if (!zoneIsCurrentIn(row)) return { adminArea };
return { adminArea, adminLevel };
}
if (nameLevel && nameLevel !== adminLevel) return null;
return { adminArea, adminLevel };
}
/** Whether the row names a country (last display_name part) other than Nepal. */
function zoneIsCurrentIn(row) {
const country = String(row?.display_name || '')
.split(',')
.slice(1)
.pop()
?.trim()
.toLocaleLowerCase();
return Boolean(country) && country !== 'nepal';
}
/** Whether supplied administrative evidence contradicts the accepted identity. */
function adminLevelFits(kind, row) {
const evidence = ADMIN_EVIDENCE[kind];
if (!evidence) return true;
const place = String(row?.category || row?.class).toLowerCase() === 'place';
// A place row already has an accepted type. Missing address evidence must
// not erase it, but explicit contradictory evidence must not bypass checks.
if (!place || Object.hasOwn(row, 'addresstype')) {
if (typeof row?.addresstype !== 'string') return false;
const type = row.addresstype.toLowerCase();
if (
!evidence.types.includes(type) &&
!(place && type === String(row.type).toLowerCase())
)
return false;
}
// Rank can corroborate or contradict an explicit identity, never invent it.
// JSONv2 supplies a number; coercing arrays or strings hides malformed data.
if (!Object.hasOwn(row, 'place_rank')) return true;
const rank = row.place_rank;
return (
Number.isInteger(rank) &&
// Boundary rank bands do not describe every accepted place type.
(place || (rank >= evidence.ranks[0] && rank <= evidence.ranks[1]))
);
}
/** Largest footprint (m²) a non-`building` result may have for a building ask. */
const BUILDING_MAX_AREA_M2 = 150_000;
/** Whether a result's class/type suits the ask kind. */
export function acceptsOutlineClass(kind, osmClass, osmType) {
const rules = OUTLINE_ACCEPT[kind];
if (!rules) return false;
const types = rules[String(osmClass || '').toLowerCase()];
if (!types) return false;
return types === ANY || types.includes(String(osmType || '').toLowerCase());
}
const STOP_WORDS = new Set(['the', 'of', 'de', 'la', 'le', 'el', 'and']);
/** Scripts written without spaces between words. */
const UNSPACED_SCRIPT_RE =
/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
/** Significant words of the part of a query before the first comma. */
function queryWords(query) {
const head = String(query ?? '').split(',')[0];
return normalizePlaceName(head)
.split(' ')
.filter((word) => word && !STOP_WORDS.has(word));
}
/**
* Words naming what a building is, which OSM rarely carries in the name or
* type ("Burj Khalifa", building=yes). Enclosures (zoo, garden, arch,
* conservatory) are left out: those name a feature inside a park.
*/
const BUILDING_WORDS = [
'building',
'tower',
'church',
'cathedral',
'basilica',
'chapel',
'temple',
'mosque',
'synagogue',
'shrine',
'museum',
'palace',
'castle',
'hall',
'library',
];
/** How well a result's names answer the ask. */
const NAME_MISS = 0;
const NAME_OVERLAP = 1;
const NAME_WHOLE = 2;
const NAME_EXACT = 3;
/**
* Settlements require one complete normalized name or explicit returned alias.
* Landmark/building asks retain their word-overlap and unspaced-script
* shorthand. An exact name ranks first, then one holding every asked word,
* then an overlap.
*/
function outlineNameMatch(row, query, kind) {
const names = new Set();
const add = (value) => {
const name = normalizePlaceName(value);
if (name) names.add(name);
};
add(row?.name);
add(String(row?.display_name || '').split(',')[0]);
// OSM name tags may hold several names separated by ";" ("York;Eboracum").
if (row?.namedetails && typeof row.namedetails === 'object')
for (const value of Object.values(row.namedetails))
for (const part of String(value ?? '').split(';')) add(part);
if (kind === 'city' || kind === 'admin' || kind === 'neighborhood') {
const head = normalizePlaceName(String(query ?? '').split(',')[0]);
return head && names.has(head) ? NAME_EXACT : NAME_MISS;
}
const words = queryWords(query);
if (!words.length) return NAME_MISS;
const whole = words.join(' ');
const unspaced = UNSPACED_SCRIPT_RE.test(whole);
// The last word usually names the feature ("Zoo", "Arch"): a name without
// it is the place around the feature ("Central Park"), unless the word is
// where the result is ("Space Needle Seattle"), what it is ("… Bridge"), or
// a building word on a single-structure result ("Burj Khalifa Tower").
const context = new Set(
normalizePlaceName(
[
String(row?.display_name || '')
.split(',')
.slice(1)
.join(' '),
row?.type,
row?.category || row?.class,
]
.join(' ')
.replace(/_/g, ' '),
).split(' '),
);
// Only a single structure, as a building ask would accept: never a campus,
// zoo or district whose name merely starts the ask ("Duke University").
if (acceptsOutlineClass('building', row?.category || row?.class, row?.type))
for (const word of BUILDING_WORDS) context.add(word);
let match = NAME_MISS;
for (const name of names) {
if (
name
.split(' ')
.filter((word) => !STOP_WORDS.has(word))
.join(' ') === whole
)
return NAME_EXACT;
// Scripts without spaces compare by characters instead.
if (unspaced && name.replace(/ /g, '').includes(whole.replace(/ /g, '')))
match = NAME_WHOLE;
// Whole words only: "York" never matches "Yorkshire Park".
const nameSet = new Set(name.split(' '));
const hits = words.filter((word) => nameSet.has(word)).length;
if (hits === words.length) match = NAME_WHOLE;
const last = words.at(-1);
if (
words.length > 1 &&
hits >= Math.ceil(words.length * 0.6) &&
(unspaced || nameSet.has(last) || context.has(last))
)
match = Math.max(match, NAME_OVERLAP);
}
return match;
}
const finiteLon = (value) =>
Number.isFinite(value) && value >= -180 && value <= 180;
const finiteLat = (value) =>
Number.isFinite(value) && value >= -90 && value <= 90;
/** A valid closed ring of [lon, lat] numbers, or null. */
function cleanRing(ring) {
if (!Array.isArray(ring)) return null;
const out = [];
for (const point of ring) {
if (!Array.isArray(point)) return null;
const [lon, lat] = point;
if (!finiteLon(lon) || !finiteLat(lat)) return null;
out.push([lon, lat]);
}
if (out.length < 4) return null;
const [first, last] = [out[0], out.at(-1)];
if (first[0] !== last[0] || first[1] !== last[1]) out.push([...first]);
return out;
}
/** Polygon/MultiPolygon GeoJSON → `[[outer, ...holes], …]`, largest first. */
function polygonsFromGeoJson(geometry) {
const raw =
geometry?.type === 'Polygon'
? [geometry.coordinates]
: geometry?.type === 'MultiPolygon'
? geometry.coordinates
: null;
if (!Array.isArray(raw) || !raw.length || raw.length > OUTLINE_MAX_PARTS)
return null;
const polygons = [];
for (const polygon of raw) {
if (
!Array.isArray(polygon) ||
!polygon.length ||
polygon.length - 1 > OUTLINE_MAX_HOLES
)
return null;
const outer = cleanRing(polygon[0]);
if (!outer) return null;
const holes = polygon.slice(1).map(cleanRing);
if (holes.some((hole) => !hole)) return null;
polygons.push({ rings: [outer, ...holes], area: ringAreaM2(outer) });
}
polygons.sort((a, b) => b.area - a.area);
return polygons.map((entry) => entry.rings);
}
/** Iterative Douglas–Peucker on a closed ring (keeps it closed, ≥ 4 points). */
function simplifyClosedRing(ring, tolerance) {
if (ring.length <= 8 || tolerance <= 0) return ring;
const out = douglasPeucker(ring, tolerance);
return out.length >= 4 ? out : ring;
}
const countVertices = (polygons) =>
polygons.reduce(
(sum, rings) => sum + rings.reduce((n, ring) => n + ring.length, 0),
0,
);
const edgeSide = (p, q, r) =>
(q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0]);
const onEdge = (p, q, r) =>
r[0] >= Math.min(p[0], q[0]) &&
r[0] <= Math.max(p[0], q[0]) &&
r[1] >= Math.min(p[1], q[1]) &&
r[1] <= Math.max(p[1], q[1]);
/** Intersection, including touching, of two planar closed-ring edges. */
function edgesIntersect(a, b, c, d) {
if (
Math.max(a[0], b[0]) < Math.min(c[0], d[0]) ||
Math.max(c[0], d[0]) < Math.min(a[0], b[0]) ||
Math.max(a[1], b[1]) < Math.min(c[1], d[1]) ||
Math.max(c[1], d[1]) < Math.min(a[1], b[1])
)
return false;
const ac = edgeSide(a, b, c);
const ad = edgeSide(a, b, d);
const ca = edgeSide(c, d, a);
const cb = edgeSide(c, d, b);
return (
(ac === 0 && onEdge(a, b, c)) ||
(ad === 0 && onEdge(a, b, d)) ||
(ca === 0 && onEdge(c, d, a)) ||
(cb === 0 && onEdge(c, d, b)) ||
(ac > 0 !== ad > 0 && ca > 0 !== cb > 0)
);
}
function ringsIntersect(a, b) {
for (let i = 1; i < a.length; i++)
for (let j = 1; j < b.length; j++)
if (edgesIntersect(a[i - 1], a[i], b[j - 1], b[j])) return true;
return false;
}
/**
* Check only generated bounded output, not a new input-geometry policy.
* Touching generated boundaries are conservatively unsupported here.
*/
function simplifiedTopologyFits(polygons) {
for (const [outer, ...holes] of polygons) {
for (const ring of [outer, ...holes]) {
if (ringAreaM2(ring) === 0) return false;
for (let i = 1; i < ring.length; i++)
for (let j = i + 2; j < ring.length; j++) {
if (i === 1 && j === ring.length - 1) continue;
if (edgesIntersect(ring[i - 1], ring[i], ring[j - 1], ring[j]))
return false;
}
}
for (let i = 0; i < holes.length; i++) {
const hole = holes[i];
if (
!hole.every(([lon, lat]) => pointInRing(outer, lat, lon)) ||
ringsIntersect(outer, hole)
)
return false;
for (let j = 0; j < i; j++) {
const other = holes[j];
if (
ringsIntersect(hole, other) ||
pointInRing(other, hole[0][1], hole[0][0]) ||
pointInRing(hole, other[0][1], other[0][0])
)
return false;
}
}
}
return true;
}
/** Simplify until the whole outline fits the vertex cap. */
export function capOutlineVertices(
polygons,
maxVertices = OUTLINE_MAX_VERTICES,
) {
let current = polygons;
let tolerance = 0.00002;
for (let round = 0; round < 12; round++) {
if (countVertices(current) <= maxVertices)
return current === polygons || simplifiedTopologyFits(current)
? current
: null;
current = polygons.map((rings) =>
rings.map((ring) => simplifyClosedRing(ring, tolerance)),
);
tolerance *= 2;
}
// Never turn a resource refusal into an apparently complete partial outline.
return countVertices(current) <= maxVertices &&
simplifiedTopologyFits(current)
? current
: null;
}
/**
* Every result within the limit that suits the ask, names the place and
* carries a polygon — best name match first, then upstream order — with the reason each other result was passed over. `outline` is
* the first of them, or null.
*
* @param {object[]} rows Nominatim jsonv2 results, in upstream order.
* @param {{kind: string, query: string}} ask
*/
export function selectOutlineResult(rows, { kind, query }) {
const skipped = [];
const found = [];
for (const row of Array.isArray(rows) ? rows : []) {
const osmClass = row?.category || row?.class;
const osmType = row?.type;
const label = `${osmClass}=${osmType}`;
if (!acceptsOutlineClass(kind, osmClass, osmType)) {
skipped.push(`${label}: wrong type`);
continue;
}
if (
(String(osmClass).toLowerCase() === 'place' ||
(String(osmClass).toLowerCase() === 'boundary' &&
String(osmType).toLowerCase() === 'administrative')) &&
!adminLevelFits(kind, row)
) {
skipped.push(`${label}: level`);
continue;
}
const nameMatch = outlineNameMatch(row, query, kind);
if (nameMatch === NAME_MISS) {
skipped.push(`${label}: name`);
continue;
}
const parsed = polygonsFromGeoJson(row?.geojson);
if (!parsed) {
skipped.push(`${label}: no polygon`);
continue;
}
// A building ask needs building evidence: the building class, or a
// footprint no larger than a large building.
if (
kind === 'building' &&
String(osmClass).toLowerCase() !== 'building' &&
ringAreaM2(parsed[0][0]) > BUILDING_MAX_AREA_M2
) {
skipped.push(`${label}: not a building`);
continue;
}
const polygons = capOutlineVertices(parsed);
if (!polygons) {
skipped.push(`${label}: too large`);
continue;
}
found.push({
nameMatch,
outline: {
name: String(row.name || '').slice(0, 160) || null,
displayName: String(row.display_name || '').slice(0, 300) || null,
class: String(osmClass),
type: String(osmType),
osm:
row.osm_type && row.osm_id != null
? `${row.osm_type}/${row.osm_id}`
: null,
...(selectedAdminIdentity(kind, row) || {}),
polygons,
},
});
}
// Stable: equal matches keep upstream order.
found.sort((a, b) => b.nameMatch - a.nameMatch);
const outlines = found.map((entry) => entry.outline);
const ranks = found.map((entry) => entry.nameMatch);
return { outline: outlines[0] || null, outlines, ranks, skipped };
}
/**
* Among the best-named outlines (`ranks` from selectOutlineResult; all of
* them without), the one holding `lat, lon` (the resolved anchor), else the
* first. A worse name never wins by containment alone: an exact building
* that misses a pin by metres is not replaced by the plaza around it.
* Without an anchor the first answers, as before.
*/
export function outlineAt(outlines, lat, lon, ranks = null) {
const list = Array.isArray(outlines) ? outlines : [];
if (!finiteLat(lat) || !finiteLon(lon)) return list[0] || null;
const best = Array.isArray(ranks) ? ranks[0] : undefined;
return (
list.find(
(outline, index) =>
(best === undefined || ranks[index] === best) &&
containsAnchor(outline.polygons, lat, lon),
) ||
list[0] ||
null
);
}
/**
* Coarse view bias: the centre is snapped to a grid (1° for places, 0.05°
* for small grounds) so small camera moves reuse the same cache entry, and
* the viewbox sent upstream is built from the snapped centre.
*/
export function outlineBias(kind, lat, lon) {
// Missing is missing: Number(null) and Number('') would read as (0, 0).
const given = (value) =>
value !== null &&
value !== undefined &&
(typeof value === 'number' || String(value).trim() !== '');
if (!given(lat) || !given(lon)) return null;
const latitude = Number(lat);
const longitude = Number(lon);
if (!finiteLat(latitude) || !finiteLon(longitude)) return null;
const small = kind === 'landmark' || kind === 'building';
const step = small ? 0.05 : 1;
const half = small ? 0.25 : 2;
const snap = (value) => Math.round(value / step) * step;
const cLat = Number(snap(latitude).toFixed(2));
const cLon = Number(snap(longitude).toFixed(2));
const clampLat = (value) => Math.max(-90, Math.min(90, value));
const clampLon = (value) => Math.max(-180, Math.min(180, value));
return {
key: `${cLat},${cLon}`,
viewbox: [
clampLon(cLon - half),
clampLat(cLat + half),
clampLon(cLon + half),
clampLat(cLat - half),
]
.map((value) => Number(value.toFixed(4)))
.join(','),
};
}
server/providers/regional/place.js
+451 -196
View File
@@ -1,114 +1,140 @@
import path from 'node:path';
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
import { coalesceProxyRequest } from '../common/http.js';
import { fetchRegionalJson } from './http.js';
import { admitSameSite } from '../common/same-site.js';
import { defaultSourceRoot } from '../common/source-root.js';
import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
import {
nominatimToGeocodeResult,
nominatimViewboxFromBounds,
} from '../../../src/nominatimGeocode.js';
/**
* The usage policy for the public Nominatim instance asks for a User-Agent or
* Referer that identifies the application, and states that stock library
* agents will not do. Both are sent.
*/
const NOMINATIM_HEADERS = Object.freeze({
'User-Agent':
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
});
/**
* Minimum spacing between upstream calls. The policy states an absolute
* maximum of one request per second; 1.1 s keeps clock jitter from crossing it.
*/
const NOMINATIM_MIN_SPACING_MS = 1100;
/**
* How many searches may be waiting for their turn.
*
* One request per second and an unbounded queue are incompatible: a burst of
* searches would keep the upstream busy long after everyone who asked has given
* up, which is exactly the load the policy asks callers not to create. Past
* this depth a search is refused at once instead of being promised a slot
* minutes away.
*/
const NOMINATIM_MAX_PENDING = 4;
/**
* How long a queued search may wait before it is not worth sending. The
* browser gives up well before this, so anything reaching the front later than
* this is answering nobody.
*/
const NOMINATIM_MAX_WAIT_MS = 10_000;
const NOMINATIM_SEARCH_CACHE_MS = 5 * 60_000;
const NOMINATIM_SEARCH_MAX_CACHE = 80;
import {
NOMINATIM_MAX_PENDING,
isNominatimBusy,
sharedNominatimGate,
} from './nominatimGate.js';
import {
NOMINATIM_CACHE_TTL_MS,
createNominatimCache,
} from './nominatimCache.js';
import {
OUTLINE_KINDS,
OUTLINE_POLYGON_THRESHOLD,
outlineAt,
outlineBias,
selectOutlineResult,
} from './outlineSelect.js';
const NOMINATIM_SEARCH_MAX_QUERY = 200;
// The pacer is deliberately module state while everything else here is
// per-instance. One request per second is a budget for the whole application,
// not for each provider object: two instances each pacing themselves would
// send two requests a second between them. Only last-resort forward searches
// use this queue.
let _nominatimQueue = Promise.resolve();
/** Results considered per outline ask: enough to skip a node or a wrong type. */
const OUTLINE_RESULT_LIMIT = 3;
let _nominatimLastRequestAt = 0;
/** Largest upstream answer read for an outline (polygons are simplified upstream). */
const OUTLINE_MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
let _nominatimPending = 0;
/** Bump when cached answers change shape or selection authority. */
const CACHE_SCHEMA = 'v3';
/** Raised when the queue is already as deep as it is allowed to get. */
function queueFullError() {
return Object.assign(new Error('Place search queue is full'), {
code: 'NOMINATIM_QUEUE_FULL',
/**
* The query text actually sent upstream: Unicode-composed, whitespace
* collapsed. Nothing else is changed, so two queries share a cache entry only
* when they are the same upstream request.
*/
export function upstreamQuery(query) {
return String(query ?? '')
.normalize('NFC')
.replace(/\s+/g, ' ')
.trim();
}
/** Cache key: endpoint, schema and the exact upstream parameters. */
function requestKey(gate, kind, params) {
const endpoint = gate.settings?.()?.endpoint || 'none';
return `${CACHE_SCHEMA}|${endpoint}|${kind}|${params}`;
}
/** A decimal coordinate query value, or NaN: never "" or " " as 0. */
function coordinateParam(value) {
return /^-?\d+(\.\d+)?$/.test(value ?? '') ? Number(value) : NaN;
}
function malformed() {
return Object.assign(new Error('Place answer was not understood'), {
code: 'NOMINATIM_MALFORMED',
});
}
/** Raised when a queued search waited so long that nobody is left to answer. */
function abandonedError() {
function abandoned() {
return Object.assign(new Error('Place search was abandoned'), {
code: 'NOMINATIM_ABANDONED',
});
}
/**
* Run one piece of upstream work, never closer than the policy spacing to the
* last one.
*
* `bounded` applies queue-depth and staleness limits to interactive forward searches.
*
* @param {() => Promise<unknown>} work
* @param {{bounded?: boolean, signal?: AbortSignal}} [options]
* Share one upstream call while keeping each caller's cancellation independent.
* The shared controller is aborted only after the last subscriber leaves.
*/
function enqueueNominatim(work, { bounded = false, signal } = {}) {
if (bounded && _nominatimPending >= NOMINATIM_MAX_PENDING)
return Promise.reject(queueFullError());
if (bounded) _nominatimPending += 1;
const queuedAt = Date.now();
const task = _nominatimQueue.then(async () => {
try {
const waitMs = Math.max(
0,
NOMINATIM_MIN_SPACING_MS - (Date.now() - _nominatimLastRequestAt),
);
if (waitMs) await new Promise((resolve) => setTimeout(resolve, waitMs));
// Nobody is waiting for this any more: sending it would spend the one
// request per second the policy allows on an answer with no reader.
if (bounded && Date.now() - queuedAt > NOMINATIM_MAX_WAIT_MS)
throw abandonedError();
if (signal?.aborted) throw abandonedError();
_nominatimLastRequestAt = Date.now();
return await work();
} finally {
if (bounded) _nominatimPending -= 1;
}
function subscribeSharedRequest(inFlight, key, signal, create) {
if (signal?.aborted) return Promise.reject(abandoned());
let flight = inFlight.get(key);
if (!flight) {
flight = {
controller: new AbortController(),
users: 0,
settled: false,
promise: null,
};
const owned = flight;
owned.promise = Promise.resolve()
.then(() => create(owned.controller.signal))
.finally(() => {
owned.settled = true;
if (inFlight.get(key) === owned) inFlight.delete(key);
});
inFlight.set(key, owned);
}
flight.users += 1;
return new Promise((resolve, reject) => {
let left = false;
const leave = () => {
if (left) return;
left = true;
signal?.removeEventListener('abort', onAbort);
flight.users -= 1;
if (
flight.users === 0 &&
!flight.settled &&
inFlight.get(key) === flight
) {
inFlight.delete(key);
flight.controller.abort();
}
};
const onAbort = () => {
leave();
reject(abandoned());
};
signal?.addEventListener('abort', onAbort, { once: true });
flight.promise.then(
(value) => {
leave();
resolve(value);
},
(error) => {
leave();
reject(error);
},
);
});
_nominatimQueue = task.catch(() => null);
return task;
}
/** Where the dev server keeps Nominatim answers and the daily count. */
export const NOMINATIM_CACHE_DIR = path.join(
defaultSourceRoot,
'.gev-cache',
'nominatim',
);
/** Construct the offline regional context provider from bundled Natural Earth polygons. */
export function createRegionalPlaceProvider() {
return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
@@ -117,137 +143,366 @@ export function createRegionalPlaceProvider() {
export const fetchRegionalPlace = createRegionalPlaceProvider();
/**
* Construct the forward search adapter with a trusted endpoint.
* Snap a `"swLat,swLng|neLat,neLng"` view to a coarse grid so small camera
* moves reuse one cache entry. The viewbox is a soft preference upstream.
*/
function coarseBounds(bounds) {
// Validate the original hint before snapping or clamping can disguise a
// wrapped, inverted or out-of-range box as an unrelated valid viewbox.
if (!nominatimViewboxFromBounds(bounds)) return null;
const match = String(bounds || '')
.trim()
.split('|')
.map((corner) => corner.split(',').map(Number));
if (match.length !== 2 || match.some((c) => c.length !== 2)) return null;
const [[swLat, swLng], [neLat, neLng]] = match;
if (![swLat, swLng, neLat, neLng].every(Number.isFinite)) return null;
const span = Math.max(neLat - swLat, neLng - swLng, 0.01);
const step = 2 ** Math.ceil(Math.log2(span));
const cLat = Math.round((swLat + neLat) / 2 / (step / 4)) * (step / 4);
const cLng = Math.round((swLng + neLng) / 2 / (step / 4)) * (step / 4);
const clamp = (v, limit) => Math.max(-limit, Math.min(limit, v));
const fix = (v) => Number(v.toFixed(4));
return `${fix(clamp(cLat - step / 2, 90))},${fix(clamp(cLng - step / 2, 180))}|${fix(clamp(cLat + step / 2, 90))},${fix(clamp(cLng + step / 2, 180))}`;
}
/**
* Construct the forward search adapter over the shared gate.
*
* The policy asks that results be cached, and warns that a client repeating the
* same query may be treated as faulty, so an answer is remembered and identical
* searches already in flight share one upstream call rather than queueing
* behind each other.
* The policy asks that results be cached, and warns that a client repeating
* the same query may be treated as faulty, so answers are kept (30 days for a
* hit, 24 hours for a genuine no-result; a failure is never kept) and
* identical searches already in flight share one upstream call.
*/
export function createNominatimSearchProvider({
endpoint = 'https://nominatim.openstreetmap.org/search',
requestJson = fetchRegionalJson,
gate = () => sharedNominatimGate(),
cache = createNominatimCache(),
} = {}) {
const cache = new Map();
const inFlight = new Map();
const trimCache = () => {
while (cache.size > NOMINATIM_SEARCH_MAX_CACHE) {
const oldest = cache.keys().next().value;
if (oldest === undefined) break;
cache.delete(oldest);
}
};
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
return async function fetchNominatimSearch(query, bounds, { signal } = {}) {
const cacheKey = `${query.toLowerCase()}|${bounds || ''}`;
const cached = cache.get(cacheKey);
if (cached && Date.now() - cached.cachedAt <= NOMINATIM_SEARCH_CACHE_MS) {
return { ...cached.payload, cached: true };
}
const { promise } = coalesceProxyRequest(inFlight, cacheKey, async () => {
const params = new URLSearchParams({
format: 'jsonv2',
q: query,
addressdetails: '1',
limit: '1',
'accept-language': 'en',
});
const viewbox = nominatimViewboxFromBounds(bounds);
if (viewbox) params.set('viewbox', viewbox);
const rows = await enqueueNominatim(
() =>
requestJson(`${endpoint}?${params}`, {
headers: NOMINATIM_HEADERS,
redirect: 'error',
}),
{ bounded: true, signal },
);
const result = nominatimToGeocodeResult(
Array.isArray(rows) ? rows[0] : null,
);
const payload = result
? { status: 'OK', results: [result] }
: { status: 'ZERO_RESULTS', results: [] };
cache.set(cacheKey, { payload, cachedAt: Date.now() });
trimCache();
return payload;
const coarse = coarseBounds(bounds);
const params = new URLSearchParams({
format: 'jsonv2',
q: upstreamQuery(query),
addressdetails: '1',
limit: '1',
'accept-language': 'en',
});
return await promise;
const viewbox = nominatimViewboxFromBounds(coarse);
if (viewbox) params.set('viewbox', viewbox);
const gateNow = currentGate();
const cacheKey = requestKey(gateNow, 'search', params);
const cached = await cache.get(cacheKey);
if (cached) return { ...cached.payload, cached: true };
return await subscribeSharedRequest(
inFlight,
cacheKey,
signal,
async (sharedSignal) => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal: sharedSignal },
);
// Only a well-formed empty list is "no such place"; anything else is a
// failure and is never cached.
if (!Array.isArray(rows)) throw malformed();
const result = nominatimToGeocodeResult(rows[0] ?? null);
if (rows.length && !result) throw malformed();
const payload = result
? { status: 'OK', results: [result] }
: { status: 'ZERO_RESULTS', results: [] };
await cache.set(
cacheKey,
payload,
result
? NOMINATIM_CACHE_TTL_MS.place
: NOMINATIM_CACHE_TTL_MS.notFound,
);
return payload;
},
);
};
}
/**
* Construct the outline lookup: one guarded search with polygons for one
* explicit ask, accepting only results whose class suits the ask kind.
* Every accepted result is kept (90 days; genuine no-results 24 hours), and
* each ask is answered with the one holding its anchor, so a same-name place
* elsewhere neither answers nor repeats from the cache.
*/
export function createNominatimOutlineProvider({
gate = () => sharedNominatimGate(),
cache = createNominatimCache(),
} = {}) {
const inFlight = new Map();
const currentGate = () => (typeof gate === 'function' ? gate() : gate);
const answer = (payload, anchorLat, anchorLon, cached) => ({
status: payload.status,
outline: outlineAt(payload.outlines, anchorLat, anchorLon, payload.ranks),
...(payload.skipped ? { skipped: payload.skipped } : {}),
...(cached ? { cached: true } : {}),
});
return async function fetchNominatimOutline(
{ query, kind, lat, lon, anchorLat = null, anchorLon = null },
{ signal } = {},
) {
const bias = outlineBias(kind, lat, lon);
const sent = upstreamQuery(query);
const params = new URLSearchParams({
format: 'jsonv2',
q: sent,
limit: String(OUTLINE_RESULT_LIMIT),
polygon_geojson: '1',
polygon_threshold: String(OUTLINE_POLYGON_THRESHOLD[kind]),
namedetails: '1',
'accept-language': 'en',
});
if (bias) params.set('viewbox', bias.viewbox);
const gateNow = currentGate();
const cacheKey = requestKey(gateNow, `outlines-${kind}`, params);
const cached = await cache.get(cacheKey);
if (cached) return answer(cached.payload, anchorLat, anchorLon, true);
const payload = await subscribeSharedRequest(
inFlight,
cacheKey,
signal,
async (sharedSignal) => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal: sharedSignal, maxBytes: OUTLINE_MAX_RESPONSE_BYTES },
);
if (!Array.isArray(rows)) throw malformed();
const { outlines, ranks, skipped } = selectOutlineResult(rows, {
kind,
query: sent,
});
const found = outlines.length
? { status: 'OK', outlines, ranks }
: { status: 'ZERO_RESULTS', outlines: [], skipped };
await cache.set(
cacheKey,
found,
outlines.length
? NOMINATIM_CACHE_TTL_MS.outline
: NOMINATIM_CACHE_TTL_MS.notFound,
);
return found;
},
);
return answer(payload, anchorLat, anchorLon, false);
};
}
export const fetchNominatimSearch = createNominatimSearchProvider();
/** Vite plugin: last-resort place search over the public Nominatim instance. */
export function geocodeProxy({ search = fetchNominatimSearch } = {}) {
const limiter = makeRateLimiter({
function sendJson(res, status, body, headers = {}) {
if (res.writableEnded) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...headers,
});
res.end(JSON.stringify(body));
}
/** Map a gate refusal to the route's answer. Never "not found". */
function sendFailure(res, error, what) {
if (error?.code === 'NOMINATIM_DISABLED') {
sendJson(res, 503, {
error: `${what} is not configured`,
code: 'NOMINATIM_DISABLED',
retryable: false,
});
return;
}
if (error?.code === 'NOMINATIM_DAILY_CAP') {
sendJson(
res,
429,
{
error: `${what} daily allowance is used`,
code: 'NOMINATIM_DAILY_CAP',
retryable: false,
},
{ 'Retry-After': '3600' },
);
return;
}
if (isNominatimBusy(error)) {
const seconds = Math.max(5, Math.ceil((error.retryAfterMs || 0) / 1000));
sendJson(
res,
429,
{ error: `${what} is busy`, code: error.code },
{ 'Retry-After': String(seconds) },
);
return;
}
sendJson(res, 503, {
error: `${what} is temporarily unavailable`,
code: 'NOMINATIM_UNAVAILABLE',
});
}
/**
* Vite plugin: last-resort place search (`/api/geocode`) and voice outlines
* (`/api/geocode/outline`) over the configured Nominatim endpoint.
*
* `storageDir` turns on the disk cache and the persisted daily count; the
* application composition supplies it, tests and embedders may omit it.
*/
export function geocodeProxy({
search,
outline,
storageDir = null,
gate = null,
} = {}) {
const currentGate = () => gate || sharedNominatimGate();
if (storageDir) {
// The shared count and pause live beside, never inside, the answer cache,
// so no cache sweep can reset them.
currentGate().persistUsage(path.join(storageDir, 'state.json'));
const diskCache = createNominatimCache({
dir: path.join(storageDir, 'answers'),
});
void diskCache.sweep();
search ||= createNominatimSearchProvider({
gate: currentGate,
cache: diskCache,
});
outline ||= createNominatimOutlineProvider({
gate: currentGate,
cache: diskCache,
});
}
search ||= gate
? createNominatimSearchProvider({ gate })
: fetchNominatimSearch;
outline ||= createNominatimOutlineProvider({ gate: currentGate });
const searchLimiter = makeRateLimiter({
windowMs: 60_000,
max: 30,
globalMax: 90,
});
// Outlines are explicit voice asks: a person cannot make many a minute.
const outlineLimiter = makeRateLimiter({
windowMs: 60_000,
max: 12,
globalMax: 30,
});
/** Common front door: method, same-site, rate limit, abandonment signal. */
function admit(req, res, limiter) {
if (admitSameSite(req, res)) return null;
if (req.method !== 'GET') {
sendJson(res, 405, { error: 'Method Not Allowed' });
return null;
}
if (!limiter(clientKey(req))) {
sendJson(
res,
429,
{ error: 'Rate limit exceeded' },
{ 'Retry-After': '10' },
);
return null;
}
// A browser that gave up is no longer waiting; the queue reads this
// before spending its slot.
const abandoned = new AbortController();
req.on?.('aborted', () => abandoned.abort());
res.on?.('close', () => abandoned.abort());
return abandoned.signal;
}
async function handleOutline(req, res, url) {
const signal = admit(req, res, outlineLimiter);
if (!signal) return;
const query = String(url.searchParams.get('q') || '').trim();
const kind = String(url.searchParams.get('kind') || '');
if (
!query ||
query.length > NOMINATIM_SEARCH_MAX_QUERY ||
!OUTLINE_KINDS.includes(kind)
) {
sendJson(res, 400, {
error:
'A place query of 1-200 characters and a known kind are required',
});
return;
}
try {
const payload = await outline(
{
query,
kind,
lat: url.searchParams.get('lat'),
lon: url.searchParams.get('lon'),
anchorLat: coordinateParam(url.searchParams.get('alat')),
anchorLon: coordinateParam(url.searchParams.get('alon')),
},
{ signal },
);
sendJson(
res,
200,
{ status: payload.status, outline: payload.outline || null },
{
'Cache-Control': payload.cached ? 'private, max-age=300' : 'no-store',
},
);
} catch (error) {
sendFailure(res, error, 'Outline lookup');
}
}
async function handleSearch(req, res, url) {
const signal = admit(req, res, searchLimiter);
if (!signal) return;
const query = String(url.searchParams.get('q') || '').trim();
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
sendJson(res, 400, {
error: 'A place query of 1-200 characters is required',
});
return;
}
try {
const payload = await search(query, url.searchParams.get('bounds'), {
signal,
});
sendJson(
res,
200,
{ status: payload.status, results: payload.results },
{
'Cache-Control': payload.cached ? 'private, max-age=60' : 'no-store',
},
);
} catch (error) {
sendFailure(res, error, 'Place search');
}
}
function install(middlewares) {
middlewares.use('/api/geocode', async (req, res) => {
if (req.method !== 'GET') {
res.writeHead(405, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
return;
}
if (!limiter(clientKey(req))) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Retry-After': '10',
});
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
middlewares.use('/api/geocode', async (req, res, next) => {
const url = new URL(req.url || '', 'http://localhost');
const query = String(url.searchParams.get('q') || '').trim();
if (!query || query.length > NOMINATIM_SEARCH_MAX_QUERY) {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(
JSON.stringify({
error: 'A place query of 1-200 characters is required',
}),
);
return;
}
// A browser that gave up is no longer waiting; the queue reads this
// before spending its slot.
const abandoned = new AbortController();
req.on?.('aborted', () => abandoned.abort());
res.on?.('close', () => abandoned.abort());
try {
const payload = await search(query, url.searchParams.get('bounds'), {
signal: abandoned.signal,
// Capability probe, like /api/overpass/status: clients skip lookups a
// server without a Nominatim endpoint could only refuse.
if (url.pathname === '/status' && req.method === 'GET')
return sendJson(res, 200, {
configured: Boolean(currentGate().settings()?.endpoint),
});
if (res.writableEnded) return;
res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': payload.cached ? 'public, max-age=60' : 'no-store',
});
res.end(
JSON.stringify({ status: payload.status, results: payload.results }),
);
} catch (error) {
if (res.writableEnded) return;
const busy =
error?.code === 'NOMINATIM_QUEUE_FULL' ||
error?.code === 'NOMINATIM_ABANDONED';
res.writeHead(busy ? 429 : 503, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(busy ? { 'Retry-After': '5' } : {}),
});
res.end(
JSON.stringify({
error: busy
? 'Place search is busy'
: 'Place search is temporarily unavailable',
}),
);
}
if (url.pathname === '/outline') return handleOutline(req, res, url);
if (url.pathname === '/' || url.pathname === '')
return handleSearch(req, res, url);
if (typeof next === 'function') return next();
sendJson(res, 404, { error: 'Not Found' });
});
}
server/providers/terrain.js
+62 -12
View File
@@ -6,6 +6,7 @@ import {
parseTerrainPoints,
resolveTerrainHeightRequest,
terrainPointKey,
terrainRetryAfterMs,
validTerrainResult,
} from '../../src/data/terrainHeightsProxy.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
@@ -55,6 +56,15 @@ export function terrainHeightsProxy(options = {}) {
// 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
// that range.
const UPSTREAM_CHUNK = 64;
// After a 429 the upstream is left alone for its Retry-After (at most
// RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a
// restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for
// new points (about once a second while the camera moves) otherwise asks
// again with its own retries, which keeps the limit in force and logs each
// time.
const RATE_LIMIT_PAUSE_MS = 60_000;
const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000;
let rateLimitedUntil = 0;
const MAX_POINTS = UPSTREAM_CHUNK;
/**
* Entry ceiling for the point cache, and with it the file written from it.
@@ -90,15 +100,23 @@ export function terrainHeightsProxy(options = {}) {
/**
* Drop the oldest entries until the map is back inside its ceiling.
* Oldest-first by fetch time, the same shape `trackBackfillProxies` uses.
* Terrain does not move, so evicting an old point costs one refetch, never
* a wrong answer.
* The map is kept in fetch-time order (sorted once on load, and the resolver
* re-inserts every point it refetches), so the oldest are at the front and
* no write sorts the map. Terrain does not move, so evicting an old point
* costs one refetch, never a wrong answer.
*/
function evictOverflow() {
if (mem.size <= CACHE_MAX_ENTRIES) return;
for (const key of mem.keys()) {
if (mem.size <= CACHE_MAX_ENTRIES) return;
mem.delete(key);
}
}
/** Put the map in fetch-time order; a file is not guaranteed to be. */
function orderByAge() {
const byAge = [...mem.entries()].sort((a, b) => a[1].at - b[1].at);
for (let i = 0; i < byAge.length && mem.size > CACHE_MAX_ENTRIES; i += 1)
mem.delete(byAge[i][0]);
mem.clear();
for (const [key, entry] of byAge) mem.set(key, entry);
}
/** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
const inflight = new Map();
@@ -165,6 +183,7 @@ export function terrainHeightsProxy(options = {}) {
}
diskDirty = mem.size > 0;
}
orderByAge();
evictOverflow();
} catch {
/* no disk cache yet */
@@ -190,7 +209,8 @@ export function terrainHeightsProxy(options = {}) {
*
* The first try keeps its empirically required 30s timeout; network errors,
* 429, and 5xx receive up to three jittered retries sharing a 10s added-time
* budget, with Retry-After honored within that bound.
* budget, with Retry-After honored within that bound; a longer Retry-After
* ends the retries at once so the 429 pause below starts without delay.
*
* A chunk that still fails yields nulls for its own positions rather than
* rejecting: with several chunks per request, throwing would discard every
@@ -202,15 +222,37 @@ export function terrainHeightsProxy(options = {}) {
* @returns {Promise<Array<object>>}
*/
async function fetchUpstreamAll(points) {
if (Date.now() < rateLimitedUntil) {
// Rate-limited: asking again only extends the limit. Cached points are
// still served; the rest resolve on the page through its geoid.
const error = new Error('HTTP 429');
error.status = 429;
error.coolingDown = true;
throw error;
}
const results = [];
let firstError = null;
for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
const chunk = points.slice(i, i + UPSTREAM_CHUNK);
let chunkResults = [];
try {
chunkResults = await fetchTerrainChunkWithRetry(chunk);
} catch (error) {
firstError = firstError || error;
// Re-checked per chunk for when a request spans several chunks again;
// today requests are capped at one chunk (MAX_POINTS).
if (Date.now() >= rateLimitedUntil) {
try {
chunkResults = await fetchTerrainChunkWithRetry(chunk);
} catch (error) {
firstError = firstError || error;
// Concurrent 429s extend the pause, never cut it short.
if (error?.status === 429)
rateLimitedUntil = Math.max(
rateLimitedUntil,
Date.now() +
Math.min(
terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS,
RATE_LIMIT_PAUSE_MAX_MS,
),
);
}
}
// Keep later chunks aligned even if a malformed upstream response omits
// trailing positions. The resolver will reject each null individually.
@@ -283,9 +325,17 @@ export function terrainHeightsProxy(options = {}) {
evictOverflow();
diskDirty = true;
}
if (outcome.upstreamError) {
if (outcome.upstreamError?.coolingDown) {
// Logged once, when the pause began.
} else if (outcome.upstreamError) {
const pause =
outcome.upstreamError.status === 429 &&
Date.now() < rateLimitedUntil
? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)`
: '';
console.warn(
'[terrain-heights-proxy] refresh incomplete' +
pause +
' — serving stale points when available',
);
} else if (outcome.absentPoints > 0) {
server/providers/traffic.js
+9 -6
View File
@@ -182,14 +182,17 @@ export function tomtomProxy() {
});
res.end(JSON.stringify(obj));
};
const sendTile = (buf, cacheStatus) => {
// x-tomtom-fetched-at is when the proxy fetched the tile (ms since the
// epoch), so a client handed a STALE-* copy can say how old it is.
const sendTile = (entry, cacheStatus) => {
if (res.headersSent) return;
res.writeHead(200, {
'Content-Type': 'application/x-protobuf',
'Cache-Control': 'no-store',
'x-tomtom-cache': cacheStatus,
'x-tomtom-fetched-at': String(Math.round(entry.at)),
});
res.end(buf);
res.end(entry.buf);
};
try {
@@ -235,14 +238,14 @@ export function tomtomProxy() {
}
// Fresh cache hit — never counts against the budget.
if (entry && now - entry.at < TILE_TTL_MS) {
sendTile(entry.buf, 'HIT');
sendTile(entry, 'HIT');
return;
}
// Budget governor: over the soft cap, last-good data beats a dead layer.
if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) {
if (entry) {
sendTile(entry.buf, 'STALE-BUDGET');
sendTile(entry, 'STALE-BUDGET');
} else {
sendJson(429, { error: 'budget' });
}
@@ -271,9 +274,9 @@ export function tomtomProxy() {
}
const fresh = await inflight.get(key);
if (fresh) {
sendTile(fresh.buf, 'MISS');
sendTile(fresh, 'MISS');
} else if (entry) {
sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty
sendTile(entry, 'STALE-ERROR'); // upstream down — stale beats empty
} else {
sendJson(502, { error: 'upstream' });
}
server/providers/vessels/ais-store.js
+55 -10
View File
@@ -43,14 +43,55 @@ export function ingestAisStreamEnvelope(envelope) {
if (!mmsi) return false;
if (messageType === 'ShipStaticData' || messageType === 'StaticDataReport') {
const previous = _aisStreamStatic.get(mmsi) || {};
const live = _aisStreamVessels.get(mmsi);
const liveName = live?.name === `MMSI ${mmsi}` ? '' : live?.name;
const fields =
messageType === 'StaticDataReport'
? {
Name:
message.ReportA?.Valid === false
? undefined
: message.ReportA?.Name,
Type:
message.ReportB?.Valid === false
? undefined
: message.ReportB?.ShipType,
}
: message;
// Static reports may be partial (including Class B's separate A/B parts).
// Keep known values in the cache as well as in the current live row.
const staticData = {
name: vesselNameFromAis(metadata, message, _aisStreamStatic.get(mmsi)),
type: vesselTypeFromAis(message, _aisStreamStatic.get(mmsi)),
destination: stringValue(message.Destination),
imo: stringValue(message.ImoNumber ?? message.IMO),
// The envelope repeats a cached name even on Class B's type-only part.
// Only an actual static name report replaces a known live name.
name:
stringValue(fields.Name ?? fields.ShipName) ||
liveName ||
previous.name ||
stringValue(metadata.ShipName),
type:
staticCode(fields.Type ?? fields.ShipType) ||
staticCode(live?.type) ||
staticCode(previous.type) ||
'',
destination:
stringValue(fields.Destination) ||
live?.destination ||
previous.destination ||
'',
imo:
staticCode(fields.ImoNumber ?? fields.IMO) ||
staticCode(live?.imo) ||
staticCode(previous.imo) ||
'',
};
_aisStreamStatic.set(mmsi, staticData);
mergeAisStaticIntoLiveVessel(mmsi, staticData);
// AISStream repeats the last known position in a static message's
// MetaData, but the message has no speed, course or heading and is not a
// new fix. Rebuilding a tracked vessel from it would blank its motion and
// restamp its position time, so only merge the static fields.
if (_aisStreamVessels.has(mmsi)) return true;
}
const lat = numberValue(
@@ -191,12 +232,10 @@ function approxMetersBetween(lat1, lon1, lat2, lon2) {
function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
const existing = _aisStreamVessels.get(mmsi);
if (!existing) return;
if (staticData.name && (!existing.name || existing.name === `MMSI ${mmsi}`))
existing.name = staticData.name;
if (staticData.type && !existing.type) existing.type = staticData.type;
if (staticData.destination && !existing.destination)
existing.destination = staticData.destination;
if (staticData.imo && !existing.imo) existing.imo = staticData.imo;
if (staticData.name) existing.name = staticData.name;
if (staticData.type) existing.type = staticData.type;
if (staticData.destination) existing.destination = staticData.destination;
if (staticData.imo) existing.imo = staticData.imo;
}
function vesselNameFromAis(metadata, message, staticData = {}) {
@@ -266,6 +305,12 @@ function stringValue(value) {
return String(value).trim();
}
// AIS uses zero for unavailable ship type and IMO, not as a replacement value.
function staticCode(value) {
const number = numberValue(value);
return Number.isInteger(number) && number > 0 ? String(number) : '';
}
function numberValue(value) {
if (value === null || value === undefined || value === '') return null;
const number = Number(value);
server/standalone/google-token-unavailable.js
+27
View File
@@ -0,0 +1,27 @@
/**
* Treat an unavailable Google Maps tile-token endpoint as a supported, silent
* outcome. Install after real token providers and before the unknown-API 404.
*/
export function googleTokenUnavailablePlugin() {
const install = (server) => {
server.middlewares.use((req, res, next) => {
const pathname = String(req.url || '').split('?')[0];
if (
(req.method !== 'GET' && req.method !== 'HEAD') ||
pathname !== '/api/google/tiles-token'
) {
next();
return;
}
res.writeHead(204, { 'Cache-Control': 'no-store' });
res.end();
});
};
return {
name: 'google-token-unavailable',
configureServer: install,
configurePreviewServer: install,
};
}
server/standalone/vite.config.js
+15 -3
View File
@@ -5,13 +5,17 @@ import { createBrowserViteConfig } from '../../build/vite.js';
import { localProviderPlugins } from '../providers/local.js';
import { localMcpPlugin } from '../mcp/plugin.js';
import { apiNotFoundPlugin } from './api-not-found.js';
import { googleTokenUnavailablePlugin } from './google-token-unavailable.js';
import { standaloneVoiceTools } from './voiceTools.js';
import { unwatchedRuntimeDirsPlugin } from './unwatched-cache.js';
const root = fileURLToPath(new URL('../../', import.meta.url));
/** Load this checkout's configuration and attach its local provider middleware. */
export default defineConfig(({ command, mode }) => {
export function standaloneConfig(
{ command, mode },
{ nominatimStorageDir } = {},
) {
const loaded = loadEnv(mode, root, '');
for (const [key, value] of Object.entries(loaded)) {
if (process.env[key] === undefined) process.env[key] = value;
@@ -19,17 +23,25 @@ export default defineConfig(({ command, mode }) => {
const tools = standaloneVoiceTools();
return createBrowserViteConfig({
plugins: [
...localProviderPlugins({ realtime: { tools }, gemini: { tools } }),
...localProviderPlugins({
realtime: { tools },
gemini: { tools },
nominatimStorageDir,
}),
localMcpPlugin(),
googleTokenUnavailablePlugin(),
apiNotFoundPlugin(),
unwatchedRuntimeDirsPlugin(),
],
googleApiKey: process.env.GOOGLE_MAPS_API_KEY,
cesiumToken: process.env.CESIUM_ION_TOKEN,
photonUrl: process.env.PHOTON_URL,
mapillaryToken: process.env.MAPILLARY_CLIENT_TOKEN,
host: process.env.HOST,
port: process.env.PORT,
allowedHosts: resolveAllowedHosts(process.env.GEV_ALLOWED_HOSTS),
command,
});
});
}
export default defineConfig(standaloneConfig);