mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 02:14:01 +00:00
Merge remote-tracking branch 'upstream/main' into fix/drone-airspace
# Conflicts: # docs/CURRENT-STATE.md # src/app/constructCatalog.test.mjs # src/app/sourceComposition.test.mjs # src/data/layerState.test.mjs # src/data/layerStateTokenLedger.test.mjs # src/i18n/locales/ar.json
This commit is contained in:
195 files changed
+23414
-1578
No files matched your search
@@ -0,0 +1,137 @@
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
/**
|
||||
* A bounded two-level cache for JSON answers: memory (entry count) and disk
|
||||
* (entry count, total bytes, age). Every memory insertion goes through one
|
||||
* function, so a disk hit is bounded like a fresh answer; disk writes are
|
||||
* followed by a prune that removes expired files first, then the oldest,
|
||||
* until both quotas hold.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string|null} options.dir Disk directory; null keeps memory only.
|
||||
* @param {number} options.ttlMs Age past which an entry is not served.
|
||||
* @param {number} [options.maxMemoryEntries]
|
||||
* @param {number} [options.maxDiskEntries]
|
||||
* @param {number} [options.maxDiskBytes]
|
||||
* @param {() => number} [options.now]
|
||||
*/
|
||||
export function createBoundedCache({
|
||||
dir,
|
||||
ttlMs,
|
||||
maxMemoryEntries = 60,
|
||||
maxDiskEntries = 400,
|
||||
maxDiskBytes = 256 * 1024 * 1024,
|
||||
now = () => Date.now(),
|
||||
}) {
|
||||
const memory = new Map();
|
||||
let pruning = null;
|
||||
let prunePending = false;
|
||||
|
||||
const fileFor = (key) =>
|
||||
path.join(dir, `${createHash('sha1').update(key).digest('hex')}.json`);
|
||||
|
||||
function remember(key, entry) {
|
||||
memory.delete(key);
|
||||
memory.set(key, entry);
|
||||
while (memory.size > maxMemoryEntries)
|
||||
memory.delete(memory.keys().next().value);
|
||||
}
|
||||
|
||||
async function prune() {
|
||||
if (!dir) return;
|
||||
let names;
|
||||
try {
|
||||
names = (await fsp.readdir(dir)).filter((name) => name.endsWith('.json'));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const files = [];
|
||||
for (const name of names) {
|
||||
const file = path.join(dir, name);
|
||||
try {
|
||||
const stat = await fsp.stat(file);
|
||||
files.push({ file, size: stat.size, mtime: stat.mtimeMs });
|
||||
} catch {
|
||||
/* removed meanwhile */
|
||||
}
|
||||
}
|
||||
const cutoff = now() - ttlMs;
|
||||
const keep = [];
|
||||
for (const entry of files) {
|
||||
if (entry.mtime < cutoff) await fsp.rm(entry.file, { force: true });
|
||||
else keep.push(entry);
|
||||
}
|
||||
keep.sort((a, b) => a.mtime - b.mtime);
|
||||
let bytes = keep.reduce((sum, entry) => sum + entry.size, 0);
|
||||
while (
|
||||
keep.length &&
|
||||
(keep.length > maxDiskEntries || bytes > maxDiskBytes)
|
||||
) {
|
||||
const oldest = keep.shift();
|
||||
bytes -= oldest.size;
|
||||
await fsp.rm(oldest.file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Run one prune at a time; a request during a prune schedules one more. */
|
||||
function schedulePrune() {
|
||||
if (pruning) {
|
||||
prunePending = true;
|
||||
return pruning;
|
||||
}
|
||||
pruning = prune()
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
pruning = null;
|
||||
if (prunePending) {
|
||||
prunePending = false;
|
||||
schedulePrune();
|
||||
}
|
||||
});
|
||||
return pruning;
|
||||
}
|
||||
|
||||
return {
|
||||
/** The cached value, or undefined. A disk hit is re-inserted into memory. */
|
||||
async get(key) {
|
||||
const held = memory.get(key);
|
||||
if (held && now() - held.cachedAt <= ttlMs) return held.value;
|
||||
if (held) memory.delete(key);
|
||||
if (!dir) return undefined;
|
||||
try {
|
||||
const entry = JSON.parse(await fsp.readFile(fileFor(key), 'utf8'));
|
||||
if (!Number.isFinite(entry?.cachedAt) || now() - entry.cachedAt > ttlMs)
|
||||
return undefined;
|
||||
remember(key, entry);
|
||||
return entry.value;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
},
|
||||
/** Store a value in memory and, when a directory is set, on disk. */
|
||||
async set(key, value) {
|
||||
const entry = { value, cachedAt: now() };
|
||||
remember(key, entry);
|
||||
if (!dir) return;
|
||||
try {
|
||||
await fsp.mkdir(dir, { recursive: true });
|
||||
await fsp.writeFile(fileFor(key), JSON.stringify(entry));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
// In the background: an answer never waits on a 400-file directory scan.
|
||||
// `prune()` returns the pending pass for callers that need it done.
|
||||
schedulePrune();
|
||||
},
|
||||
/** Run a prune and wait until no pass is running or queued. */
|
||||
async prune() {
|
||||
schedulePrune();
|
||||
while (pruning) await pruning;
|
||||
},
|
||||
get memorySize() {
|
||||
return memory.size;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { isIPv6 } from 'node:net';
|
||||
import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
|
||||
@@ -60,3 +61,46 @@ export function makeCostRateLimiter(envValue, defaultPerMin) {
|
||||
export function clientKey(req) {
|
||||
return String(req.socket?.remoteAddress || 'local');
|
||||
}
|
||||
|
||||
const IPV4_MAPPED_RE = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/i;
|
||||
|
||||
/**
|
||||
* Whether a socket peer address is this machine: IPv4 loopback in any of its
|
||||
* forms (`127.0.0.0/8`, including the IPv4-mapped `::ffff:127.x` a dual-stack
|
||||
* listener reports) or the IPv6 loopback. Non-address keys are not loopback.
|
||||
* @param {string} address - A `clientKey` value.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLoopbackPeer(address) {
|
||||
const raw = String(address || '');
|
||||
const v4 = IPV4_MAPPED_RE.exec(raw)?.[1] ?? raw;
|
||||
return v4.startsWith('127.') || raw === '::1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key that survives address rotation, for limiters whose global
|
||||
* backstop everyone shares. IPv6 privacy extensions hand one host a fresh
|
||||
* address at will inside its /64, so keying by the full address lets a single
|
||||
* host mint per-peer quota until the global window closes for everyone else.
|
||||
* IPv6 peers are keyed by their /64 prefix instead, and an IPv4-mapped
|
||||
* address folds to its IPv4 form so one host cannot hold two buckets. Any
|
||||
* other value (IPv4, or the `local` fallback) passes through unchanged.
|
||||
* @param {import('node:http').IncomingMessage} req
|
||||
* @returns {string}
|
||||
*/
|
||||
export function clientNetworkKey(req) {
|
||||
const address = clientKey(req);
|
||||
const mapped = IPV4_MAPPED_RE.exec(address);
|
||||
if (mapped) return mapped[1];
|
||||
const bare = address.replace(/%.*$/, ''); // link-local zone id
|
||||
if (!isIPv6(bare)) return address;
|
||||
const [head, tail = ''] = bare.split('::');
|
||||
const headParts = head ? head.split(':') : [];
|
||||
const tailParts = tail ? tail.split(':') : [];
|
||||
const missing = Math.max(0, 8 - headParts.length - tailParts.length);
|
||||
const hextets = [...headParts, ...Array(missing).fill('0'), ...tailParts];
|
||||
return `${hextets
|
||||
.slice(0, 4)
|
||||
.map((h) => h.toLowerCase().replace(/^0+(?=.)/, ''))
|
||||
.join(':')}::/64`;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { clientNetworkKey, isLoopbackPeer } from './rate-limit.js';
|
||||
|
||||
const req = (remoteAddress) => ({ socket: { remoteAddress } });
|
||||
|
||||
test('clientNetworkKey keys IPv6 peers by /64 and folds IPv4-mapped peers', () => {
|
||||
assert.equal(clientNetworkKey(req('2001:db8:1:2::1')), '2001:db8:1:2::/64');
|
||||
assert.equal(
|
||||
clientNetworkKey(req('2001:0db8:0001:0002:abcd:ef01:2345:6789')),
|
||||
'2001:db8:1:2::/64',
|
||||
);
|
||||
assert.equal(clientNetworkKey(req('2001:DB8::1')), '2001:db8:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('fe80::1%en0')), 'fe80:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::1')), '0:0:0:0::/64');
|
||||
assert.equal(clientNetworkKey(req('::ffff:10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req('10.0.0.5')), '10.0.0.5');
|
||||
assert.equal(clientNetworkKey(req(undefined)), 'local');
|
||||
assert.equal(clientNetworkKey(req('not-an-address')), 'not-an-address');
|
||||
});
|
||||
|
||||
test('isLoopbackPeer recognizes every loopback form and nothing else', () => {
|
||||
for (const address of ['127.0.0.1', '127.9.9.9', '::1', '::ffff:127.0.0.1']) {
|
||||
assert.equal(isLoopbackPeer(address), true, address);
|
||||
}
|
||||
for (const address of [
|
||||
'10.0.0.1',
|
||||
'::ffff:10.0.0.1',
|
||||
'2001:db8::1',
|
||||
'::2',
|
||||
'local',
|
||||
'',
|
||||
undefined,
|
||||
]) {
|
||||
assert.equal(isLoopbackPeer(address), false, String(address));
|
||||
}
|
||||
});
|
||||
@@ -58,7 +58,7 @@ function forwardedProtocol(req) {
|
||||
|
||||
/**
|
||||
* Cross-site gate for keyless routes the map needs in order to render
|
||||
* (`/api/terrain/heights`). It refuses the same cross-site browser shapes as
|
||||
* (`/api/terrain/heights`, `/api/overpass`). It refuses the same cross-site browser shapes as
|
||||
* `admitSameSite` (a `Sec-Fetch-Site` other than same-origin/none, a foreign
|
||||
* or opaque `Origin`) but does not refuse reverse-proxy forwarding headers:
|
||||
* those protect credentials and spend, which a keyless route has neither of,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { realtimeInstructions } from '../openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
|
||||
|
||||
export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
|
||||
export const GEMINI_TOKEN_ENDPOINT =
|
||||
@@ -14,22 +14,26 @@ export function normalizeGeminiModel(value) {
|
||||
return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
|
||||
}
|
||||
|
||||
/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
|
||||
/**
|
||||
* REST BidiGenerateContentSetup fields, excluding its model resource name.
|
||||
* `overpass` says an operator configured Overpass, which `osm_query` needs.
|
||||
*/
|
||||
export function createGeminiLiveConfig(
|
||||
annotationGuidance,
|
||||
inputMode = 'open-mic',
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
if (!['open-mic', 'push-to-talk'].includes(inputMode))
|
||||
throw new Error('Invalid Gemini input mode');
|
||||
const config = {
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
systemInstruction: {
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance) }],
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance, { overpass }) }],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: tools.map(
|
||||
functionDeclarations: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
isGeminiRequestOriginAllowed,
|
||||
readGeminiTokenRequest,
|
||||
} from './request.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
|
||||
const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
|
||||
const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
@@ -19,6 +20,8 @@ const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
tools,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
resolveModel = () =>
|
||||
@@ -104,6 +107,7 @@ export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
inputMode,
|
||||
tools,
|
||||
{ overpass: overpassConfigured() },
|
||||
);
|
||||
const issuedAt = now();
|
||||
// REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
import { SNAPSHOT_FEED_STATES } from '../../../src/data/layerSnapshot.js';
|
||||
import {
|
||||
HUD_SUMMARY_INSTRUCTIONS,
|
||||
HUD_SUMMARY_MAX_CHARS,
|
||||
HUD_SUMMARY_MAX_UNSPACED_CHARS,
|
||||
HUD_SUMMARY_RESPONSE_FORMAT,
|
||||
hudSummaryInput,
|
||||
hudSummaryInstructions,
|
||||
hudSummaryIsUnspaced,
|
||||
hudSummaryLocale,
|
||||
keylessHudSummaryResponse,
|
||||
} from '../../../src/hudSummaryResponse.js';
|
||||
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
@@ -18,14 +25,58 @@ function extractOpenAiResponseText(data) {
|
||||
.trim();
|
||||
}
|
||||
|
||||
function toFiveWordHudSummary(value) {
|
||||
return String(value || '')
|
||||
.replace(/[^\p{L}\p{N}\s-]/gu, ' ')
|
||||
.trim()
|
||||
/** Parse the summary and normalize the canonical feed state. */
|
||||
function parseHudSummaryReply(text, locale) {
|
||||
let reply = null;
|
||||
try {
|
||||
reply = JSON.parse(text);
|
||||
} catch {
|
||||
/* not JSON: no summary */
|
||||
}
|
||||
const feedState = SNAPSHOT_FEED_STATES.includes(reply?.feedState)
|
||||
? reply.feedState
|
||||
: null;
|
||||
return {
|
||||
summary: toHudSummary(reply?.summary, locale),
|
||||
feedState,
|
||||
};
|
||||
}
|
||||
|
||||
function toHudSummary(value, requested) {
|
||||
const locale = hudSummaryLocale(requested);
|
||||
const words = String(value || '')
|
||||
.normalize('NFC')
|
||||
// Marks, and apostrophes and dots inside a word, belong to it: ar vowel
|
||||
// signs, fr d'Orléans, ja サンフランシスコ・ベイエリア.
|
||||
.replace(/[^\p{L}\p{M}\p{N}\s'’·・-]/gu, ' ')
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.slice(0, 5)
|
||||
.join(' ');
|
||||
.map((word) => word.replace(/^['’·・-]+|['’·・-]+$/gu, ''))
|
||||
.filter(Boolean);
|
||||
// Characters, not words: the model counts Rue de Rivoli as one word.
|
||||
const max = hudSummaryIsUnspaced(locale)
|
||||
? HUD_SUMMARY_MAX_UNSPACED_CHARS
|
||||
: HUD_SUMMARY_MAX_CHARS;
|
||||
return capAtWord(words.join(' '), max, locale);
|
||||
}
|
||||
|
||||
/**
|
||||
* Cut `text` to `max` characters at a word boundary; a first word longer
|
||||
* than that is cut where the cap falls.
|
||||
*/
|
||||
function capAtWord(text, max, locale) {
|
||||
const graphemes = [
|
||||
...new Intl.Segmenter(locale, { granularity: 'grapheme' }).segment(text),
|
||||
];
|
||||
if (graphemes.length <= max) return text;
|
||||
const cut = graphemes[max].index;
|
||||
let end = 0;
|
||||
for (const { index, segment } of new Intl.Segmenter(locale, {
|
||||
granularity: 'word',
|
||||
}).segment(text)) {
|
||||
if (index + segment.length > cut) break;
|
||||
end = index + segment.length;
|
||||
}
|
||||
return text.slice(0, end).trim() || text.slice(0, cut);
|
||||
}
|
||||
|
||||
async function handleHudSummary(req, res) {
|
||||
@@ -77,14 +128,18 @@ async function handleHudSummary(req, res) {
|
||||
model:
|
||||
process.env.OPENAI_HUD_SUMMARY_MODEL ||
|
||||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
|
||||
instructions: HUD_SUMMARY_INSTRUCTIONS,
|
||||
input: JSON.stringify(context),
|
||||
instructions: hudSummaryInstructions(context?.locale),
|
||||
input: hudSummaryInput(context),
|
||||
text: { format: HUD_SUMMARY_RESPONSE_FORMAT },
|
||||
reasoning: { effort: 'minimal' },
|
||||
max_output_tokens: 100,
|
||||
max_output_tokens: 200,
|
||||
}),
|
||||
});
|
||||
const data = await response.json().catch(() => ({}));
|
||||
const summary = toFiveWordHudSummary(extractOpenAiResponseText(data));
|
||||
const { summary, feedState } = parseHudSummaryReply(
|
||||
extractOpenAiResponseText(data),
|
||||
context?.locale,
|
||||
);
|
||||
res.statusCode = response.ok && summary ? 200 : response.status || 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -93,6 +148,7 @@ async function handleHudSummary(req, res) {
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
summary: summary || null,
|
||||
feedState,
|
||||
// Never relay `data.error.message`: that is OpenAI's own wording, and
|
||||
// it carries request ids, organization hints and quota phrasing.
|
||||
error: response.ok ? null : 'OpenAI HUD summary request failed',
|
||||
@@ -110,4 +166,4 @@ async function handleHudSummary(req, res) {
|
||||
}
|
||||
}
|
||||
|
||||
export { handleHudSummary };
|
||||
export { handleHudSummary, parseHudSummaryReply };
|
||||
@@ -1,8 +1,14 @@
|
||||
const DEFAULT_ANNOTATION_GUIDANCE =
|
||||
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
|
||||
|
||||
/**
|
||||
* The Realtime session instructions. `overpass` says an operator configured
|
||||
* Overpass, which is what offers osm_query (see tools.js); without it the
|
||||
* prompt never mentions the tool.
|
||||
*/
|
||||
function realtimeInstructions(
|
||||
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
return [
|
||||
"You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
|
||||
@@ -21,16 +27,20 @@ function realtimeInstructions(
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
|
||||
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
|
||||
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names, or an areaId you already have; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'AREAS: resolve_area turns a named boundary (country, state, province, county, city, district, natural region) into an areaId; to outline it pass draw=true; annotate_map is for marks, not boundaries. Count inside it → analyst_query kind=area with the areaId (or kind=region); "the area I drew" → kind=drawn; "the area you outlined" → kind=annotation. Imagery of a place → resolve_area, then find_imagery. Ambiguous name: call resolve_area; on needsClarification ask which, never guess.' +
|
||||
(overpass
|
||||
? ' "All hospitals/schools/… in <place>" → resolve_area, then osm_query; in view → osm_query alone; follow-ups → analyst_query layers [osm-places], scope anywhere.'
|
||||
: ''),
|
||||
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
|
||||
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
|
||||
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
|
||||
'Do not mention disabled layers or stale selections.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target, find_imagery area. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers, approximate outlines and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
|
||||
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
|
||||
//
|
||||
@@ -60,7 +70,7 @@ function realtimeInstructions(
|
||||
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
|
||||
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
|
||||
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus or district, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park or district footprint; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
annotationGuidance,
|
||||
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
|
||||
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
|
||||
|
||||
@@ -13,7 +13,8 @@ import {
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
|
||||
} from './constants.js';
|
||||
import { realtimeInstructions } from './instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from './tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
import {
|
||||
preferCodexOAuth,
|
||||
readCodexOAuthAccessToken,
|
||||
@@ -136,6 +137,8 @@ function createRealtimeTokenHandler({
|
||||
environment = process.env,
|
||||
models = {},
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
} = {}) {
|
||||
return async (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -273,6 +276,7 @@ function createRealtimeTokenHandler({
|
||||
transcribeModel.toLowerCase() === 'off'
|
||||
? {}
|
||||
: { transcription: { model: transcribeModel } };
|
||||
const overpass = overpassConfigured();
|
||||
const sessionConfig = {
|
||||
session: {
|
||||
type: 'realtime',
|
||||
@@ -298,8 +302,8 @@ function createRealtimeTokenHandler({
|
||||
},
|
||||
output: { voice },
|
||||
},
|
||||
instructions: realtimeInstructions(annotationGuidance),
|
||||
tools,
|
||||
instructions: realtimeInstructions(annotationGuidance, { overpass }),
|
||||
tools: offeredTools(tools, { overpass }),
|
||||
tool_choice: 'auto',
|
||||
},
|
||||
};
|
||||
|
||||
@@ -405,7 +405,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
annotate_map: {
|
||||
description:
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -547,9 +547,48 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
resolve_area: {
|
||||
description:
|
||||
'Named place → reusable areaId for analyst_query scope {kind:area} and other area tools. draw=true outlines it ("outline Bagmati province"); around=true is "an area around <landmark>": its enclosing mapped area, else a buffer marked approximate. needsClarification: ask which, then call again with its candidateId. AREA_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
query: {
|
||||
description: 'The place as said, with any kind word.',
|
||||
$position: 1,
|
||||
},
|
||||
level: {
|
||||
description: 'Only when the user names the kind.',
|
||||
$position: 1,
|
||||
},
|
||||
within: {
|
||||
description: 'Country or state the user added.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
find_imagery: {
|
||||
description:
|
||||
'Recent satellite imagery (Sentinel-2/Landsat 30 m, VIIRS daily; last 30 days): pins the best acquisition in the Recent Imagery panel. areaId from resolve_area; area=pointer for "this spot"; omit for the view. BOX_REFUSED: offer to zoom in. DISPLACED: leave it.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
maxCloudPct: {
|
||||
description: '"cloud-free" → 10.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
osm_query: {
|
||||
description:
|
||||
'OpenStreetMap places of one kind ("hospitals in Kathmandu"). areaId from resolve_area; omit for the view. Shows them in OSM Places with numbered referents; they become the last answer. countScope bounding-box means an approximate count. OSM_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), NEEDS_CLARIFICATION (ask which), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -560,7 +599,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
scope: {
|
||||
description:
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"); area + areaId from resolve_area; drawn = "the area I drew"; annotation = "the area you outlined". Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates.',
|
||||
$position: 2,
|
||||
properties: {
|
||||
name: {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createActionTools } from '../../../src/voice/actionSchemas.js';
|
||||
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
|
||||
|
||||
/** Every voice tool, as the Realtime session describes them. */
|
||||
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
|
||||
|
||||
/**
|
||||
@@ -14,3 +15,27 @@ export function realtimeSessionTools(additional = []) {
|
||||
...additional.filter((tool) => !names.has(tool.name)),
|
||||
];
|
||||
}
|
||||
|
||||
/** Tools that need an operator-configured Overpass (OVERPASS_UPSTREAMS). */
|
||||
export const OVERPASS_ONLY_TOOLS = Object.freeze(['osm_query']);
|
||||
|
||||
/**
|
||||
* The tools one session is offered. Bulk OpenStreetMap place search runs
|
||||
* only on an operator's Overpass, so without one `osm_query` is left out
|
||||
* rather than offered and refused.
|
||||
* @param {Array<{name: string}>} tools
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function offeredTools(tools, { overpass = false } = {}) {
|
||||
return overpass
|
||||
? tools
|
||||
: tools.filter((tool) => !OVERPASS_ONLY_TOOLS.includes(tool.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* The app actions one session is offered.
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function realtimeTools({ overpass = false } = {}) {
|
||||
return offeredTools(GEV_REALTIME_TOOLS, { overpass });
|
||||
}
|
||||
@@ -1,6 +1,12 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import {
|
||||
makeRateLimiter,
|
||||
clientKey,
|
||||
clientNetworkKey,
|
||||
isLoopbackPeer,
|
||||
} from './common/rate-limit.js';
|
||||
import { hasProxySignals } from '../../src/localRequestGate.mjs';
|
||||
import { readRequestBodyCapped } from './common/request.js';
|
||||
import { admitSameSite } from './common/same-site.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
@@ -22,6 +28,7 @@ import {
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import { installRouteMiddleware } from './places/routes.js';
|
||||
import { installOsmFeaturesRoute } from './overpass/osmFeaturesRoute.js';
|
||||
|
||||
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
|
||||
const _overpassInFlight = new Map();
|
||||
@@ -29,10 +36,21 @@ const _overpassInFlight = new Map();
|
||||
let _overpassConcurrent = 0;
|
||||
|
||||
// Bound body-reading and validation work, including malformed requests and
|
||||
// cache hits. Keep the tighter upstream budget below independent of this gate.
|
||||
const _overpassRequestLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 300,
|
||||
// cache hits, before a byte is read. Remote peers share one global window so
|
||||
// address rotation cannot mint admission without bound (IPv6 peers are keyed
|
||||
// by /64 for the same reason). A direct loopback request — the owner's own
|
||||
// browser and the repo QA harnesses, with no forwarding headers — has its own
|
||||
// per-peer window outside that global, so busy LAN peers cannot lock the owner
|
||||
// out of the map. A request relayed by a reverse proxy on this machine also
|
||||
// arrives from loopback, but it is remote traffic: it is admitted from the
|
||||
// remote pool under its own socket-keyed bucket, never from the owner's. The
|
||||
// forwarded client address is not trusted as identity. Only admission is
|
||||
// isolated: the upstream budget below (90/min per key, 300/min globally) is
|
||||
// shared, so an owner request that misses cache can still be refused there.
|
||||
const OVERPASS_ADMISSION_WINDOW = { windowMs: 60_000, max: 300 };
|
||||
const _overpassLoopbackAdmission = makeRateLimiter(OVERPASS_ADMISSION_WINDOW);
|
||||
const _overpassPeerAdmission = makeRateLimiter({
|
||||
...OVERPASS_ADMISSION_WINDOW,
|
||||
globalMax: 900,
|
||||
});
|
||||
|
||||
@@ -42,6 +60,33 @@ const _overpassRateLimiter = makeRateLimiter({
|
||||
globalMax: 300,
|
||||
});
|
||||
|
||||
/**
|
||||
* Classify a request for admission. Direct loopback — a loopback socket with
|
||||
* no forwarding header — is the owner's own pool; everything else, including a
|
||||
* loopback socket that carries forwarding headers (a local reverse proxy), is
|
||||
* remote. A proxied request keeps its socket identity under a `proxied:`
|
||||
* prefix, so it never shares a bucket with the owner's direct requests.
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @returns {{direct: boolean, key: string}}
|
||||
*/
|
||||
function overpassAdmissionClass(req) {
|
||||
const peer = clientNetworkKey(req);
|
||||
if (!isLoopbackPeer(clientKey(req))) return { direct: false, key: peer };
|
||||
if (hasProxySignals(req.headers || {}))
|
||||
return { direct: false, key: `proxied:${peer}` };
|
||||
return { direct: true, key: peer };
|
||||
}
|
||||
|
||||
/**
|
||||
* Spend one admission slot for this request; false when its bucket is over its
|
||||
* window (or, for remote traffic, when the shared global window is full).
|
||||
* @param {{direct: boolean, key: string}} admission - `overpassAdmissionClass(req)`.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function admitOverpassRequest({ direct, key }) {
|
||||
return (direct ? _overpassLoopbackAdmission : _overpassPeerAdmission)(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a completed Overpass payload to the HTTP response.
|
||||
*
|
||||
@@ -103,8 +148,13 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (!_overpassRequestLimiter(clientKey(req))) {
|
||||
// Keyless gate: refuses cross-site browser requests but, unlike the
|
||||
// cost-bearing routes, admits the forwarding headers a reverse proxy
|
||||
// adds — the layers that depend on Overpass must keep working for an
|
||||
// instance deployed the way the README documents.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
const admission = overpassAdmissionClass(req);
|
||||
if (!admitOverpassRequest(admission)) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
@@ -163,7 +213,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
// requests; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () =>
|
||||
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
allowUpstream: () => _overpassRateLimiter(admission.key),
|
||||
loadUpstream: async () => {
|
||||
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
|
||||
return {
|
||||
@@ -225,6 +275,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
});
|
||||
|
||||
installRouteMiddleware(server.middlewares, routing);
|
||||
installOsmFeaturesRoute(server.middlewares);
|
||||
};
|
||||
return {
|
||||
name: 'overpass-proxy',
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { OSM_PRESETS } from '../../../src/data/osmPresets.js';
|
||||
|
||||
/**
|
||||
* Bounded, typed OpenStreetMap feature search: one preset from the curated
|
||||
* table inside a bounding box.
|
||||
*
|
||||
* The request carries a preset id, a box, a mode and a limit — never query
|
||||
* text. The server writes the Overpass query itself and bounds it: box size
|
||||
* per mode (a count may cover more ground than a fetch), result count,
|
||||
* Overpass timeout and memory, response bytes, and one query in flight. It
|
||||
* runs only against the operator's `OVERPASS_UPSTREAMS`; the generic
|
||||
* `/api/overpass` guard is untouched.
|
||||
*
|
||||
* There is deliberately no boundary (`area`) filter: Overpass servers differ
|
||||
* in whether they hold area data, and one without it answers zero rather than
|
||||
* failing. The browser filters the box's features by the exact area geometry
|
||||
* instead, so a count inside a province is exact and a box count is labelled
|
||||
* as a box count.
|
||||
*/
|
||||
export const OSM_FEATURE_LIMITS = Object.freeze({
|
||||
/** Largest box a fetch may cover, in square degrees (~200 × 200 km at the equator). */
|
||||
maxFetchDeg2: 4,
|
||||
/** Largest box a count may cover. */
|
||||
maxCountDeg2: 36,
|
||||
/** Most features one fetch returns. */
|
||||
maxLimit: 1000,
|
||||
/** Overpass server-side timeout, seconds: it gives up before we stop waiting. */
|
||||
timeoutS: 20,
|
||||
/** Overpass server-side memory cap, bytes. */
|
||||
maxsize: 64 * 1024 * 1024,
|
||||
/** Response bytes read from the upstream. */
|
||||
maxResponseBytes: 16 * 1024 * 1024,
|
||||
/** Per-upstream wait: the Overpass timeout plus transfer time. */
|
||||
upstreamTimeoutMs: 22_000,
|
||||
/**
|
||||
* The whole search, across every configured endpoint in turn. It ends
|
||||
* before the voice answer's own deadline (`OSM_QUERY_LIMITS.budgetMs` in
|
||||
* `src/voice/osmActions.js`, 25 s): a browser that gives up first closes
|
||||
* the request, which cancels the upstream query, so nothing would be
|
||||
* cached for the retry.
|
||||
*/
|
||||
upstreamBudgetMs: 22_000,
|
||||
});
|
||||
|
||||
/** Tags a returned feature keeps, besides the preset's own. */
|
||||
const KEPT_TAGS = [
|
||||
'name',
|
||||
'name:en',
|
||||
'operator',
|
||||
'brand',
|
||||
'opening_hours',
|
||||
'wheelchair',
|
||||
'emergency',
|
||||
'healthcare',
|
||||
'addr:street',
|
||||
'addr:city',
|
||||
];
|
||||
|
||||
const quote = (text) =>
|
||||
`"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
||||
|
||||
/**
|
||||
* Parse and bound a request. Returns the compiled query or a refusal.
|
||||
* @param {{preset: string, bbox: string|number[], mode?: string, limit?: string|number}} request
|
||||
* @returns {{ok: true, ql: string, mode: string, preset: object, limit: number, boxes: number[][]}|{ok: false, code: string, error: string}}
|
||||
*/
|
||||
export function compileOsmFeatureQuery(request = {}) {
|
||||
const refuse = (code, error) => ({ ok: false, code, error });
|
||||
const presetId = String(request.preset || '');
|
||||
const preset = Object.hasOwn(OSM_PRESETS, presetId)
|
||||
? OSM_PRESETS[presetId]
|
||||
: null;
|
||||
if (!preset) return refuse('UNKNOWN_PRESET', 'Unknown feature kind.');
|
||||
const mode = request.mode === 'count' ? 'count' : 'features';
|
||||
const parts = (
|
||||
Array.isArray(request.bbox)
|
||||
? request.bbox
|
||||
: String(request.bbox || '').split(',')
|
||||
).map(Number);
|
||||
if (parts.length !== 4 || !parts.every(Number.isFinite))
|
||||
return refuse('BAD_BOX', 'bbox needs west,south,east,north.');
|
||||
const [west, south, east, north] = parts;
|
||||
if (
|
||||
Math.abs(west) > 180 ||
|
||||
Math.abs(east) > 180 ||
|
||||
south < -90 ||
|
||||
north > 90 ||
|
||||
south >= north ||
|
||||
west === east
|
||||
)
|
||||
return refuse('BAD_BOX', 'bbox edges are out of range.');
|
||||
// A box crossing the antimeridian is searched as its two halves.
|
||||
const boxes =
|
||||
west < east
|
||||
? [[south, west, north, east]]
|
||||
: [
|
||||
[south, west, north, 180],
|
||||
[south, -180, north, east],
|
||||
];
|
||||
const deg2 = boxes.reduce((sum, [s, w, n, e]) => sum + (n - s) * (e - w), 0);
|
||||
const cap =
|
||||
mode === 'count'
|
||||
? OSM_FEATURE_LIMITS.maxCountDeg2
|
||||
: OSM_FEATURE_LIMITS.maxFetchDeg2;
|
||||
if (deg2 > cap)
|
||||
return refuse(
|
||||
'AREA_TOO_LARGE',
|
||||
mode === 'count'
|
||||
? 'That area is too large to search; narrow it down.'
|
||||
: 'That area is too large to list; count first or narrow it down.',
|
||||
);
|
||||
// An absent or empty limit is the default, not Number(null) = 0.
|
||||
const requested =
|
||||
request.limit === null ||
|
||||
request.limit === undefined ||
|
||||
String(request.limit).trim() === ''
|
||||
? NaN
|
||||
: Number(request.limit);
|
||||
const limit = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
OSM_FEATURE_LIMITS.maxLimit,
|
||||
Number.isFinite(requested) ? Math.trunc(requested) : 500,
|
||||
),
|
||||
);
|
||||
const selectors = [];
|
||||
for (const [key, value] of preset.tags)
|
||||
for (const [s, w, n, e] of boxes)
|
||||
selectors.push(
|
||||
`nwr[${quote(key)}=${quote(value)}](${s},${w},${n},${e});`,
|
||||
);
|
||||
const ql =
|
||||
`[out:json][timeout:${OSM_FEATURE_LIMITS.timeoutS}][maxsize:${OSM_FEATURE_LIMITS.maxsize}];` +
|
||||
`(${selectors.join('')});` +
|
||||
(mode === 'count' ? 'out count;' : `out center tags ${limit};`);
|
||||
return { ok: true, ql, mode, preset, limit, boxes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Overpass JSON → the compact answer the browser gets.
|
||||
* @param {object} payload Parsed Overpass JSON.
|
||||
* @param {{mode: string, preset: object, limit: number}} compiled
|
||||
*/
|
||||
export function summarizeOsmFeatures(payload, compiled) {
|
||||
if (!Array.isArray(payload?.elements))
|
||||
throw new TypeError('Malformed Overpass feature response');
|
||||
const elements = payload.elements;
|
||||
if (compiled.mode === 'count') {
|
||||
const tags = elements.find((el) => el?.type === 'count')?.tags;
|
||||
const rawTotal = tags?.total;
|
||||
const total = Number(rawTotal);
|
||||
if (
|
||||
!tags ||
|
||||
!Object.hasOwn(tags, 'total') ||
|
||||
!['string', 'number'].includes(typeof rawTotal) ||
|
||||
String(rawTotal).trim() === '' ||
|
||||
!Number.isSafeInteger(total) ||
|
||||
total < 0
|
||||
)
|
||||
throw new TypeError('Malformed Overpass count response');
|
||||
return {
|
||||
mode: 'count',
|
||||
preset: compiled.preset.id,
|
||||
count: total,
|
||||
nodes: Number(tags.nodes) || 0,
|
||||
ways: Number(tags.ways) || 0,
|
||||
relations: Number(tags.relations) || 0,
|
||||
};
|
||||
}
|
||||
const features = [];
|
||||
let unplaced = 0;
|
||||
for (const el of elements) {
|
||||
if (
|
||||
!['node', 'way', 'relation'].includes(el?.type) ||
|
||||
(!Number.isFinite(Number(el?.id)) && typeof el?.id !== 'string')
|
||||
) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const lat = Number.isFinite(el?.lat) ? el.lat : el?.center?.lat;
|
||||
const lon = Number.isFinite(el?.lon) ? el.lon : el?.center?.lon;
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const tags = {};
|
||||
for (const key of [...KEPT_TAGS, ...compiled.preset.tags.map(([k]) => k)])
|
||||
if (typeof el.tags?.[key] === 'string')
|
||||
tags[key] = el.tags[key].slice(0, 120);
|
||||
features.push({ id: `${el.type}/${el.id}`, lat, lon, tags });
|
||||
}
|
||||
// A genuinely empty `elements` array is authoritative. A non-empty answer
|
||||
// whose every row is unusable is a schema failure, not "zero places"; do
|
||||
// not let the route cache that as current state for a day.
|
||||
if (elements.length && features.length === 0)
|
||||
throw new TypeError('Overpass feature response has no usable rows');
|
||||
return {
|
||||
mode: 'features',
|
||||
preset: compiled.preset.id,
|
||||
count: features.length,
|
||||
features,
|
||||
truncated: elements.length >= compiled.limit,
|
||||
...(unplaced ? { unplaced } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import path from 'node:path';
|
||||
import { makeRateLimiter, clientNetworkKey } from '../common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from '../common/same-site.js';
|
||||
import { createBoundedCache } from '../common/boundedCache.js';
|
||||
import {
|
||||
compileOsmFeatureQuery,
|
||||
OSM_FEATURE_LIMITS,
|
||||
summarizeOsmFeatures,
|
||||
} from './osmFeatures.js';
|
||||
import {
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
} from './constants.js';
|
||||
import { fetchOverpassPayload, overpassPayloadIsData } from './transport.js';
|
||||
|
||||
/** Places of interest change slowly: answers are kept a day. */
|
||||
const OSM_FEATURES_TTL_MS = 86_400_000;
|
||||
/** Upstream searches at once, across all clients (one per client). */
|
||||
const OSM_MAX_RUNNING = 2;
|
||||
/** Cache-key version; nothing cached under another version is reused. */
|
||||
const CACHE_VERSION = 'v2';
|
||||
|
||||
/** Never simplify: these answers are points and counts, not drawings. */
|
||||
const keepBody = (body) => body;
|
||||
|
||||
const jsonError = (status, error, extra = {}) => ({
|
||||
status,
|
||||
body: JSON.stringify({ error, ...extra }),
|
||||
});
|
||||
|
||||
/**
|
||||
* One upstream Overpass query at a time, with identical requests sharing it
|
||||
* and answers in a bounded memory + disk cache.
|
||||
*/
|
||||
function createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache,
|
||||
maxResponseBytes,
|
||||
upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
maxRunning = OSM_MAX_RUNNING,
|
||||
}) {
|
||||
const inFlight = new Map();
|
||||
// One search per client, a few in all: one client can't hold the route.
|
||||
const runningBy = new Map();
|
||||
let running = 0;
|
||||
return async function answer(key, ql, summarize, signal = null, owner = '') {
|
||||
signal?.throwIfAborted?.();
|
||||
const held = await cache.get(key);
|
||||
signal?.throwIfAborted?.();
|
||||
if (typeof held === 'string')
|
||||
return { status: 200, body: held, cache: 'HIT' };
|
||||
let operation = inFlight.get(key);
|
||||
if (operation?.controller.signal.aborted) {
|
||||
inFlight.delete(key);
|
||||
operation = null;
|
||||
}
|
||||
if (!operation) {
|
||||
if (runningBy.get(owner) || running >= maxRunning)
|
||||
return {
|
||||
...jsonError(
|
||||
503,
|
||||
runningBy.get(owner)
|
||||
? 'Your previous map search is still running — try again shortly.'
|
||||
: 'Another map search is running — try again shortly.',
|
||||
),
|
||||
retryAfter: '3',
|
||||
};
|
||||
running += 1;
|
||||
runningBy.set(owner, (runningBy.get(owner) || 0) + 1);
|
||||
const controller = new AbortController();
|
||||
// The whole search, every configured endpoint in turn, gets one budget.
|
||||
const budget = AbortSignal.timeout(upstreamBudgetMs);
|
||||
operation = { controller, waiters: 0, settled: false, promise: null };
|
||||
const owned = operation;
|
||||
owned.promise = (async () => {
|
||||
try {
|
||||
const payload = await fetchPayload(
|
||||
`data=${encodeURIComponent(ql)}`,
|
||||
maxResponseBytes,
|
||||
{
|
||||
simplify: keepBody,
|
||||
timeoutMs: upstreamTimeoutMs,
|
||||
signal: AbortSignal.any([controller.signal, budget]),
|
||||
},
|
||||
);
|
||||
if (!overpassPayloadIsData(payload))
|
||||
return {
|
||||
...jsonError(
|
||||
payload?.rateLimited ? 429 : 503,
|
||||
'OpenStreetMap search is unavailable right now.',
|
||||
),
|
||||
retryAfter: '10',
|
||||
};
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(payload.body);
|
||||
} catch {
|
||||
return jsonError(
|
||||
502,
|
||||
'OpenStreetMap returned an unreadable answer.',
|
||||
);
|
||||
}
|
||||
const body = JSON.stringify(summarize(parsed));
|
||||
controller.signal.throwIfAborted();
|
||||
await cache.set(key, body);
|
||||
controller.signal.throwIfAborted();
|
||||
return { status: 200, body, cache: 'MISS' };
|
||||
} catch {
|
||||
return budget.aborted && !controller.signal.aborted
|
||||
? jsonError(504, 'OpenStreetMap took too long to answer.')
|
||||
: jsonError(502, 'OpenStreetMap search failed.');
|
||||
} finally {
|
||||
owned.settled = true;
|
||||
running -= 1;
|
||||
const left = (runningBy.get(owner) || 1) - 1;
|
||||
if (left > 0) runningBy.set(owner, left);
|
||||
else runningBy.delete(owner);
|
||||
if (inFlight.get(key) === owned) inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
inFlight.set(key, owned);
|
||||
}
|
||||
|
||||
operation.waiters += 1;
|
||||
let abort = null;
|
||||
const abandoned = signal
|
||||
? new Promise((_, reject) => {
|
||||
abort = () => {
|
||||
operation.waiters -= 1;
|
||||
if (operation.waiters === 0 && !operation.settled)
|
||||
operation.controller.abort();
|
||||
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
||||
};
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
if (signal.aborted) abort();
|
||||
})
|
||||
: null;
|
||||
try {
|
||||
return await (abandoned
|
||||
? Promise.race([operation.promise, abandoned])
|
||||
: operation.promise);
|
||||
} finally {
|
||||
if (signal && abort) signal.removeEventListener('abort', abort);
|
||||
if (!signal?.aborted) operation.waiters -= 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sender(res) {
|
||||
return ({ status, body, cache, retryAfter }) => {
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(cache ? { 'X-OSM-Cache': cache } : {}),
|
||||
...(retryAfter ? { 'Retry-After': retryAfter } : {}),
|
||||
});
|
||||
res.end(body);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Install `GET /api/osm/features?preset=&bbox=w,s,e,n[&mode=count][&limit=]`:
|
||||
* one curated preset inside a box, answered by the operator's Overpass
|
||||
* (`OVERPASS_UPSTREAMS`). With none configured the route answers
|
||||
* `OVERPASS_NOT_CONFIGURED` and fetches nothing; public Nominatim is never a
|
||||
* substitute for bulk place search.
|
||||
*
|
||||
* @param {{use: Function}} middlewares
|
||||
* @param {{fetchPayload?: Function, featuresCache?: object, cacheRoot?: string|null, configured?: () => boolean, upstreamTag?: () => string}} [deps]
|
||||
*/
|
||||
export function installOsmFeaturesRoute(
|
||||
middlewares,
|
||||
{
|
||||
fetchPayload = fetchOverpassPayload,
|
||||
configured = () => resolveOverpassUpstreams().length > 0,
|
||||
// Keys each answer to the instance that gave it, as the other Overpass
|
||||
// caches do, so repointing OVERPASS_UPSTREAMS never serves the old one's.
|
||||
upstreamTag = resolveOverpassUpstreamTag,
|
||||
upstreamBudgetMs = OSM_FEATURE_LIMITS.upstreamBudgetMs,
|
||||
cacheRoot = path.join(process.cwd(), '.gev-cache'),
|
||||
featuresCache = createBoundedCache({
|
||||
dir: cacheRoot ? path.join(cacheRoot, 'osm-features') : null,
|
||||
ttlMs: OSM_FEATURES_TTL_MS,
|
||||
maxMemoryEntries: 60,
|
||||
maxDiskEntries: 400,
|
||||
maxDiskBytes: 256 * 1024 * 1024,
|
||||
}),
|
||||
} = {},
|
||||
) {
|
||||
const limiter = makeRateLimiter({ windowMs: 60_000, max: 12, globalMax: 30 });
|
||||
const features = createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache: featuresCache,
|
||||
maxResponseBytes: OSM_FEATURE_LIMITS.maxResponseBytes,
|
||||
upstreamTimeoutMs: OSM_FEATURE_LIMITS.upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
});
|
||||
|
||||
middlewares.use('/api/osm/features', async (req, res) => {
|
||||
const send = sender(res);
|
||||
if (req.method !== 'GET') return send(jsonError(405, 'Method Not Allowed'));
|
||||
// A page elsewhere must not spend the operator's Overpass. The keyless
|
||||
// gate, as on /api/overpass: voice works behind a reverse proxy too.
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
if (!configured())
|
||||
return send(
|
||||
jsonError(503, 'Place search needs a configured Overpass server.', {
|
||||
code: 'OVERPASS_NOT_CONFIGURED',
|
||||
retryable: false,
|
||||
}),
|
||||
);
|
||||
const url = new URL(req.url || '', 'http://localhost');
|
||||
const compiled = compileOsmFeatureQuery({
|
||||
preset: url.searchParams.get('preset'),
|
||||
bbox: url.searchParams.get('bbox'),
|
||||
mode: url.searchParams.get('mode'),
|
||||
limit: url.searchParams.get('limit'),
|
||||
});
|
||||
if (!compiled.ok)
|
||||
return send(jsonError(400, compiled.error, { code: compiled.code }));
|
||||
// IPv6 peers by /64, as /api/overpass does: rotating addresses inside
|
||||
// one prefix earns neither a second rate bucket nor a second slot.
|
||||
const client = clientNetworkKey(req);
|
||||
if (!limiter(client))
|
||||
return send({
|
||||
...jsonError(429, 'Rate limit exceeded'),
|
||||
retryAfter: '10',
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
req.once?.('aborted', abort);
|
||||
res.once?.('close', abort);
|
||||
try {
|
||||
const answer = await features(
|
||||
`${CACHE_VERSION}|features|${upstreamTag()}|${compiled.ql}`,
|
||||
compiled.ql,
|
||||
(parsed) => summarizeOsmFeatures(parsed, compiled),
|
||||
controller.signal,
|
||||
client,
|
||||
);
|
||||
if (!controller.signal.aborted) send(answer);
|
||||
} catch {
|
||||
if (!controller.signal.aborted)
|
||||
send(jsonError(502, 'OpenStreetMap search failed.'));
|
||||
} finally {
|
||||
req.removeListener?.('aborted', abort);
|
||||
res.removeListener?.('close', abort);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -122,12 +122,14 @@ function sanitizeOverpassBody(rawBody) {
|
||||
const boundedSets = new Set();
|
||||
for (let stmt of withoutTags.split(';')) {
|
||||
stmt = stmt.trim();
|
||||
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
|
||||
// The settings block (`[out:json][timeout:25]`) was blanked together with
|
||||
// the tag filters, so it arrives here as an empty statement.
|
||||
if (!stmt || /^out\b/.test(stmt)) continue;
|
||||
|
||||
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
|
||||
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
|
||||
// misread as a spatial bound. Bounds live in (...) / function calls / set
|
||||
// refs, never inside [...], so the probe loses nothing real.
|
||||
// Strip output-set assignments (NOT input bounds). With the tag filters
|
||||
// already gone, a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot
|
||||
// be misread as a spatial bound; bounds live in (...) / function calls /
|
||||
// set refs, never inside [...], so nothing real was lost.
|
||||
const outSets = [];
|
||||
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
outSets.push(name);
|
||||
@@ -166,8 +168,10 @@ function sanitizeOverpassBody(rawBody) {
|
||||
if (bounded) for (const name of outSets) boundedSets.add(name);
|
||||
}
|
||||
|
||||
// Tolerate whitespace around the directive so `[ timeout : N ]` is clamped
|
||||
// rather than passed through as written.
|
||||
const clamped = data.replace(
|
||||
/\[timeout:\s*(\d+)\s*\]/gi,
|
||||
/\[\s*timeout\s*:\s*(\d+)\s*\]/gi,
|
||||
(_, n) =>
|
||||
`[timeout:${Math.min(Number(n) || OVERPASS_MAX_QL_TIMEOUT, OVERPASS_MAX_QL_TIMEOUT)}]`,
|
||||
);
|
||||
|
||||
@@ -3,10 +3,24 @@ import assert from 'node:assert/strict';
|
||||
import { Readable } from 'node:stream';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { overpassProxy } from '../overpass.js';
|
||||
import { _overpassCache } from './cache.js';
|
||||
import { sanitizeOverpassBody } from './query.js';
|
||||
|
||||
const form = (query) => `data=${encodeURIComponent(query)}`;
|
||||
|
||||
// The limiters are module-level, so every test that spends admission mocks a
|
||||
// clock a full day past the previous one in execution order: no test can see
|
||||
// another's window, whatever order (or randomization) they run in. A new
|
||||
// test that reaches admission must call it too: earlier tests leave
|
||||
// timestamps days ahead of the real clock, and those still count.
|
||||
let clockBase = Date.now() + 86_400_000;
|
||||
function freshClock(t) {
|
||||
clockBase += 86_400_000;
|
||||
const clock = { now: clockBase };
|
||||
t.mock.method(Date, 'now', () => clock.now);
|
||||
return clock;
|
||||
}
|
||||
|
||||
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
|
||||
for (const query of [
|
||||
'node[',
|
||||
@@ -37,6 +51,15 @@ test('bounded queries preserve quoted brackets, comments, multiple tags and time
|
||||
query.replace('[timeout:999]', '[timeout:30]'),
|
||||
);
|
||||
}
|
||||
// Whitespace inside the directive is clamped too, not passed through.
|
||||
const spaced = sanitizeOverpassBody(
|
||||
form('[out:json][ timeout : 999 ];node(around:20,1,1);out;'),
|
||||
);
|
||||
assert.equal(spaced.ok, true);
|
||||
assert.equal(
|
||||
new URLSearchParams(spaced.body).get('data'),
|
||||
'[out:json][timeout:30];node(around:20,1,1);out;',
|
||||
);
|
||||
});
|
||||
|
||||
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
|
||||
@@ -126,7 +149,13 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
{ 'sec-fetch-site': 'cross-site' },
|
||||
{ 'sec-fetch-site': 'same-site' },
|
||||
{ origin: 'null' },
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
// Forwarding headers do not launder a cross-site request.
|
||||
{ 'x-forwarded-for': '10.0.0.9', 'sec-fetch-site': 'cross-site' },
|
||||
{
|
||||
'x-forwarded-proto': 'https',
|
||||
origin: 'http://localhost:4173',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
},
|
||||
]) {
|
||||
const response = await call(route, { headers, forbidRead: true });
|
||||
assert.equal(response.statusCode, 403);
|
||||
@@ -136,12 +165,24 @@ for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
});
|
||||
}
|
||||
|
||||
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
|
||||
test('same-origin, LAN, proxied and non-browser requests still reach validation; GET status stays available', async (t) => {
|
||||
freshClock(t);
|
||||
const route = handler();
|
||||
for (const headers of [
|
||||
{},
|
||||
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
|
||||
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
|
||||
// The README's reverse-proxy deployment: Overpass is keyless, so the
|
||||
// forwarding headers the proxy adds are admitted (as for terrain heights).
|
||||
{
|
||||
host: 'globe.example.lan',
|
||||
origin: 'https://globe.example.lan',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'x-forwarded-for': '203.0.113.7',
|
||||
'x-forwarded-proto': 'https',
|
||||
'x-forwarded-host': 'globe.example.lan',
|
||||
},
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
]) {
|
||||
const response = await call(route, { headers });
|
||||
assert.equal(response.statusCode, 400);
|
||||
@@ -158,8 +199,7 @@ test('same-origin, LAN and non-browser requests still reach validation; GET stat
|
||||
});
|
||||
|
||||
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
|
||||
let now = Date.now() + 120000;
|
||||
t.mock.method(Date, 'now', () => now);
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
@@ -173,6 +213,216 @@ test('invalid requests spend admission quota before parsing, including on an unc
|
||||
assert.equal(denied.reads, 0);
|
||||
assert.equal(denied.headers['retry-after'], '60');
|
||||
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
|
||||
now += 60001;
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('the 900/min global window spans non-loopback peers only; loopback keeps its own window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2', '10.0.0.3']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.4', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'global window is full for LAN peers');
|
||||
assert.equal(starved.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const owner = await call(route, { peer });
|
||||
assert.equal(owner.statusCode, 400, `${peer} is admitted past the global`);
|
||||
assert.equal(owner.reads, 1);
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.0.0.4' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('IPv6 peers share a /64 admission bucket and IPv4-mapped peers fold to IPv4', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: `2001:db8:1:2::${(i % 200) + 1}` }))
|
||||
.statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const rotated = await call(route, {
|
||||
peer: '2001:db8:1:2:ffff:ffff:ffff:9',
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(rotated.statusCode, 429, 'a rotating host stays in one bucket');
|
||||
assert.equal(
|
||||
(await call(route, { peer: '2001:db8:1:3::1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer: '::ffff:10.1.1.1' })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const unmapped = await call(route, { peer: '10.1.1.1', forbidRead: true });
|
||||
assert.equal(unmapped.statusCode, 429, 'mapped and bare IPv4 share a bucket');
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, { peer: '10.1.1.1' })).statusCode, 400);
|
||||
});
|
||||
|
||||
test('cache hits spend admission budget but not the 90/min upstream budget', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
let fetches = 0;
|
||||
t.mock.method(globalThis, 'fetch', async () => {
|
||||
fetches++;
|
||||
// Not data: nothing is cached or written to disk by this test.
|
||||
return new Response('upstream unavailable', { status: 503 });
|
||||
});
|
||||
const hit = 'node(around:20,1,1);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
const peer = '10.7.7.7';
|
||||
// More cache hits than the upstream limiter would allow per peer.
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const response = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(response.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
assert.equal(fetches, 0);
|
||||
const miss = await call(route, {
|
||||
peer,
|
||||
body: form('node(around:20,2,2);out;'),
|
||||
});
|
||||
assert.equal(fetches, 1, 'the first miss still reaches upstream');
|
||||
assert.notEqual(miss.statusCode, 429);
|
||||
for (let i = 0; i < 199; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { peer, body: form(hit) })).statusCode,
|
||||
200,
|
||||
);
|
||||
}
|
||||
const exhausted = await call(route, {
|
||||
peer,
|
||||
body: form(hit),
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(exhausted.statusCode, 429, 'hits spent the admission window');
|
||||
assert.equal(exhausted.reads, 0);
|
||||
});
|
||||
|
||||
test('a local reverse proxy cannot spend the direct-loopback reserve; owner cache hits survive it', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
const hit = 'node(around:20,3,3);out;';
|
||||
const hitKey = `data=${encodeURIComponent(hit)}`;
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
_overpassCache.delete(hitKey);
|
||||
});
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
_overpassCache.set(hitKey, {
|
||||
status: 200,
|
||||
body: '{"elements":[]}',
|
||||
contentType: 'application/json',
|
||||
endpoint: 'https://overpass.example/api/interpreter',
|
||||
cachedAt: clock.now,
|
||||
});
|
||||
const route = handler();
|
||||
// The proxy connects over loopback; a rotating forwarded client address
|
||||
// must not mint fresh buckets, so all 300 land in one socket-keyed bucket.
|
||||
for (const proxyPeer of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
|
||||
const proxied = (i) => ({
|
||||
peer: proxyPeer,
|
||||
headers: { 'x-forwarded-for': `203.0.113.${i % 250}` },
|
||||
body: form(hit),
|
||||
});
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, proxied(i))).statusCode, 200, proxyPeer);
|
||||
}
|
||||
const refused = await call(route, { ...proxied(300), forbidRead: true });
|
||||
assert.equal(refused.statusCode, 429, `${proxyPeer} proxy bucket is full`);
|
||||
assert.equal(refused.reads, 0);
|
||||
for (const peer of ['127.0.0.1', '::1']) {
|
||||
const owner = await call(route, { peer, body: form(hit) });
|
||||
assert.equal(owner.statusCode, 200, `direct ${peer} is still admitted`);
|
||||
assert.equal(owner.headers['x-overpass-cache'], 'HIT');
|
||||
}
|
||||
clock.now += 60_001;
|
||||
assert.equal((await call(route, proxied(0))).statusCode, 200);
|
||||
}
|
||||
});
|
||||
|
||||
test('proxied loopback traffic draws on the shared remote global window', async (t) => {
|
||||
const clock = freshClock(t);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
for (const peer of ['10.0.0.1', '10.0.0.2']) {
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400, peer);
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal(
|
||||
(await call(route, { headers: { via: '1.1 proxy' } })).statusCode,
|
||||
400,
|
||||
);
|
||||
}
|
||||
const starved = await call(route, { peer: '10.0.0.3', forbidRead: true });
|
||||
assert.equal(starved.statusCode, 429, 'proxied requests counted globally');
|
||||
assert.equal((await call(route, {})).statusCode, 400, 'owner unaffected');
|
||||
});
|
||||
|
||||
test("a local proxy has its own upstream bucket, so its misses cannot spend the owner's", async (t) => {
|
||||
freshClock(t);
|
||||
const prior = process.env.OVERPASS_UPSTREAMS;
|
||||
process.env.OVERPASS_UPSTREAMS = 'https://overpass.example/api/interpreter';
|
||||
t.after(() => {
|
||||
if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS;
|
||||
else process.env.OVERPASS_UPSTREAMS = prior;
|
||||
});
|
||||
// Not data: nothing is cached, so every request below is a miss.
|
||||
t.mock.method(
|
||||
globalThis,
|
||||
'fetch',
|
||||
async () => new Response('upstream unavailable', { status: 503 }),
|
||||
);
|
||||
const route = handler();
|
||||
const miss = form('node(around:20,4,4);out;');
|
||||
// The upstream limiter's own refusal is the only miss without this header.
|
||||
const passedLimiter = (response) =>
|
||||
response.headers['x-overpass-cache'] !== undefined;
|
||||
const proxied = { peer: '127.0.0.1', headers: { via: '1.1 proxy' } };
|
||||
for (let i = 0; i < 90; i++) {
|
||||
const response = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(passedLimiter(response), true, `proxied miss ${i}`);
|
||||
}
|
||||
const spent = await call(route, { ...proxied, body: miss });
|
||||
assert.equal(spent.statusCode, 429);
|
||||
assert.equal(passedLimiter(spent), false, 'the proxy bucket is spent');
|
||||
const owner = await call(route, { peer: '127.0.0.1', body: miss });
|
||||
assert.equal(passedLimiter(owner), true, 'the owner keeps its own bucket');
|
||||
});
|
||||
@@ -124,6 +124,8 @@ async function fetchOverpassPayload(
|
||||
readBody = readResponseTextCapped,
|
||||
simplify = simplifyOverpassPayloadBody,
|
||||
now = Date.now,
|
||||
timeoutMs = OVERPASS_TIMEOUT_MS,
|
||||
signal = null,
|
||||
} = {},
|
||||
) {
|
||||
if (!endpoints.length) return overpassNotConfigured();
|
||||
@@ -135,7 +137,10 @@ async function fetchOverpassPayload(
|
||||
continue;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const requestSignal = signal
|
||||
? AbortSignal.any([signal, controller.signal])
|
||||
: controller.signal;
|
||||
try {
|
||||
const requestUrl = new URL(endpoint);
|
||||
const authorization =
|
||||
@@ -156,7 +161,7 @@ async function fetchOverpassPayload(
|
||||
...(authorization ? { Authorization: authorization } : {}),
|
||||
},
|
||||
body,
|
||||
signal: controller.signal,
|
||||
signal: requestSignal,
|
||||
});
|
||||
const responseBody = await readBody(upstream, maxResponseBytes);
|
||||
const parsed = parseJson(responseBody);
|
||||
@@ -207,7 +212,12 @@ async function fetchOverpassPayload(
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
rateLimited: false,
|
||||
};
|
||||
} catch {
|
||||
} catch (error) {
|
||||
// A route whose last requester disconnected owns this cancellation. It
|
||||
// is not evidence that the configured upstream is unhealthy, so do not
|
||||
// put the endpoint on cooldown or translate the abort into a response
|
||||
// that a now-gone client could cache.
|
||||
if (signal?.aborted) throw signal.reason ?? error;
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const delay = retryDelay(null, failures - 1, now());
|
||||
cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
|
||||
|
||||
@@ -7,9 +7,13 @@ export {
|
||||
publicRadioHttpsUrl,
|
||||
} from './radio/stations.js';
|
||||
export function radioBrowserProxy() {
|
||||
const middleware = createRadioProxyMiddleware();
|
||||
let middleware;
|
||||
const initialize = () =>
|
||||
(middleware ||= createRadioProxyMiddleware({
|
||||
priorityCountries: process.env.RADIO_PRIORITY_COUNTRIES || '',
|
||||
}));
|
||||
const install = (server) => {
|
||||
server.middlewares.use('/api/radio', middleware);
|
||||
server.middlewares.use('/api/radio', initialize());
|
||||
};
|
||||
return {
|
||||
name: 'radio-browser-proxy',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js';
|
||||
import { lookup as lookupDns } from 'node:dns/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
@@ -27,6 +28,34 @@ import {
|
||||
RADIO_FALLBACK_MIRRORS,
|
||||
RADIO_REJECTED_STATION_COUNT_MAX,
|
||||
} from './constants.js';
|
||||
const PRIORITY_COUNTRY_LIMIT = 8;
|
||||
const COUNTRY_QUERY_LIMIT = 200;
|
||||
const COUNTRY_RESERVED_STATIONS = 40;
|
||||
const REFRESH_RETRY_MS = 60_000;
|
||||
const MAX_MIRROR_ATTEMPTS = 3;
|
||||
|
||||
/** Validate an operator-owned list; malformed configuration must not silently widen queries. */
|
||||
export function parseRadioPriorityCountries(value = '') {
|
||||
const entries = Array.isArray(value) ? value : String(value).split(',');
|
||||
const countries = [];
|
||||
for (const entry of entries) {
|
||||
const input = String(entry).trim().toUpperCase();
|
||||
if (!input) continue;
|
||||
const normalized = normalizeRadioCountryInput(input);
|
||||
if (!/^[A-Z]{2}$/.test(input) || !normalized.valid)
|
||||
throw new TypeError(
|
||||
'RADIO_PRIORITY_COUNTRIES requires ISO alpha-2 country codes',
|
||||
);
|
||||
const code = normalized.code;
|
||||
if (!countries.includes(code)) countries.push(code);
|
||||
}
|
||||
if (countries.length > PRIORITY_COUNTRY_LIMIT)
|
||||
throw new RangeError(
|
||||
'RADIO_PRIORITY_COUNTRIES supports at most eight countries',
|
||||
);
|
||||
return countries;
|
||||
}
|
||||
|
||||
async function mapRadioConcurrent(values, concurrency, mapper) {
|
||||
const results = new Array(values.length);
|
||||
let cursor = 0;
|
||||
@@ -49,7 +78,9 @@ export function createRadioProxyMiddleware({
|
||||
fetchImpl = null,
|
||||
lookupImpl = lookupDns,
|
||||
now = Date.now,
|
||||
priorityCountries = [],
|
||||
} = {}) {
|
||||
const countries = parseRadioPriorityCountries(priorityCountries);
|
||||
let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 };
|
||||
let mirrorPromise = null;
|
||||
let catalogCache = null;
|
||||
@@ -61,6 +92,9 @@ export function createRadioProxyMiddleware({
|
||||
const catalogInstance = randomUUID();
|
||||
let servedStationIds = new Set();
|
||||
let refreshPromise = null;
|
||||
let retryAfter = 0;
|
||||
let partialCatalog = null;
|
||||
let refreshError = null;
|
||||
|
||||
async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) {
|
||||
const destination = radioProxyDestination(url);
|
||||
@@ -139,7 +173,7 @@ export function createRadioProxyMiddleware({
|
||||
|
||||
async function fetchPath(pathname) {
|
||||
let lastError = null;
|
||||
for (const origin of await mirrors()) {
|
||||
for (const origin of (await mirrors()).slice(0, MAX_MIRROR_ATTEMPTS)) {
|
||||
try {
|
||||
return await fetchJson(`${origin}${pathname}`);
|
||||
} catch (error) {
|
||||
@@ -160,20 +194,27 @@ export function createRadioProxyMiddleware({
|
||||
'aviation',
|
||||
'marine',
|
||||
'traffic',
|
||||
];
|
||||
].map((tag) => ({ tag }));
|
||||
const baseQueryCount = queries.length;
|
||||
queries.push(...countries.map((country) => ({ country })));
|
||||
const outcomes = await mapRadioConcurrent(
|
||||
queries,
|
||||
3,
|
||||
async (tag, index) => {
|
||||
async ({ tag, country }, index) => {
|
||||
const params = new URLSearchParams({
|
||||
has_geo_info: 'true',
|
||||
is_https: 'true',
|
||||
hidebroken: 'true',
|
||||
order: 'clickcount',
|
||||
reverse: 'true',
|
||||
limit: index === 0 ? '1800' : '220',
|
||||
limit: country
|
||||
? String(COUNTRY_QUERY_LIMIT)
|
||||
: index === 0
|
||||
? '1800'
|
||||
: '220',
|
||||
});
|
||||
if (tag) params.set('tag', tag);
|
||||
if (country) params.set('countrycode', country);
|
||||
try {
|
||||
const rows = await fetchPath(`/json/stations/search?${params}`);
|
||||
if (!Array.isArray(rows))
|
||||
@@ -195,7 +236,12 @@ export function createRadioProxyMiddleware({
|
||||
const rejectedRows = rows.length - structuralRows.length;
|
||||
const stations = structuralRows
|
||||
.map(normalizeRadioBrowserStation)
|
||||
.filter(Boolean);
|
||||
.filter(
|
||||
(station) =>
|
||||
station && (!country || station.countryCode === country),
|
||||
);
|
||||
if (country && rows.length > COUNTRY_QUERY_LIMIT)
|
||||
throw new Error('Radio Browser exceeded the country query limit');
|
||||
const requestedTag = normalizeRadioTag(tag);
|
||||
const requestedTagCovered =
|
||||
!requestedTag ||
|
||||
@@ -207,10 +253,12 @@ export function createRadioProxyMiddleware({
|
||||
),
|
||||
);
|
||||
return {
|
||||
// Query coverage is based on accepted rows, not merely a payload that
|
||||
// happens to match the upstream schema. Specialist responses must
|
||||
// also contain an accepted station tagged for the requested category.
|
||||
succeeded: stations.length > 0 && requestedTagCovered,
|
||||
// A valid country response can contain no playable stations. Country
|
||||
// coverage is advisory; only broad/specialist queries gate health.
|
||||
// Specialist responses need an accepted station with matching tags.
|
||||
succeeded: country
|
||||
? rows.length === 0 || structuralRows.length > 0
|
||||
: stations.length > 0 && requestedTagCovered,
|
||||
stations,
|
||||
rejectedRows,
|
||||
};
|
||||
@@ -233,9 +281,24 @@ export function createRadioProxyMiddleware({
|
||||
seen.add(station.id);
|
||||
selected.push(station);
|
||||
};
|
||||
// Reserve country slots first; low-click regional stations cannot lose all
|
||||
// representation to the global charts. Keep room for all eight specialist
|
||||
// seeds (8 * 45) and global fill even at the eight-country limit (8 * 40).
|
||||
for (const rows of resultSets.slice(baseQueryCount)) {
|
||||
const ranked = [...rows].sort(
|
||||
(a, b) => b.clickCount - a.clickCount || a.id.localeCompare(b.id),
|
||||
);
|
||||
let added = 0;
|
||||
for (const station of ranked) {
|
||||
if (seen.has(station.id)) continue;
|
||||
take(station);
|
||||
if (++added >= COUNTRY_RESERVED_STATIONS) break;
|
||||
}
|
||||
}
|
||||
// Seed specialist station-tag queries before popularity fill so operational
|
||||
// categories remain represented even when global click charts skew musical.
|
||||
for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take);
|
||||
for (const rows of resultSets.slice(1, baseQueryCount))
|
||||
rows.slice(0, 45).forEach(take);
|
||||
resultSets
|
||||
.flat()
|
||||
.sort(
|
||||
@@ -243,9 +306,9 @@ export function createRadioProxyMiddleware({
|
||||
)
|
||||
.forEach(take);
|
||||
const timestamp = now();
|
||||
const successfulQueries = outcomes.filter(
|
||||
(outcome) => outcome.succeeded,
|
||||
).length;
|
||||
const successfulQueries = outcomes
|
||||
.slice(0, baseQueryCount)
|
||||
.filter((outcome) => outcome.succeeded).length;
|
||||
const broadQueryHealthy =
|
||||
outcomes[0].succeeded && outcomes[0].stations.length > 0;
|
||||
const healthReasons = [];
|
||||
@@ -261,7 +324,18 @@ export function createRadioProxyMiddleware({
|
||||
);
|
||||
const coverage = {
|
||||
successfulQueries,
|
||||
totalQueries: queries.length,
|
||||
totalQueries: baseQueryCount,
|
||||
...(countries.length
|
||||
? {
|
||||
priorityCountries: countries.map((country, index) => ({
|
||||
countryCode: country,
|
||||
succeeded: outcomes[baseQueryCount + index].succeeded,
|
||||
stationCount: selected.filter(
|
||||
(station) => station.countryCode === country,
|
||||
).length,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
stationCount: selected.length,
|
||||
healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS,
|
||||
};
|
||||
@@ -315,7 +389,24 @@ export function createRadioProxyMiddleware({
|
||||
return { ...catalogCache, stale: false };
|
||||
}
|
||||
if (!refreshPromise) {
|
||||
refreshPromise = refreshCatalog().finally(() => {
|
||||
refreshPromise = (async () => {
|
||||
if (now() < retryAfter) {
|
||||
if (refreshError) throw refreshError;
|
||||
return partialCatalog;
|
||||
}
|
||||
try {
|
||||
const catalog = await refreshCatalog();
|
||||
refreshError = null;
|
||||
partialCatalog = catalog.degraded ? catalog : null;
|
||||
retryAfter = catalog.degraded ? now() + REFRESH_RETRY_MS : 0;
|
||||
return catalog;
|
||||
} catch (error) {
|
||||
refreshError = error;
|
||||
partialCatalog = null;
|
||||
retryAfter = now() + REFRESH_RETRY_MS;
|
||||
throw error;
|
||||
}
|
||||
})().finally(() => {
|
||||
refreshPromise = null;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -204,6 +204,97 @@ const ADMIN_EVIDENCE = Object.freeze({
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Nominatim address types whose meaning is stable enough for the area consumer's
|
||||
* country/admin1/admin2 contract. `region`, `district` and numeric admin levels
|
||||
* vary by country, so they remain unverified instead of being guessed.
|
||||
*/
|
||||
const VERIFIED_ADMIN_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
state: 'admin1',
|
||||
province: 'admin1',
|
||||
county: 'admin2',
|
||||
state_district: 'admin2',
|
||||
});
|
||||
|
||||
const EXPLICIT_NAME_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
nation: 'country',
|
||||
province: 'admin1',
|
||||
state: 'admin1',
|
||||
region: 'admin1',
|
||||
oblast: 'admin1',
|
||||
prefecture: 'admin1',
|
||||
canton: 'admin1',
|
||||
governorate: 'admin1',
|
||||
county: 'admin2',
|
||||
district: 'admin2',
|
||||
department: 'admin2',
|
||||
zone: 'retired',
|
||||
});
|
||||
|
||||
function explicitNameEvidence(value) {
|
||||
const normalized = String(value || '')
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
|
||||
.trim();
|
||||
const prefix = normalized.match(
|
||||
/^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/,
|
||||
)?.[1];
|
||||
const suffix = normalized.match(
|
||||
/\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/,
|
||||
)?.[1];
|
||||
const levels = new Set(
|
||||
[prefix, suffix]
|
||||
.filter(Boolean)
|
||||
.map((type) => EXPLICIT_NAME_LEVEL[type])
|
||||
.filter(Boolean),
|
||||
);
|
||||
if (levels.size > 1) return { contradictory: true, level: null };
|
||||
return { contradictory: false, level: levels.values().next().value || null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verified administrative identity carried by the selected result itself.
|
||||
* Never infer it from the request, desired kind, rank, or a separate geocoder
|
||||
* anchor. A retired name is preserved without a level so downstream policy can
|
||||
* reject it explicitly; every other contradiction stays wholly unverified.
|
||||
*/
|
||||
function selectedAdminIdentity(kind, row) {
|
||||
if (kind !== 'admin') return null;
|
||||
const adminLevel =
|
||||
VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()];
|
||||
const authoritativeAdminArea = String(row?.name || '').trim();
|
||||
if (!adminLevel || !authoritativeAdminArea) return null;
|
||||
// Validate the complete authoritative name. `adminArea` is truncated only
|
||||
// for display after every prefix/suffix signal has been considered.
|
||||
const { contradictory, level: nameLevel } = explicitNameEvidence(
|
||||
authoritativeAdminArea,
|
||||
);
|
||||
if (contradictory) return null;
|
||||
const adminArea = authoritativeAdminArea.slice(0, 160);
|
||||
if (nameLevel === 'retired') {
|
||||
// Nepal replaced its zones in 2015, but zones are current units elsewhere
|
||||
// (Ethiopia). Without a country the name stays retired.
|
||||
if (!zoneIsCurrentIn(row)) return { adminArea };
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
if (nameLevel && nameLevel !== adminLevel) return null;
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
|
||||
/** Whether the row names a country (last display_name part) other than Nepal. */
|
||||
function zoneIsCurrentIn(row) {
|
||||
const country = String(row?.display_name || '')
|
||||
.split(',')
|
||||
.slice(1)
|
||||
.pop()
|
||||
?.trim()
|
||||
.toLocaleLowerCase();
|
||||
return Boolean(country) && country !== 'nepal';
|
||||
}
|
||||
|
||||
/** Whether supplied administrative evidence contradicts the accepted identity. */
|
||||
function adminLevelFits(kind, row) {
|
||||
const evidence = ADMIN_EVIDENCE[kind];
|
||||
@@ -584,6 +675,7 @@ export function selectOutlineResult(rows, { kind, query }) {
|
||||
row.osm_type && row.osm_id != null
|
||||
? `${row.osm_type}/${row.osm_id}`
|
||||
: null,
|
||||
...(selectedAdminIdentity(kind, row) || {}),
|
||||
polygons,
|
||||
},
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user