mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 06:42:23 +00:00
Merge branch 'main' into upstream/route-outage-unavailable
This commit is contained in:
215 files changed
+31425
-1171
No files matched your search
@@ -28,6 +28,7 @@ import {
|
||||
loadNycSourcesFromOpenData,
|
||||
loadTorontoSourcesFromOpenData,
|
||||
loadFloridaSourcesFromOpenData,
|
||||
loadHoustonSourcesFromOpenData,
|
||||
loadA22SourcesFromOpenDataHub,
|
||||
} from './sources.js';
|
||||
|
||||
@@ -166,6 +167,11 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
|
||||
load: loadTaiwanSources,
|
||||
},
|
||||
{
|
||||
name: 'houston',
|
||||
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
|
||||
load: loadHoustonSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'a22',
|
||||
enabled: () => envEnabled('CCTV_A22_ENABLED'),
|
||||
|
||||
@@ -443,6 +443,37 @@ export const NYC_ANCHORS = [
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** Houston TranStar cameras: one keyless JSON list for the metro (~1,000
|
||||
* cameras with a numeric id). Frames are stills synthesized from that id on
|
||||
* the city's snapshot host — never taken from the payload. The catalog is
|
||||
* served as `application/javascript`; asking for `application/json` gets HTTP
|
||||
* 406, so the loader accepts any media type. */
|
||||
export const HOUSTON_CAMERAS_URL =
|
||||
'https://traffic.houstontranstar.org/data/layers/cctvSnapshots_json.js';
|
||||
/** The only origin Houston camera frames may come from. The catalog's `url`
|
||||
* is a `showWin(...)` viewer helper and is ignored; the registered path is
|
||||
* always this origin plus `/snapshots/cctv/<id>.jpg`. */
|
||||
export const HOUSTON_IMAGE_ORIGIN = 'https://www.houstontranstar.org/';
|
||||
export const HOUSTON_IMAGE_URL = (id) =>
|
||||
`${HOUSTON_IMAGE_ORIGIN}snapshots/cctv/${id}.jpg`;
|
||||
/** Numeric TranStar snapshot ids: 3–5 digits, no leading zero. Id `0` is a
|
||||
* different (non-snapshot) camera class in the same list. */
|
||||
export const HOUSTON_CAMERA_ID_RE = /^[1-9]\d{2,4}$/;
|
||||
export const DEFAULT_HOUSTON_MAX_SOURCES = 250;
|
||||
export const HOUSTON_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres. The TranStar payload carries none, and
|
||||
* Houston sits a few metres above sea level. */
|
||||
export const HOUSTON_GROUND_ELEVATION_M = 15;
|
||||
/** Downtown Houston (Main @ Texas): the primary prioritization anchor. */
|
||||
export const HOUSTON_DOWNTOWN = { lat: 29.7604, lon: -95.3698 };
|
||||
export const HOUSTON_ANCHORS = [
|
||||
{ lat: 29.7604, lon: -95.3698 }, // Downtown
|
||||
{ lat: 29.739, lon: -95.461 }, // Uptown / Galleria
|
||||
{ lat: 29.71, lon: -95.398 }, // Texas Medical Center
|
||||
{ lat: 29.9902, lon: -95.3368 }, // IAH
|
||||
{ lat: 29.3013, lon: -94.7977 }, // Galveston
|
||||
];
|
||||
|
||||
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
|
||||
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
|
||||
* own camera list with coordinates, one request for the whole motorway
|
||||
|
||||
@@ -410,6 +410,18 @@ export function isLikelyHongKongCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** Greater Houston (the Woodlands to Galveston, Katy to Baytown). Farther
|
||||
* TranStar partners (Dallas, Columbus) are out of this city pack. */
|
||||
export function isLikelyHoustonCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 29.2 &&
|
||||
lat <= 30.35 &&
|
||||
lon >= -96.05 &&
|
||||
lon <= -94.7
|
||||
);
|
||||
}
|
||||
|
||||
/** Mainland Norway, with slack for the coast and the Finnmark border. */
|
||||
export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
return (
|
||||
|
||||
@@ -127,6 +127,13 @@ import {
|
||||
A22_MAX_CATALOG_BYTES,
|
||||
A22_GROUND_ELEVATION_BY_KM,
|
||||
A22_DEFAULT_GROUND_ELEVATION_M,
|
||||
HOUSTON_CAMERAS_URL,
|
||||
HOUSTON_IMAGE_URL,
|
||||
HOUSTON_CAMERA_ID_RE,
|
||||
DEFAULT_HOUSTON_MAX_SOURCES,
|
||||
HOUSTON_MAX_CATALOG_BYTES,
|
||||
HOUSTON_GROUND_ELEVATION_M,
|
||||
HOUSTON_ANCHORS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -147,6 +154,7 @@ import {
|
||||
isLikelyCataloniaCoordinate,
|
||||
decodeNumericEntities,
|
||||
isLikelyNycCoordinate,
|
||||
isLikelyHoustonCoordinate,
|
||||
isLikelyHongKongCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
isLikelyTorontoCoordinate,
|
||||
@@ -3771,3 +3779,162 @@ export async function loadA22SourcesFromOpenDataHub() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one Houston TranStar camera: the interchange an operator
|
||||
* recognises ("10 EAST @ SAN JACINTO").
|
||||
*
|
||||
* @param {object} record - Raw TranStar camera row.
|
||||
* @param {string} cameraId - Derived stable id.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function houstonCameraName(record, cameraId) {
|
||||
const name = String(record?.location ?? '').trim();
|
||||
if (name) return name;
|
||||
const short = String(cameraId || '').replace(/^houston-/, '');
|
||||
return short ? `Houston Camera ${short}` : 'Houston Camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* TranStar `dir` (the camera's side of the freeway) -> the travel direction
|
||||
* of the nearest lanes, as TranStar's map script `translateCameraDirection`
|
||||
* reads it. Only these four values are mapped; anything else is unknown.
|
||||
*/
|
||||
const HOUSTON_TRAVEL_HEADING_BY_SIDE = Object.freeze({
|
||||
south: 90,
|
||||
north: 270,
|
||||
west: 180,
|
||||
east: 0,
|
||||
});
|
||||
|
||||
/**
|
||||
* One Houston TranStar camera row -> one catalog source, or null.
|
||||
*
|
||||
* Frame URLs are synthesized from a validated numeric id on the pinned
|
||||
* snapshot origin. The payload's `url` is a `showWin(...)` viewer helper
|
||||
* and is ignored so a catalog edit cannot steer the frame proxy off-host.
|
||||
* TranStar's `dir` is the side of the freeway the camera sits on, not the
|
||||
* way it faces: TranStar's own map labels it "Traffic closest to the camera
|
||||
* is traveling …" (South→Eastbound, North→Westbound, West→Southbound,
|
||||
* East→Northbound). That travel direction is only a low-confidence prior
|
||||
* for the view, so the heading stays estimated with the low-confidence
|
||||
* pose; `None`, blank or unknown values fall back to the shared id-hash
|
||||
* heading.
|
||||
*
|
||||
* @param {object} record - Raw TranStar camera row.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function houstonCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
|
||||
const lat = toFiniteNumber(record.lat);
|
||||
const lon = toFiniteNumber(record.lng);
|
||||
if (!isLikelyHoustonCoordinate(lat, lon)) return null;
|
||||
|
||||
const rawId = String(record.id ?? '').trim();
|
||||
if (!HOUSTON_CAMERA_ID_RE.test(rawId)) return null;
|
||||
const cameraId = `houston-${rawId}`;
|
||||
const imageUrl = HOUSTON_IMAGE_URL(rawId);
|
||||
const name = houstonCameraName(record, cameraId);
|
||||
const travelHeading =
|
||||
HOUSTON_TRAVEL_HEADING_BY_SIDE[
|
||||
String(record.dir ?? '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
];
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city: 'Houston',
|
||||
cityId: 'houston',
|
||||
provider: 'Houston TranStar',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: Number.isFinite(travelHeading)
|
||||
? travelHeading
|
||||
: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: HOUSTON_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'houston-transtar',
|
||||
license:
|
||||
'TranStar disclaimer — redistribution and external framing need TxDOT written permission',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Houston TranStar cameras from the keyless `cctvSnapshots_json.js`
|
||||
* catalog. Frames are stills on `www.houstontranstar.org`, registered from
|
||||
* a validated numeric id rather than from the payload URL. The catalog host
|
||||
* answers `Accept: application/json` with HTTP 406, so this request accepts
|
||||
* any media type.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadHoustonSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint =
|
||||
process.env.CCTV_HOUSTON_CAMERAS_URL || HOUSTON_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: '*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Houston catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Houston camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
resp,
|
||||
HOUSTON_MAX_CATALOG_BYTES,
|
||||
);
|
||||
const rows = Array.isArray(payload?.cameras) ? payload.cameras : [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = houstonCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_HOUSTON_MAX_SOURCES || DEFAULT_HOUSTON_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_HOUSTON_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, HOUSTON_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Houston camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Houston camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
/**
|
||||
* A bounded two-level cache for JSON answers: memory (entry count) and disk
|
||||
* (entry count, total bytes, age). Every memory insertion goes through one
|
||||
* function, so a disk hit is bounded like a fresh answer; disk writes are
|
||||
* followed by a prune that removes expired files first, then the oldest,
|
||||
* until both quotas hold.
|
||||
*
|
||||
* @param {object} options
|
||||
* @param {string|null} options.dir Disk directory; null keeps memory only.
|
||||
* @param {number} options.ttlMs Age past which an entry is not served.
|
||||
* @param {number} [options.maxMemoryEntries]
|
||||
* @param {number} [options.maxDiskEntries]
|
||||
* @param {number} [options.maxDiskBytes]
|
||||
* @param {() => number} [options.now]
|
||||
*/
|
||||
export function createBoundedCache({
|
||||
dir,
|
||||
ttlMs,
|
||||
maxMemoryEntries = 60,
|
||||
maxDiskEntries = 400,
|
||||
maxDiskBytes = 256 * 1024 * 1024,
|
||||
now = () => Date.now(),
|
||||
}) {
|
||||
const memory = new Map();
|
||||
let pruning = null;
|
||||
let prunePending = false;
|
||||
|
||||
const fileFor = (key) =>
|
||||
path.join(dir, `${createHash('sha1').update(key).digest('hex')}.json`);
|
||||
|
||||
function remember(key, entry) {
|
||||
memory.delete(key);
|
||||
memory.set(key, entry);
|
||||
while (memory.size > maxMemoryEntries)
|
||||
memory.delete(memory.keys().next().value);
|
||||
}
|
||||
|
||||
async function prune() {
|
||||
if (!dir) return;
|
||||
let names;
|
||||
try {
|
||||
names = (await fsp.readdir(dir)).filter((name) => name.endsWith('.json'));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const files = [];
|
||||
for (const name of names) {
|
||||
const file = path.join(dir, name);
|
||||
try {
|
||||
const stat = await fsp.stat(file);
|
||||
files.push({ file, size: stat.size, mtime: stat.mtimeMs });
|
||||
} catch {
|
||||
/* removed meanwhile */
|
||||
}
|
||||
}
|
||||
const cutoff = now() - ttlMs;
|
||||
const keep = [];
|
||||
for (const entry of files) {
|
||||
if (entry.mtime < cutoff) await fsp.rm(entry.file, { force: true });
|
||||
else keep.push(entry);
|
||||
}
|
||||
keep.sort((a, b) => a.mtime - b.mtime);
|
||||
let bytes = keep.reduce((sum, entry) => sum + entry.size, 0);
|
||||
while (
|
||||
keep.length &&
|
||||
(keep.length > maxDiskEntries || bytes > maxDiskBytes)
|
||||
) {
|
||||
const oldest = keep.shift();
|
||||
bytes -= oldest.size;
|
||||
await fsp.rm(oldest.file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Run one prune at a time; a request during a prune schedules one more. */
|
||||
function schedulePrune() {
|
||||
if (pruning) {
|
||||
prunePending = true;
|
||||
return pruning;
|
||||
}
|
||||
pruning = prune()
|
||||
.catch(() => {})
|
||||
.finally(() => {
|
||||
pruning = null;
|
||||
if (prunePending) {
|
||||
prunePending = false;
|
||||
schedulePrune();
|
||||
}
|
||||
});
|
||||
return pruning;
|
||||
}
|
||||
|
||||
return {
|
||||
/** The cached value, or undefined. A disk hit is re-inserted into memory. */
|
||||
async get(key) {
|
||||
const held = memory.get(key);
|
||||
if (held && now() - held.cachedAt <= ttlMs) return held.value;
|
||||
if (held) memory.delete(key);
|
||||
if (!dir) return undefined;
|
||||
try {
|
||||
const entry = JSON.parse(await fsp.readFile(fileFor(key), 'utf8'));
|
||||
if (!Number.isFinite(entry?.cachedAt) || now() - entry.cachedAt > ttlMs)
|
||||
return undefined;
|
||||
remember(key, entry);
|
||||
return entry.value;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
},
|
||||
/** Store a value in memory and, when a directory is set, on disk. */
|
||||
async set(key, value) {
|
||||
const entry = { value, cachedAt: now() };
|
||||
remember(key, entry);
|
||||
if (!dir) return;
|
||||
try {
|
||||
await fsp.mkdir(dir, { recursive: true });
|
||||
await fsp.writeFile(fileFor(key), JSON.stringify(entry));
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
// In the background: an answer never waits on a 400-file directory scan.
|
||||
// `prune()` returns the pending pass for callers that need it done.
|
||||
schedulePrune();
|
||||
},
|
||||
/** Run a prune and wait until no pass is running or queued. */
|
||||
async prune() {
|
||||
schedulePrune();
|
||||
while (pruning) await pruning;
|
||||
},
|
||||
get memorySize() {
|
||||
return memory.size;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import { readRequestBody } from './request.js';
|
||||
import {
|
||||
clientKey,
|
||||
makeCostRateLimiter,
|
||||
resolvePerMinuteCap,
|
||||
} from './rate-limit.js';
|
||||
import { waitForSignal } from '../../../src/sources/requestDeadline.js';
|
||||
import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
|
||||
import { isGeminiRequestOriginAllowed } from '../gemini/request.js';
|
||||
import { parseUsage } from './textTools.js';
|
||||
|
||||
const MAX_BYTES = 512 * 1024;
|
||||
const FAILURE = 'Could not connect. Try again.';
|
||||
// A rejected key is not retried upstream for this long, unless the key changes.
|
||||
const REJECTED_KEY_COOLDOWN_MS = 60_000;
|
||||
|
||||
// What an upstream failure says about the key or the account. Gemini answers
|
||||
// an invalid key with 400 and an ErrorInfo reason, not 401; OpenAI answers an
|
||||
// account without credit with 429 insufficient_quota, which waiting won't fix.
|
||||
// A 403 is the key's project, region or permissions, so it cools down too but
|
||||
// does not blame the key. Only small error bodies are read.
|
||||
async function upstreamRefusal(response, signal) {
|
||||
const { status } = response;
|
||||
if (status === 401) return 'API_KEY_REJECTED';
|
||||
if (status === 403) return 'ACCESS_DENIED';
|
||||
if (![400, 429].includes(status)) return null;
|
||||
let error;
|
||||
try {
|
||||
error = (await readResponseJsonCapped(response, 16 * 1024, signal))?.error;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
status === 400 &&
|
||||
Array.isArray(error?.details) &&
|
||||
error.details.some((detail) => detail?.reason === 'API_KEY_INVALID')
|
||||
)
|
||||
return 'API_KEY_REJECTED';
|
||||
if (status === 429 && error?.code === 'insufficient_quota')
|
||||
return 'QUOTA_EXCEEDED';
|
||||
return null;
|
||||
}
|
||||
|
||||
// Pass through only a delay in whole seconds, capped at an hour.
|
||||
function upstreamRetryAfter(response) {
|
||||
const value = String(response.headers?.get?.('retry-after') ?? '').trim();
|
||||
if (!/^\d{1,6}$/.test(value)) return null;
|
||||
return String(Math.min(Math.max(Number(value), 1), 3600));
|
||||
}
|
||||
|
||||
export function createTextChatHandler({
|
||||
provider,
|
||||
prepare,
|
||||
parse,
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey,
|
||||
resolveHost = () => process.env.HOST,
|
||||
resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
|
||||
resolveRateLimit = () =>
|
||||
process.env[`GEV_RATELIMIT_${provider.toUpperCase()}_CHAT_PER_MIN`],
|
||||
// An unset chat throttle follows the provider's own cost throttle.
|
||||
resolveProviderRateLimit = () => undefined,
|
||||
providerPerMin = 30,
|
||||
// Returns { error, code } when the route must refuse before any key is used.
|
||||
refuse = () => null,
|
||||
now = Date.now,
|
||||
timeoutMs = 60000,
|
||||
} = {}) {
|
||||
let limiter;
|
||||
let rejected = null;
|
||||
return async (req, res) => {
|
||||
if (req.aborted || res.destroyed || res.writableEnded) return;
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('X-Content-Type-Options', 'nosniff');
|
||||
const reply = (status, value, code) => {
|
||||
if (req.aborted || res.destroyed || res.writableEnded) return;
|
||||
res.statusCode = status;
|
||||
if (status >= 400) res.setHeader('Connection', 'close');
|
||||
res.end(
|
||||
JSON.stringify(
|
||||
typeof value === 'string'
|
||||
? { error: value, ...(code ? { code } : {}) }
|
||||
: value,
|
||||
),
|
||||
);
|
||||
};
|
||||
if (req.method !== 'POST') {
|
||||
res.setHeader('Allow', 'POST');
|
||||
return reply(405, 'Method not allowed');
|
||||
}
|
||||
const lifetime = new AbortController();
|
||||
const disconnect = () => {
|
||||
if (!res.writableEnded) lifetime.abort();
|
||||
};
|
||||
req.once('aborted', disconnect);
|
||||
res.once('close', disconnect);
|
||||
const timer = setTimeout(() => lifetime.abort(), timeoutMs);
|
||||
timer.unref?.();
|
||||
let response;
|
||||
let usage = null;
|
||||
try {
|
||||
if (
|
||||
!isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
|
||||
)
|
||||
return reply(403, FAILURE);
|
||||
const refusal = refuse();
|
||||
if (refusal) return reply(503, refusal.error, refusal.code);
|
||||
const apiKey = String(resolveApiKey() || '').trim();
|
||||
if (!apiKey)
|
||||
return reply(
|
||||
503,
|
||||
'Add an API key in Provider Settings',
|
||||
'API_KEY_MISSING',
|
||||
);
|
||||
if (rejected && (rejected.key !== apiKey || now() >= rejected.until))
|
||||
rejected = null;
|
||||
if (rejected) {
|
||||
res.setHeader(
|
||||
'Retry-After',
|
||||
String(Math.max(1, Math.ceil((rejected.until - now()) / 1000))),
|
||||
);
|
||||
return reply(502, FAILURE, rejected.code);
|
||||
}
|
||||
if (limiter === undefined)
|
||||
limiter = makeCostRateLimiter(
|
||||
resolveRateLimit(),
|
||||
resolvePerMinuteCap(resolveProviderRateLimit(), providerPerMin),
|
||||
);
|
||||
if (limiter && !limiter(clientKey(req))) {
|
||||
res.setHeader('Retry-After', '60');
|
||||
return reply(429, FAILURE, 'RATE_LIMITED');
|
||||
}
|
||||
if (
|
||||
!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')
|
||||
)
|
||||
return reply(415, FAILURE);
|
||||
if (Number(req.headers['content-length']) > MAX_BYTES)
|
||||
return reply(413, FAILURE);
|
||||
const raw = await readRequestBody(req, MAX_BYTES, lifetime.signal);
|
||||
let input;
|
||||
try {
|
||||
input = JSON.parse(raw);
|
||||
} catch {
|
||||
return reply(400, FAILURE);
|
||||
}
|
||||
const prepared = prepare(input);
|
||||
if (!prepared) return reply(400, FAILURE);
|
||||
if (prepared.errorStatus) return reply(prepared.errorStatus, FAILURE);
|
||||
const { endpoint, body, headers } = prepared;
|
||||
lifetime.signal.throwIfAborted();
|
||||
const acquisition = Promise.resolve(
|
||||
fetchImpl(endpoint, {
|
||||
method: 'POST',
|
||||
redirect: 'error',
|
||||
signal: lifetime.signal,
|
||||
headers: { 'Content-Type': 'application/json', ...headers(apiKey) },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
);
|
||||
void acquisition.then(
|
||||
(late) => {
|
||||
response = late;
|
||||
if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
|
||||
},
|
||||
() => {},
|
||||
);
|
||||
response = await waitForSignal(acquisition, lifetime.signal);
|
||||
lifetime.signal.throwIfAborted();
|
||||
if (response.redirected || (response.url && response.url !== endpoint))
|
||||
throw new Error('Unexpected destination');
|
||||
if (!response.ok) {
|
||||
const refusal = await upstreamRefusal(response, lifetime.signal);
|
||||
if (refusal) {
|
||||
void response.body?.cancel().catch(() => {});
|
||||
rejected = {
|
||||
key: apiKey,
|
||||
code: refusal,
|
||||
until: now() + REJECTED_KEY_COOLDOWN_MS,
|
||||
};
|
||||
return reply(502, FAILURE, refusal);
|
||||
}
|
||||
void response.body?.cancel().catch(() => {});
|
||||
if (![429, 503].includes(response.status)) return reply(502, FAILURE);
|
||||
const retryAfter = upstreamRetryAfter(response);
|
||||
if (retryAfter) res.setHeader('Retry-After', retryAfter);
|
||||
return reply(
|
||||
response.status,
|
||||
FAILURE,
|
||||
response.status === 429 ? 'RATE_LIMITED' : undefined,
|
||||
);
|
||||
}
|
||||
const data = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_BYTES,
|
||||
lifetime.signal,
|
||||
);
|
||||
lifetime.signal.throwIfAborted();
|
||||
usage = parseUsage(provider, data);
|
||||
const result = parse(data);
|
||||
if (!result || JSON.stringify(result).includes(apiKey))
|
||||
return reply(502, { error: FAILURE, usage });
|
||||
reply(200, { ...result, usage });
|
||||
} catch (error) {
|
||||
reply(
|
||||
error?.code === 'BODY_TOO_LARGE'
|
||||
? 413
|
||||
: lifetime.signal.aborted
|
||||
? 504
|
||||
: 502,
|
||||
{ error: FAILURE, usage },
|
||||
);
|
||||
} finally {
|
||||
if (lifetime.signal.aborted)
|
||||
void response?.body?.cancel().catch(() => {});
|
||||
clearTimeout(timer);
|
||||
req.off('aborted', disconnect);
|
||||
res.off('close', disconnect);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { realtimeDirectives } from '../openai/instructions.js';
|
||||
|
||||
// Voice wording in the shared directives and its written form. A test checks
|
||||
// each phrase still occurs in the directives, so rewording one there fails
|
||||
// rather than leaking spoken wording into chat.
|
||||
export const TEXT_WORDING = Object.freeze([
|
||||
['without speaking first', 'without writing a preamble'],
|
||||
['before speaking', 'before confirming'],
|
||||
['may first say ONE sentence', 'may first write ONE sentence'],
|
||||
["speak the result's say line", "write the result's say line"],
|
||||
['on the voice card: speak a caveat', 'in the tool result: include a caveat'],
|
||||
['as you talk', 'as you write'],
|
||||
['you can read aloud', 'you can report'],
|
||||
]);
|
||||
|
||||
// Shared-manual adaptation follows Darren Dignam's GEV COMMAND approach (#976).
|
||||
// Keep our Radio handoff and annotation outcome policies from the shared manual.
|
||||
// `options.overpass` matches the tools offered, as for voice.
|
||||
export function textInstructions(annotationGuidance, options) {
|
||||
return realtimeDirectives(annotationGuidance, options)
|
||||
.map((line, index) => {
|
||||
if (index === 0)
|
||||
return "You are GEV Chat, a concise text assistant for a Cesium geospatial app called God's Eye View.";
|
||||
if (index === 1)
|
||||
return 'Have a natural written conversation. This is text-only chat; no microphone or generated audio is used.';
|
||||
return TEXT_WORDING.reduce(
|
||||
(text, [spoken, written]) => text.replace(spoken, written),
|
||||
line,
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { validateValue } from '../../../src/tools/schema.js';
|
||||
|
||||
export function invalidCalls(calls, tools) {
|
||||
const schemas = new Map(tools.map((tool) => [tool.name, tool.parameters]));
|
||||
return calls.flatMap((call, index) => {
|
||||
const schema = schemas.get(call.name);
|
||||
const errors = !schema
|
||||
? ['Unknown tool']
|
||||
: call.parseError
|
||||
? ['Arguments must be valid JSON']
|
||||
: validateValue(schema, call.args === undefined ? {} : call.args);
|
||||
return errors.length
|
||||
? [{ index, error: errors.slice(0, 4).join('; ').slice(0, 1000) }]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
|
||||
// Missing usage stays unknown, never an invented zero-cost request.
|
||||
export function parseUsage(provider, data) {
|
||||
const raw = provider === 'gemini' ? data?.usageMetadata : data?.usage;
|
||||
const number = (value) =>
|
||||
Number.isSafeInteger(value) && value >= 0 ? value : null;
|
||||
const inputTokens = number(
|
||||
provider === 'gemini' ? raw?.promptTokenCount : raw?.prompt_tokens,
|
||||
);
|
||||
const candidateTokens = number(
|
||||
provider === 'gemini' ? raw?.candidatesTokenCount : raw?.completion_tokens,
|
||||
);
|
||||
const totalTokens = number(
|
||||
provider === 'gemini' ? raw?.totalTokenCount : raw?.total_tokens,
|
||||
);
|
||||
if (inputTokens === null || candidateTokens === null || totalTokens === null)
|
||||
return null;
|
||||
const reasoningTokens =
|
||||
number(
|
||||
provider === 'gemini'
|
||||
? raw?.thoughtsTokenCount
|
||||
: raw?.completion_tokens_details?.reasoning_tokens,
|
||||
) ?? 0;
|
||||
const outputTokens =
|
||||
candidateTokens + (provider === 'gemini' ? reasoningTokens : 0);
|
||||
return {
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens,
|
||||
reasoningTokens,
|
||||
cachedInputTokens:
|
||||
number(
|
||||
provider === 'gemini'
|
||||
? raw?.cachedContentTokenCount
|
||||
: raw?.prompt_tokens_details?.cached_tokens,
|
||||
) ?? 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { normalizeDroneAirspaceSnapshot } from '../../src/layers/droneAirspace/records.js';
|
||||
import { readResponseJsonCapped, coalesceProxyRequest } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
|
||||
// FAA Special Use Airspace (public, keyless). TYPE_CODE P (prohibited) and
|
||||
// R (restricted) only — the small, always-in-force slice of the national
|
||||
// SUA table. maxAllowableOffset trades ~100 m of boundary fidelity for a
|
||||
// payload small enough to refresh (~330 KB for the current P+R set).
|
||||
const API_URL =
|
||||
'https://services6.arcgis.com/ssFJjBXIUyZDrSYZ/arcgis/rest/services/' +
|
||||
'Special_Use_Airspace/FeatureServer/0/query?' +
|
||||
new URLSearchParams({
|
||||
where: "TYPE_CODE IN ('P','R')",
|
||||
outFields: [
|
||||
'NAME',
|
||||
'TYPE_CODE',
|
||||
'UPPER_VAL',
|
||||
'UPPER_UOM',
|
||||
'UPPER_CODE',
|
||||
'LOWER_VAL',
|
||||
'LOWER_UOM',
|
||||
'LOWER_CODE',
|
||||
'TIMESOFUSE',
|
||||
'CONT_AGENT',
|
||||
'CITY',
|
||||
'STATE',
|
||||
'OBJECTID',
|
||||
].join(','),
|
||||
maxAllowableOffset: '0.001',
|
||||
outSR: '4326',
|
||||
f: 'geojson',
|
||||
}).toString();
|
||||
|
||||
const MAX_PAGES = 5;
|
||||
const MIB = 1024 * 1024;
|
||||
const SNAPSHOT_TTL_MS = 3_600_000;
|
||||
/** Longest a failed refresh keeps serving the last snapshot, marked stale. */
|
||||
const STALE_MAX_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
const exceededTransferLimit = (payload) =>
|
||||
payload?.exceededTransferLimit === true ||
|
||||
payload?.properties?.exceededTransferLimit === true;
|
||||
|
||||
/** Fixed-origin, bounded FAA SUA routes for dev and preview. */
|
||||
export function droneAirspaceProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 60, globalMax: 1200 });
|
||||
|
||||
async function upstream(url, cap, timeout) {
|
||||
const signal = AbortSignal.timeout(timeout);
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
return readResponseJsonCapped(response, cap, signal);
|
||||
}
|
||||
|
||||
// A snapshot is accepted only once the service says it is complete. A page
|
||||
// limit reached while still flagged, a flagged page that adds nothing, or
|
||||
// no valid zone at all is a failed refresh, so the last good snapshot stays.
|
||||
async function fetchZones() {
|
||||
const features = [];
|
||||
let complete = false;
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const url =
|
||||
page === 0 ? API_URL : `${API_URL}&resultOffset=${features.length}`;
|
||||
const payload = await upstream(url, 8 * MIB, 30_000);
|
||||
if (!Array.isArray(payload?.features))
|
||||
throw new Error('invalid_snapshot');
|
||||
features.push(...payload.features);
|
||||
if (!exceededTransferLimit(payload)) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
if (!payload.features.length) break;
|
||||
}
|
||||
if (!complete) throw new Error('incomplete_snapshot');
|
||||
const rows = normalizeDroneAirspaceSnapshot({ features });
|
||||
if (!rows?.length) throw new Error('invalid_snapshot');
|
||||
return { fetchedAt: now(), rows };
|
||||
}
|
||||
|
||||
async function acquire(key, ttl, load) {
|
||||
const previous = cache.get(key);
|
||||
if (previous && now() - previous.savedAt < ttl)
|
||||
return { value: previous.value, stale: false };
|
||||
if (!inFlight.has(key) && inFlight.size >= 256)
|
||||
throw Object.assign(new Error('busy'), { status: 429 });
|
||||
try {
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
const value = await load();
|
||||
cache.set(key, { value, savedAt: now() });
|
||||
return value;
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
// A failed refresh serves the last snapshot, marked stale, for a day.
|
||||
if (previous && now() - previous.savedAt <= STALE_MAX_MS)
|
||||
return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
// Keyless and needed to render the layer: refuse cross-site browsers
|
||||
// before they spend a client's quota or start a refresh, but keep the
|
||||
// documented reverse-proxy deployment working.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
|
||||
try {
|
||||
const { value, stale } = await acquire(
|
||||
'zones',
|
||||
SNAPSHOT_TTL_MS,
|
||||
fetchZones,
|
||||
);
|
||||
json(200, stale ? { ...value, stale: true } : value, stale);
|
||||
} catch (error) {
|
||||
json(error.status === 429 ? 429 : 502, {
|
||||
error: 'drone_airspace_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'drone-airspace',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/drone-airspace', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/drone-airspace', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
import { createGeminiChatHandler } from './gemini/chat.js';
|
||||
import { createGeminiTokenHandler } from './gemini/realtime.js';
|
||||
|
||||
/** Mint constrained Live API credentials; the permanent key stays on Node. */
|
||||
function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
|
||||
function install(middlewares) {
|
||||
middlewares.use(
|
||||
'/api/gemini/chat',
|
||||
createGeminiChatHandler({ ...realtime, annotationGuidance }),
|
||||
);
|
||||
middlewares.use(
|
||||
'/api/gemini/token',
|
||||
createGeminiTokenHandler({ ...realtime, annotationGuidance }),
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
import { createTextChatHandler } from '../common/textChat.js';
|
||||
import { invalidCalls } from '../common/textTools.js';
|
||||
import { textInstructions } from '../common/textInstructions.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
import { normalizeGeminiModel } from './config.js';
|
||||
|
||||
// The Gemini token route's throttle default (GEV_RATELIMIT_GEMINI_PER_MIN).
|
||||
export const GEMINI_DEFAULT_PER_MIN = 30;
|
||||
|
||||
const object = (value) =>
|
||||
value && typeof value === 'object' && !Array.isArray(value);
|
||||
const keys = (value, allowed) =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
|
||||
// Only text and function exchanges may cross this route, never media or config.
|
||||
export function validChatContents(contents, names) {
|
||||
return (
|
||||
Array.isArray(contents) &&
|
||||
contents.length > 0 &&
|
||||
contents.length <= 96 &&
|
||||
contents.every(
|
||||
(content) =>
|
||||
object(content) &&
|
||||
keys(content, ['role', 'parts']) &&
|
||||
['user', 'model'].includes(content.role) &&
|
||||
Array.isArray(content.parts) &&
|
||||
content.parts.length > 0 &&
|
||||
content.parts.length <= 32 &&
|
||||
content.parts.every((part) => {
|
||||
if (
|
||||
!object(part) ||
|
||||
!keys(part, [
|
||||
'text',
|
||||
'functionCall',
|
||||
'functionResponse',
|
||||
'thoughtSignature',
|
||||
])
|
||||
)
|
||||
return false;
|
||||
if (
|
||||
part.thoughtSignature !== undefined &&
|
||||
(typeof part.thoughtSignature !== 'string' ||
|
||||
part.thoughtSignature.length > 32000 ||
|
||||
content.role !== 'model')
|
||||
)
|
||||
return false;
|
||||
if (Object.hasOwn(part, 'text'))
|
||||
return (
|
||||
keys(part, ['text', 'thoughtSignature']) &&
|
||||
typeof part.text === 'string'
|
||||
);
|
||||
const call = part.functionCall ?? part.functionResponse;
|
||||
if (
|
||||
!object(call) ||
|
||||
typeof call.name !== 'string' ||
|
||||
call.name.length > 128 ||
|
||||
(names && !names.has(call.name)) ||
|
||||
(call.id !== undefined &&
|
||||
(typeof call.id !== 'string' || call.id.length > 1024))
|
||||
)
|
||||
return false;
|
||||
if (part.functionCall)
|
||||
return (
|
||||
content.role === 'model' &&
|
||||
keys(part, ['functionCall', 'thoughtSignature']) &&
|
||||
keys(call, ['name', 'args', 'id']) &&
|
||||
(call.args === null || object(call.args ?? {}))
|
||||
);
|
||||
return (
|
||||
content.role === 'user' &&
|
||||
keys(part, ['functionResponse']) &&
|
||||
keys(call, ['name', 'response', 'id']) &&
|
||||
object(call.response)
|
||||
);
|
||||
}),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function createGeminiChatHandler({
|
||||
annotationGuidance,
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered, and the prompt names it, only with Overpass, as
|
||||
// for voice.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
resolveChatModel = () =>
|
||||
process.env.GEMINI_CHAT_MODEL || 'gemini-3.5-flash-lite',
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN,
|
||||
...options
|
||||
} = {}) {
|
||||
return createTextChatHandler({
|
||||
...options,
|
||||
provider: 'gemini',
|
||||
resolveApiKey,
|
||||
resolveProviderRateLimit,
|
||||
providerPerMin: GEMINI_DEFAULT_PER_MIN,
|
||||
prepare(input) {
|
||||
const model = normalizeGeminiModel(resolveChatModel());
|
||||
if (!model || /live|audio|tts/i.test(model)) return { errorStatus: 503 };
|
||||
if (
|
||||
!object(input) ||
|
||||
!keys(input, ['contents']) ||
|
||||
!validChatContents(input.contents, null)
|
||||
)
|
||||
return null;
|
||||
if (!pairedContents(input.contents)) return null;
|
||||
const overpass = overpassConfigured();
|
||||
return {
|
||||
endpoint: `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
|
||||
headers: (key) => ({ 'x-goog-api-key': key }),
|
||||
body: {
|
||||
contents: input.contents,
|
||||
systemInstruction: {
|
||||
parts: [
|
||||
{ text: textInstructions(annotationGuidance, { overpass }) },
|
||||
],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
parametersJsonSchema: parameters,
|
||||
}),
|
||||
),
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseModalities: ['TEXT'],
|
||||
maxOutputTokens: 4096,
|
||||
thinkingConfig: { thinkingLevel: 'LOW' },
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
parse(data) {
|
||||
const candidate = data?.candidates?.[0];
|
||||
const content = candidate?.content;
|
||||
if (
|
||||
!content ||
|
||||
!validChatContents([content], null) ||
|
||||
content.role !== 'model'
|
||||
)
|
||||
return null;
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
return {
|
||||
content,
|
||||
finishReason: candidate.finishReason,
|
||||
invalidCalls: invalidCalls(
|
||||
calls,
|
||||
offeredTools(tools, { overpass: overpassConfigured() }),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Never forward an orphan result or an unfinished batch in browser history.
|
||||
function pairedContents(contents) {
|
||||
let pending = [];
|
||||
for (const content of contents) {
|
||||
const calls = content.parts
|
||||
.filter((part) => part.functionCall)
|
||||
.map((part) => part.functionCall);
|
||||
const results = content.parts
|
||||
.filter((part) => part.functionResponse)
|
||||
.map((part) => part.functionResponse);
|
||||
if (pending.length) {
|
||||
if (
|
||||
calls.length ||
|
||||
results.length !== pending.length ||
|
||||
results.some(
|
||||
(result, i) =>
|
||||
result.name !== pending[i].name || result.id !== pending[i].id,
|
||||
)
|
||||
)
|
||||
return false;
|
||||
pending = [];
|
||||
} else if (results.length) return false;
|
||||
if (calls.length) pending = calls;
|
||||
}
|
||||
return !pending.length;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { realtimeInstructions } from '../openai/instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from '../openai/tools.js';
|
||||
|
||||
export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
|
||||
export const GEMINI_TOKEN_ENDPOINT =
|
||||
@@ -14,22 +14,26 @@ export function normalizeGeminiModel(value) {
|
||||
return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
|
||||
}
|
||||
|
||||
/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
|
||||
/**
|
||||
* REST BidiGenerateContentSetup fields, excluding its model resource name.
|
||||
* `overpass` says an operator configured Overpass, which `osm_query` needs.
|
||||
*/
|
||||
export function createGeminiLiveConfig(
|
||||
annotationGuidance,
|
||||
inputMode = 'open-mic',
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
if (!['open-mic', 'push-to-talk'].includes(inputMode))
|
||||
throw new Error('Invalid Gemini input mode');
|
||||
const config = {
|
||||
generationConfig: { responseModalities: ['AUDIO'] },
|
||||
systemInstruction: {
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance) }],
|
||||
parts: [{ text: realtimeInstructions(annotationGuidance, { overpass }) }],
|
||||
},
|
||||
tools: [
|
||||
{
|
||||
functionDeclarations: tools.map(
|
||||
functionDeclarations: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
name,
|
||||
description,
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
isGeminiRequestOriginAllowed,
|
||||
readGeminiTokenRequest,
|
||||
} from './request.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
|
||||
const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
|
||||
const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
@@ -19,6 +20,8 @@ const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
|
||||
export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
tools,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey = () => process.env.GEMINI_API_KEY,
|
||||
resolveModel = () =>
|
||||
@@ -104,6 +107,7 @@ export function createGeminiTokenHandler({
|
||||
annotationGuidance,
|
||||
inputMode,
|
||||
tools,
|
||||
{ overpass: overpassConfigured() },
|
||||
);
|
||||
const issuedAt = now();
|
||||
// REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
|
||||
|
||||
@@ -28,6 +28,7 @@ import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { ghostMapsProxy } from './ghostMaps.js';
|
||||
import { droneAirspaceProxy } from './droneAirspace.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
@@ -79,6 +80,7 @@ function localProviderPlugins({
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
ghostMapsProxy(),
|
||||
droneAirspaceProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createOpenAiChatHandler } from './openai/chat.js';
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
import { handleHudSummary } from './openai/hud-summary.js';
|
||||
import { createDebugLogHandler } from './openai/debug-log.js';
|
||||
@@ -22,6 +23,10 @@ function openAiRealtimeProxy({
|
||||
} = {}) {
|
||||
const oauthLogin = createCodexOAuthLogin(realtime);
|
||||
function install(middlewares) {
|
||||
middlewares.use(
|
||||
'/api/openai/chat',
|
||||
createOpenAiChatHandler({ ...realtime, annotationGuidance }),
|
||||
);
|
||||
// Cost-bearing and log endpoints refuse cross-site browser requests
|
||||
// (see server/providers/common/same-site.js and SECURITY.md).
|
||||
middlewares.use('/api/openai/hud-summary', sameSiteGated(handleHudSummary));
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
import { createTextChatHandler } from '../common/textChat.js';
|
||||
import { preferCodexOAuth } from './codex-auth.js';
|
||||
import { OPENAI_DEFAULT_PER_MIN } from './rate-limit.js';
|
||||
import { invalidCalls } from '../common/textTools.js';
|
||||
import { textInstructions } from '../common/textInstructions.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
|
||||
const object = (value) =>
|
||||
value && typeof value === 'object' && !Array.isArray(value);
|
||||
const keys = (value, allowed) =>
|
||||
Object.keys(value).every((key) => allowed.includes(key));
|
||||
const callShape = (call) =>
|
||||
object(call) &&
|
||||
keys(call, ['id', 'type', 'function']) &&
|
||||
typeof call.id === 'string' &&
|
||||
call.id.length > 0 &&
|
||||
call.id.length <= 1024 &&
|
||||
call.type === 'function' &&
|
||||
object(call.function) &&
|
||||
keys(call.function, ['name', 'arguments']) &&
|
||||
typeof call.function.name === 'string' &&
|
||||
call.function.name.length <= 128 &&
|
||||
typeof call.function.arguments === 'string';
|
||||
function assistantShape(message) {
|
||||
return (
|
||||
object(message) &&
|
||||
keys(message, ['role', 'content', 'tool_calls']) &&
|
||||
message.role === 'assistant' &&
|
||||
(message.content == null || typeof message.content === 'string') &&
|
||||
(message.tool_calls === undefined ||
|
||||
(Array.isArray(message.tool_calls) &&
|
||||
message.tool_calls.length > 0 &&
|
||||
message.tool_calls.length <= 32 &&
|
||||
message.tool_calls.every(callShape) &&
|
||||
new Set(message.tool_calls.map((call) => call.id)).size ===
|
||||
message.tool_calls.length))
|
||||
);
|
||||
}
|
||||
function validMessages(messages) {
|
||||
if (!Array.isArray(messages) || !messages.length || messages.length > 96)
|
||||
return false;
|
||||
let pending = [];
|
||||
for (const message of messages) {
|
||||
if (!object(message)) return false;
|
||||
if (pending.length) {
|
||||
if (
|
||||
!keys(message, ['role', 'content', 'tool_call_id']) ||
|
||||
message.role !== 'tool' ||
|
||||
typeof message.content !== 'string' ||
|
||||
message.tool_call_id !== pending.shift()
|
||||
)
|
||||
return false;
|
||||
} else if (message.role === 'user') {
|
||||
if (
|
||||
!keys(message, ['role', 'content']) ||
|
||||
typeof message.content !== 'string'
|
||||
)
|
||||
return false;
|
||||
} else if (assistantShape(message)) {
|
||||
pending = (message.tool_calls || []).map((call) => call.id);
|
||||
} else return false;
|
||||
}
|
||||
return !pending.length;
|
||||
}
|
||||
|
||||
export function createOpenAiChatHandler({
|
||||
annotationGuidance,
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered, and the prompt names it, only with Overpass, as
|
||||
// for voice.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
resolveApiKey = () => process.env.OPENAI_API_KEY,
|
||||
resolveChatModel = () => process.env.OPENAI_CHAT_MODEL || 'gpt-4.1-mini',
|
||||
resolveProviderRateLimit = () => process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
|
||||
environment = process.env,
|
||||
...options
|
||||
} = {}) {
|
||||
return createTextChatHandler({
|
||||
...options,
|
||||
provider: 'openai',
|
||||
resolveApiKey,
|
||||
resolveProviderRateLimit,
|
||||
providerPerMin: OPENAI_DEFAULT_PER_MIN,
|
||||
// Chat can only bill the API key, which this operator lock forbids.
|
||||
refuse() {
|
||||
let error =
|
||||
'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH, and typed OpenAI chat needs an API key.';
|
||||
try {
|
||||
if (!preferCodexOAuth(environment)) return null;
|
||||
} catch (invalid) {
|
||||
error = invalid.message;
|
||||
}
|
||||
return { error, code: 'CODEX_OAUTH_REQUIRED' };
|
||||
},
|
||||
prepare(input) {
|
||||
const model = String(resolveChatModel()).trim();
|
||||
if (
|
||||
!/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,119}$/.test(model) ||
|
||||
/realtime|audio|tts/i.test(model)
|
||||
)
|
||||
return { errorStatus: 503 };
|
||||
if (
|
||||
!object(input) ||
|
||||
!keys(input, ['messages']) ||
|
||||
!validMessages(input.messages)
|
||||
)
|
||||
return null;
|
||||
const overpass = overpassConfigured();
|
||||
return {
|
||||
endpoint: 'https://api.openai.com/v1/chat/completions',
|
||||
headers: (key) => ({ Authorization: `Bearer ${key}` }),
|
||||
body: {
|
||||
model,
|
||||
n: 1,
|
||||
stream: false,
|
||||
store: false,
|
||||
max_completion_tokens: 4096,
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: textInstructions(annotationGuidance, { overpass }),
|
||||
},
|
||||
...input.messages,
|
||||
],
|
||||
tools: offeredTools(tools, { overpass }).map(
|
||||
({ name, description, parameters }) => ({
|
||||
type: 'function',
|
||||
function: { name, description, parameters, strict: false },
|
||||
}),
|
||||
),
|
||||
},
|
||||
};
|
||||
},
|
||||
parse(data) {
|
||||
const choice = data?.choices?.[0];
|
||||
const raw = choice?.message;
|
||||
if (!raw) return null;
|
||||
const content = {
|
||||
role: raw.role,
|
||||
content: raw.content ?? null,
|
||||
...(raw.tool_calls?.length ? { tool_calls: raw.tool_calls } : {}),
|
||||
};
|
||||
if (
|
||||
!assistantShape(content) ||
|
||||
(!content.content && !content.tool_calls?.length)
|
||||
)
|
||||
return null;
|
||||
const calls = (content.tool_calls || []).map((call) => {
|
||||
try {
|
||||
return {
|
||||
name: call.function.name,
|
||||
args: JSON.parse(call.function.arguments),
|
||||
};
|
||||
} catch {
|
||||
return { name: call.function.name, parseError: true };
|
||||
}
|
||||
});
|
||||
return {
|
||||
content,
|
||||
finishReason: choice.finish_reason,
|
||||
invalidCalls: invalidCalls(
|
||||
calls,
|
||||
offeredTools(tools, { overpass: overpassConfigured() }),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,8 +1,14 @@
|
||||
const DEFAULT_ANNOTATION_GUIDANCE =
|
||||
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
|
||||
|
||||
function realtimeInstructions(
|
||||
/**
|
||||
* The Realtime session directives, one per line. `overpass` says an operator
|
||||
* configured Overpass, which is what offers osm_query (see tools.js); without
|
||||
* it the prompt never mentions the tool.
|
||||
*/
|
||||
function realtimeDirectives(
|
||||
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
|
||||
{ overpass = false } = {},
|
||||
) {
|
||||
return [
|
||||
"You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
|
||||
@@ -21,16 +27,20 @@ function realtimeInstructions(
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
|
||||
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
|
||||
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names, or an areaId you already have; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'AREAS: resolve_area turns a named boundary (country, state, province, county, city, district, natural region) into an areaId; to outline it pass draw=true; annotate_map is for marks, not boundaries. Count inside it → analyst_query kind=area with the areaId (or kind=region); "the area I drew" → kind=drawn; "the area you outlined" → kind=annotation. Imagery of a place → resolve_area, then find_imagery. Ambiguous name: call resolve_area; on needsClarification ask which, never guess.' +
|
||||
(overpass
|
||||
? ' "All hospitals/schools/… in <place>" → resolve_area, then osm_query; in view → osm_query alone; follow-ups → analyst_query layers [osm-places], scope anywhere.'
|
||||
: ''),
|
||||
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
|
||||
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
|
||||
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
|
||||
'Do not mention disabled layers or stale selections.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target, find_imagery area. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, resolve_area, find_imagery, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers, approximate outlines and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
|
||||
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
|
||||
//
|
||||
@@ -60,11 +70,15 @@ function realtimeInstructions(
|
||||
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
|
||||
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
|
||||
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus or district, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park or district footprint; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
annotationGuidance,
|
||||
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
|
||||
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
|
||||
].join('\n');
|
||||
];
|
||||
}
|
||||
|
||||
export { realtimeInstructions };
|
||||
function realtimeInstructions(annotationGuidance, options) {
|
||||
return realtimeDirectives(annotationGuidance, options).join('\n');
|
||||
}
|
||||
|
||||
export { realtimeInstructions, realtimeDirectives };
|
||||
@@ -13,7 +13,8 @@ import {
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
|
||||
} from './constants.js';
|
||||
import { realtimeInstructions } from './instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from './tools.js';
|
||||
import { GEV_REALTIME_TOOLS, offeredTools } from './tools.js';
|
||||
import { resolveOverpassUpstreams } from '../overpass/constants.js';
|
||||
import {
|
||||
preferCodexOAuth,
|
||||
readCodexOAuthAccessToken,
|
||||
@@ -136,6 +137,8 @@ function createRealtimeTokenHandler({
|
||||
environment = process.env,
|
||||
models = {},
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
// osm_query is offered only when an operator configured Overpass.
|
||||
overpassConfigured = () => resolveOverpassUpstreams().length > 0,
|
||||
} = {}) {
|
||||
return async (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -273,6 +276,7 @@ function createRealtimeTokenHandler({
|
||||
transcribeModel.toLowerCase() === 'off'
|
||||
? {}
|
||||
: { transcription: { model: transcribeModel } };
|
||||
const overpass = overpassConfigured();
|
||||
const sessionConfig = {
|
||||
session: {
|
||||
type: 'realtime',
|
||||
@@ -298,8 +302,8 @@ function createRealtimeTokenHandler({
|
||||
},
|
||||
output: { voice },
|
||||
},
|
||||
instructions: realtimeInstructions(annotationGuidance),
|
||||
tools,
|
||||
instructions: realtimeInstructions(annotationGuidance, { overpass }),
|
||||
tools: offeredTools(tools, { overpass }),
|
||||
tool_choice: 'auto',
|
||||
},
|
||||
};
|
||||
|
||||
@@ -405,7 +405,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
annotate_map: {
|
||||
description:
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -547,9 +547,48 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
resolve_area: {
|
||||
description:
|
||||
'Named place → reusable areaId for analyst_query scope {kind:area} and other area tools. draw=true outlines it ("outline Bagmati province"); around=true is "an area around <landmark>": its enclosing mapped area, else a buffer marked approximate. needsClarification: ask which, then call again with its candidateId. AREA_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
query: {
|
||||
description: 'The place as said, with any kind word.',
|
||||
$position: 1,
|
||||
},
|
||||
level: {
|
||||
description: 'Only when the user names the kind.',
|
||||
$position: 1,
|
||||
},
|
||||
within: {
|
||||
description: 'Country or state the user added.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
find_imagery: {
|
||||
description:
|
||||
'Recent satellite imagery (Sentinel-2/Landsat 30 m, VIIRS daily; last 30 days): pins the best acquisition in the Recent Imagery panel. areaId from resolve_area; area=pointer for "this spot"; omit for the view. BOX_REFUSED: offer to zoom in. DISPLACED: leave it.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
maxCloudPct: {
|
||||
description: '"cloud-free" → 10.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
osm_query: {
|
||||
description:
|
||||
'OpenStreetMap places of one kind ("hospitals in Kathmandu"). areaId from resolve_area; omit for the view. Shows them in OSM Places with numbered referents; they become the last answer. countScope bounding-box means an approximate count. OSM_TIMEOUT: retry once.',
|
||||
$position: 1,
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), NEEDS_CLARIFICATION (ask which), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -560,7 +599,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
scope: {
|
||||
description:
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"); area + areaId from resolve_area; drawn = "the area I drew"; annotation = "the area you outlined". Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates.',
|
||||
$position: 2,
|
||||
properties: {
|
||||
name: {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { createActionTools } from '../../../src/voice/actionSchemas.js';
|
||||
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
|
||||
|
||||
/** Every voice tool, as the Realtime session describes them. */
|
||||
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
|
||||
|
||||
/**
|
||||
@@ -14,3 +15,27 @@ export function realtimeSessionTools(additional = []) {
|
||||
...additional.filter((tool) => !names.has(tool.name)),
|
||||
];
|
||||
}
|
||||
|
||||
/** Tools that need an operator-configured Overpass (OVERPASS_UPSTREAMS). */
|
||||
export const OVERPASS_ONLY_TOOLS = Object.freeze(['osm_query']);
|
||||
|
||||
/**
|
||||
* The tools one session is offered. Bulk OpenStreetMap place search runs
|
||||
* only on an operator's Overpass, so without one `osm_query` is left out
|
||||
* rather than offered and refused.
|
||||
* @param {Array<{name: string}>} tools
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function offeredTools(tools, { overpass = false } = {}) {
|
||||
return overpass
|
||||
? tools
|
||||
: tools.filter((tool) => !OVERPASS_ONLY_TOOLS.includes(tool.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* The app actions one session is offered.
|
||||
* @param {{overpass?: boolean}} [options]
|
||||
*/
|
||||
export function realtimeTools({ overpass = false } = {}) {
|
||||
return offeredTools(GEV_REALTIME_TOOLS, { overpass });
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import { installRouteMiddleware } from './places/routes.js';
|
||||
import { installOsmFeaturesRoute } from './overpass/osmFeaturesRoute.js';
|
||||
|
||||
/** @type {Map<string,Promise>} In-flight Overpass requests keyed by normalized query body. */
|
||||
const _overpassInFlight = new Map();
|
||||
@@ -274,6 +275,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
});
|
||||
|
||||
installRouteMiddleware(server.middlewares, routing);
|
||||
installOsmFeaturesRoute(server.middlewares);
|
||||
};
|
||||
return {
|
||||
name: 'overpass-proxy',
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { OSM_PRESETS } from '../../../src/data/osmPresets.js';
|
||||
|
||||
/**
|
||||
* Bounded, typed OpenStreetMap feature search: one preset from the curated
|
||||
* table inside a bounding box.
|
||||
*
|
||||
* The request carries a preset id, a box, a mode and a limit — never query
|
||||
* text. The server writes the Overpass query itself and bounds it: box size
|
||||
* per mode (a count may cover more ground than a fetch), result count,
|
||||
* Overpass timeout and memory, response bytes, and one query in flight. It
|
||||
* runs only against the operator's `OVERPASS_UPSTREAMS`; the generic
|
||||
* `/api/overpass` guard is untouched.
|
||||
*
|
||||
* There is deliberately no boundary (`area`) filter: Overpass servers differ
|
||||
* in whether they hold area data, and one without it answers zero rather than
|
||||
* failing. The browser filters the box's features by the exact area geometry
|
||||
* instead, so a count inside a province is exact and a box count is labelled
|
||||
* as a box count.
|
||||
*/
|
||||
export const OSM_FEATURE_LIMITS = Object.freeze({
|
||||
/** Largest box a fetch may cover, in square degrees (~200 × 200 km at the equator). */
|
||||
maxFetchDeg2: 4,
|
||||
/** Largest box a count may cover. */
|
||||
maxCountDeg2: 36,
|
||||
/** Most features one fetch returns. */
|
||||
maxLimit: 1000,
|
||||
/** Overpass server-side timeout, seconds: it gives up before we stop waiting. */
|
||||
timeoutS: 20,
|
||||
/** Overpass server-side memory cap, bytes. */
|
||||
maxsize: 64 * 1024 * 1024,
|
||||
/** Response bytes read from the upstream. */
|
||||
maxResponseBytes: 16 * 1024 * 1024,
|
||||
/** Per-upstream wait: the Overpass timeout plus transfer time. */
|
||||
upstreamTimeoutMs: 22_000,
|
||||
/**
|
||||
* The whole search, across every configured endpoint in turn. It ends
|
||||
* before the voice answer's own deadline (`OSM_QUERY_LIMITS.budgetMs` in
|
||||
* `src/voice/osmActions.js`, 25 s): a browser that gives up first closes
|
||||
* the request, which cancels the upstream query, so nothing would be
|
||||
* cached for the retry.
|
||||
*/
|
||||
upstreamBudgetMs: 22_000,
|
||||
});
|
||||
|
||||
/** Tags a returned feature keeps, besides the preset's own. */
|
||||
const KEPT_TAGS = [
|
||||
'name',
|
||||
'name:en',
|
||||
'operator',
|
||||
'brand',
|
||||
'opening_hours',
|
||||
'wheelchair',
|
||||
'emergency',
|
||||
'healthcare',
|
||||
'addr:street',
|
||||
'addr:city',
|
||||
];
|
||||
|
||||
const quote = (text) =>
|
||||
`"${String(text).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
||||
|
||||
/**
|
||||
* Parse and bound a request. Returns the compiled query or a refusal.
|
||||
* @param {{preset: string, bbox: string|number[], mode?: string, limit?: string|number}} request
|
||||
* @returns {{ok: true, ql: string, mode: string, preset: object, limit: number, boxes: number[][]}|{ok: false, code: string, error: string}}
|
||||
*/
|
||||
export function compileOsmFeatureQuery(request = {}) {
|
||||
const refuse = (code, error) => ({ ok: false, code, error });
|
||||
const presetId = String(request.preset || '');
|
||||
const preset = Object.hasOwn(OSM_PRESETS, presetId)
|
||||
? OSM_PRESETS[presetId]
|
||||
: null;
|
||||
if (!preset) return refuse('UNKNOWN_PRESET', 'Unknown feature kind.');
|
||||
const mode = request.mode === 'count' ? 'count' : 'features';
|
||||
const parts = (
|
||||
Array.isArray(request.bbox)
|
||||
? request.bbox
|
||||
: String(request.bbox || '').split(',')
|
||||
).map(Number);
|
||||
if (parts.length !== 4 || !parts.every(Number.isFinite))
|
||||
return refuse('BAD_BOX', 'bbox needs west,south,east,north.');
|
||||
const [west, south, east, north] = parts;
|
||||
if (
|
||||
Math.abs(west) > 180 ||
|
||||
Math.abs(east) > 180 ||
|
||||
south < -90 ||
|
||||
north > 90 ||
|
||||
south >= north ||
|
||||
west === east
|
||||
)
|
||||
return refuse('BAD_BOX', 'bbox edges are out of range.');
|
||||
// A box crossing the antimeridian is searched as its two halves.
|
||||
const boxes =
|
||||
west < east
|
||||
? [[south, west, north, east]]
|
||||
: [
|
||||
[south, west, north, 180],
|
||||
[south, -180, north, east],
|
||||
];
|
||||
const deg2 = boxes.reduce((sum, [s, w, n, e]) => sum + (n - s) * (e - w), 0);
|
||||
const cap =
|
||||
mode === 'count'
|
||||
? OSM_FEATURE_LIMITS.maxCountDeg2
|
||||
: OSM_FEATURE_LIMITS.maxFetchDeg2;
|
||||
if (deg2 > cap)
|
||||
return refuse(
|
||||
'AREA_TOO_LARGE',
|
||||
mode === 'count'
|
||||
? 'That area is too large to search; narrow it down.'
|
||||
: 'That area is too large to list; count first or narrow it down.',
|
||||
);
|
||||
// An absent or empty limit is the default, not Number(null) = 0.
|
||||
const requested =
|
||||
request.limit === null ||
|
||||
request.limit === undefined ||
|
||||
String(request.limit).trim() === ''
|
||||
? NaN
|
||||
: Number(request.limit);
|
||||
const limit = Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
OSM_FEATURE_LIMITS.maxLimit,
|
||||
Number.isFinite(requested) ? Math.trunc(requested) : 500,
|
||||
),
|
||||
);
|
||||
const selectors = [];
|
||||
for (const [key, value] of preset.tags)
|
||||
for (const [s, w, n, e] of boxes)
|
||||
selectors.push(
|
||||
`nwr[${quote(key)}=${quote(value)}](${s},${w},${n},${e});`,
|
||||
);
|
||||
const ql =
|
||||
`[out:json][timeout:${OSM_FEATURE_LIMITS.timeoutS}][maxsize:${OSM_FEATURE_LIMITS.maxsize}];` +
|
||||
`(${selectors.join('')});` +
|
||||
(mode === 'count' ? 'out count;' : `out center tags ${limit};`);
|
||||
return { ok: true, ql, mode, preset, limit, boxes };
|
||||
}
|
||||
|
||||
/**
|
||||
* Overpass JSON → the compact answer the browser gets.
|
||||
* @param {object} payload Parsed Overpass JSON.
|
||||
* @param {{mode: string, preset: object, limit: number}} compiled
|
||||
*/
|
||||
export function summarizeOsmFeatures(payload, compiled) {
|
||||
if (!Array.isArray(payload?.elements))
|
||||
throw new TypeError('Malformed Overpass feature response');
|
||||
const elements = payload.elements;
|
||||
if (compiled.mode === 'count') {
|
||||
const tags = elements.find((el) => el?.type === 'count')?.tags;
|
||||
const rawTotal = tags?.total;
|
||||
const total = Number(rawTotal);
|
||||
if (
|
||||
!tags ||
|
||||
!Object.hasOwn(tags, 'total') ||
|
||||
!['string', 'number'].includes(typeof rawTotal) ||
|
||||
String(rawTotal).trim() === '' ||
|
||||
!Number.isSafeInteger(total) ||
|
||||
total < 0
|
||||
)
|
||||
throw new TypeError('Malformed Overpass count response');
|
||||
return {
|
||||
mode: 'count',
|
||||
preset: compiled.preset.id,
|
||||
count: total,
|
||||
nodes: Number(tags.nodes) || 0,
|
||||
ways: Number(tags.ways) || 0,
|
||||
relations: Number(tags.relations) || 0,
|
||||
};
|
||||
}
|
||||
const features = [];
|
||||
let unplaced = 0;
|
||||
for (const el of elements) {
|
||||
if (
|
||||
!['node', 'way', 'relation'].includes(el?.type) ||
|
||||
(!Number.isFinite(Number(el?.id)) && typeof el?.id !== 'string')
|
||||
) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const lat = Number.isFinite(el?.lat) ? el.lat : el?.center?.lat;
|
||||
const lon = Number.isFinite(el?.lon) ? el.lon : el?.center?.lon;
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) {
|
||||
unplaced += 1;
|
||||
continue;
|
||||
}
|
||||
const tags = {};
|
||||
for (const key of [...KEPT_TAGS, ...compiled.preset.tags.map(([k]) => k)])
|
||||
if (typeof el.tags?.[key] === 'string')
|
||||
tags[key] = el.tags[key].slice(0, 120);
|
||||
features.push({ id: `${el.type}/${el.id}`, lat, lon, tags });
|
||||
}
|
||||
// A genuinely empty `elements` array is authoritative. A non-empty answer
|
||||
// whose every row is unusable is a schema failure, not "zero places"; do
|
||||
// not let the route cache that as current state for a day.
|
||||
if (elements.length && features.length === 0)
|
||||
throw new TypeError('Overpass feature response has no usable rows');
|
||||
return {
|
||||
mode: 'features',
|
||||
preset: compiled.preset.id,
|
||||
count: features.length,
|
||||
features,
|
||||
truncated: elements.length >= compiled.limit,
|
||||
...(unplaced ? { unplaced } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import path from 'node:path';
|
||||
import { makeRateLimiter, clientNetworkKey } from '../common/rate-limit.js';
|
||||
import { admitSameSiteKeyless } from '../common/same-site.js';
|
||||
import { createBoundedCache } from '../common/boundedCache.js';
|
||||
import {
|
||||
compileOsmFeatureQuery,
|
||||
OSM_FEATURE_LIMITS,
|
||||
summarizeOsmFeatures,
|
||||
} from './osmFeatures.js';
|
||||
import {
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
} from './constants.js';
|
||||
import { fetchOverpassPayload, overpassPayloadIsData } from './transport.js';
|
||||
|
||||
/** Places of interest change slowly: answers are kept a day. */
|
||||
const OSM_FEATURES_TTL_MS = 86_400_000;
|
||||
/** Upstream searches at once, across all clients (one per client). */
|
||||
const OSM_MAX_RUNNING = 2;
|
||||
/** Cache-key version; nothing cached under another version is reused. */
|
||||
const CACHE_VERSION = 'v2';
|
||||
|
||||
/** Never simplify: these answers are points and counts, not drawings. */
|
||||
const keepBody = (body) => body;
|
||||
|
||||
const jsonError = (status, error, extra = {}) => ({
|
||||
status,
|
||||
body: JSON.stringify({ error, ...extra }),
|
||||
});
|
||||
|
||||
/**
|
||||
* One upstream Overpass query at a time, with identical requests sharing it
|
||||
* and answers in a bounded memory + disk cache.
|
||||
*/
|
||||
function createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache,
|
||||
maxResponseBytes,
|
||||
upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
maxRunning = OSM_MAX_RUNNING,
|
||||
}) {
|
||||
const inFlight = new Map();
|
||||
// One search per client, a few in all: one client can't hold the route.
|
||||
const runningBy = new Map();
|
||||
let running = 0;
|
||||
return async function answer(key, ql, summarize, signal = null, owner = '') {
|
||||
signal?.throwIfAborted?.();
|
||||
const held = await cache.get(key);
|
||||
signal?.throwIfAborted?.();
|
||||
if (typeof held === 'string')
|
||||
return { status: 200, body: held, cache: 'HIT' };
|
||||
let operation = inFlight.get(key);
|
||||
if (operation?.controller.signal.aborted) {
|
||||
inFlight.delete(key);
|
||||
operation = null;
|
||||
}
|
||||
if (!operation) {
|
||||
if (runningBy.get(owner) || running >= maxRunning)
|
||||
return {
|
||||
...jsonError(
|
||||
503,
|
||||
runningBy.get(owner)
|
||||
? 'Your previous map search is still running — try again shortly.'
|
||||
: 'Another map search is running — try again shortly.',
|
||||
),
|
||||
retryAfter: '3',
|
||||
};
|
||||
running += 1;
|
||||
runningBy.set(owner, (runningBy.get(owner) || 0) + 1);
|
||||
const controller = new AbortController();
|
||||
// The whole search, every configured endpoint in turn, gets one budget.
|
||||
const budget = AbortSignal.timeout(upstreamBudgetMs);
|
||||
operation = { controller, waiters: 0, settled: false, promise: null };
|
||||
const owned = operation;
|
||||
owned.promise = (async () => {
|
||||
try {
|
||||
const payload = await fetchPayload(
|
||||
`data=${encodeURIComponent(ql)}`,
|
||||
maxResponseBytes,
|
||||
{
|
||||
simplify: keepBody,
|
||||
timeoutMs: upstreamTimeoutMs,
|
||||
signal: AbortSignal.any([controller.signal, budget]),
|
||||
},
|
||||
);
|
||||
if (!overpassPayloadIsData(payload))
|
||||
return {
|
||||
...jsonError(
|
||||
payload?.rateLimited ? 429 : 503,
|
||||
'OpenStreetMap search is unavailable right now.',
|
||||
),
|
||||
retryAfter: '10',
|
||||
};
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(payload.body);
|
||||
} catch {
|
||||
return jsonError(
|
||||
502,
|
||||
'OpenStreetMap returned an unreadable answer.',
|
||||
);
|
||||
}
|
||||
const body = JSON.stringify(summarize(parsed));
|
||||
controller.signal.throwIfAborted();
|
||||
await cache.set(key, body);
|
||||
controller.signal.throwIfAborted();
|
||||
return { status: 200, body, cache: 'MISS' };
|
||||
} catch {
|
||||
return budget.aborted && !controller.signal.aborted
|
||||
? jsonError(504, 'OpenStreetMap took too long to answer.')
|
||||
: jsonError(502, 'OpenStreetMap search failed.');
|
||||
} finally {
|
||||
owned.settled = true;
|
||||
running -= 1;
|
||||
const left = (runningBy.get(owner) || 1) - 1;
|
||||
if (left > 0) runningBy.set(owner, left);
|
||||
else runningBy.delete(owner);
|
||||
if (inFlight.get(key) === owned) inFlight.delete(key);
|
||||
}
|
||||
})();
|
||||
inFlight.set(key, owned);
|
||||
}
|
||||
|
||||
operation.waiters += 1;
|
||||
let abort = null;
|
||||
const abandoned = signal
|
||||
? new Promise((_, reject) => {
|
||||
abort = () => {
|
||||
operation.waiters -= 1;
|
||||
if (operation.waiters === 0 && !operation.settled)
|
||||
operation.controller.abort();
|
||||
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
||||
};
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
if (signal.aborted) abort();
|
||||
})
|
||||
: null;
|
||||
try {
|
||||
return await (abandoned
|
||||
? Promise.race([operation.promise, abandoned])
|
||||
: operation.promise);
|
||||
} finally {
|
||||
if (signal && abort) signal.removeEventListener('abort', abort);
|
||||
if (!signal?.aborted) operation.waiters -= 1;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sender(res) {
|
||||
return ({ status, body, cache, retryAfter }) => {
|
||||
if (res.writableEnded) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(cache ? { 'X-OSM-Cache': cache } : {}),
|
||||
...(retryAfter ? { 'Retry-After': retryAfter } : {}),
|
||||
});
|
||||
res.end(body);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Install `GET /api/osm/features?preset=&bbox=w,s,e,n[&mode=count][&limit=]`:
|
||||
* one curated preset inside a box, answered by the operator's Overpass
|
||||
* (`OVERPASS_UPSTREAMS`). With none configured the route answers
|
||||
* `OVERPASS_NOT_CONFIGURED` and fetches nothing; public Nominatim is never a
|
||||
* substitute for bulk place search.
|
||||
*
|
||||
* @param {{use: Function}} middlewares
|
||||
* @param {{fetchPayload?: Function, featuresCache?: object, cacheRoot?: string|null, configured?: () => boolean, upstreamTag?: () => string}} [deps]
|
||||
*/
|
||||
export function installOsmFeaturesRoute(
|
||||
middlewares,
|
||||
{
|
||||
fetchPayload = fetchOverpassPayload,
|
||||
configured = () => resolveOverpassUpstreams().length > 0,
|
||||
// Keys each answer to the instance that gave it, as the other Overpass
|
||||
// caches do, so repointing OVERPASS_UPSTREAMS never serves the old one's.
|
||||
upstreamTag = resolveOverpassUpstreamTag,
|
||||
upstreamBudgetMs = OSM_FEATURE_LIMITS.upstreamBudgetMs,
|
||||
cacheRoot = path.join(process.cwd(), '.gev-cache'),
|
||||
featuresCache = createBoundedCache({
|
||||
dir: cacheRoot ? path.join(cacheRoot, 'osm-features') : null,
|
||||
ttlMs: OSM_FEATURES_TTL_MS,
|
||||
maxMemoryEntries: 60,
|
||||
maxDiskEntries: 400,
|
||||
maxDiskBytes: 256 * 1024 * 1024,
|
||||
}),
|
||||
} = {},
|
||||
) {
|
||||
const limiter = makeRateLimiter({ windowMs: 60_000, max: 12, globalMax: 30 });
|
||||
const features = createOverpassAnswerer({
|
||||
fetchPayload,
|
||||
cache: featuresCache,
|
||||
maxResponseBytes: OSM_FEATURE_LIMITS.maxResponseBytes,
|
||||
upstreamTimeoutMs: OSM_FEATURE_LIMITS.upstreamTimeoutMs,
|
||||
upstreamBudgetMs,
|
||||
});
|
||||
|
||||
middlewares.use('/api/osm/features', async (req, res) => {
|
||||
const send = sender(res);
|
||||
if (req.method !== 'GET') return send(jsonError(405, 'Method Not Allowed'));
|
||||
// A page elsewhere must not spend the operator's Overpass. The keyless
|
||||
// gate, as on /api/overpass: voice works behind a reverse proxy too.
|
||||
if (admitSameSiteKeyless(req, res)) return undefined;
|
||||
if (!configured())
|
||||
return send(
|
||||
jsonError(503, 'Place search needs a configured Overpass server.', {
|
||||
code: 'OVERPASS_NOT_CONFIGURED',
|
||||
retryable: false,
|
||||
}),
|
||||
);
|
||||
const url = new URL(req.url || '', 'http://localhost');
|
||||
const compiled = compileOsmFeatureQuery({
|
||||
preset: url.searchParams.get('preset'),
|
||||
bbox: url.searchParams.get('bbox'),
|
||||
mode: url.searchParams.get('mode'),
|
||||
limit: url.searchParams.get('limit'),
|
||||
});
|
||||
if (!compiled.ok)
|
||||
return send(jsonError(400, compiled.error, { code: compiled.code }));
|
||||
// IPv6 peers by /64, as /api/overpass does: rotating addresses inside
|
||||
// one prefix earns neither a second rate bucket nor a second slot.
|
||||
const client = clientNetworkKey(req);
|
||||
if (!limiter(client))
|
||||
return send({
|
||||
...jsonError(429, 'Rate limit exceeded'),
|
||||
retryAfter: '10',
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
req.once?.('aborted', abort);
|
||||
res.once?.('close', abort);
|
||||
try {
|
||||
const answer = await features(
|
||||
`${CACHE_VERSION}|features|${upstreamTag()}|${compiled.ql}`,
|
||||
compiled.ql,
|
||||
(parsed) => summarizeOsmFeatures(parsed, compiled),
|
||||
controller.signal,
|
||||
client,
|
||||
);
|
||||
if (!controller.signal.aborted) send(answer);
|
||||
} catch {
|
||||
if (!controller.signal.aborted)
|
||||
send(jsonError(502, 'OpenStreetMap search failed.'));
|
||||
} finally {
|
||||
req.removeListener?.('aborted', abort);
|
||||
res.removeListener?.('close', abort);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -124,6 +124,8 @@ async function fetchOverpassPayload(
|
||||
readBody = readResponseTextCapped,
|
||||
simplify = simplifyOverpassPayloadBody,
|
||||
now = Date.now,
|
||||
timeoutMs = OVERPASS_TIMEOUT_MS,
|
||||
signal = null,
|
||||
} = {},
|
||||
) {
|
||||
if (!endpoints.length) return overpassNotConfigured();
|
||||
@@ -135,7 +137,10 @@ async function fetchOverpassPayload(
|
||||
continue;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
const requestSignal = signal
|
||||
? AbortSignal.any([signal, controller.signal])
|
||||
: controller.signal;
|
||||
try {
|
||||
const requestUrl = new URL(endpoint);
|
||||
const authorization =
|
||||
@@ -156,7 +161,7 @@ async function fetchOverpassPayload(
|
||||
...(authorization ? { Authorization: authorization } : {}),
|
||||
},
|
||||
body,
|
||||
signal: controller.signal,
|
||||
signal: requestSignal,
|
||||
});
|
||||
const responseBody = await readBody(upstream, maxResponseBytes);
|
||||
const parsed = parseJson(responseBody);
|
||||
@@ -207,7 +212,12 @@ async function fetchOverpassPayload(
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
rateLimited: false,
|
||||
};
|
||||
} catch {
|
||||
} catch (error) {
|
||||
// A route whose last requester disconnected owns this cancellation. It
|
||||
// is not evidence that the configured upstream is unhealthy, so do not
|
||||
// put the endpoint on cooldown or translate the abort into a response
|
||||
// that a now-gone client could cache.
|
||||
if (signal?.aborted) throw signal.reason ?? error;
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const delay = retryDelay(null, failures - 1, now());
|
||||
cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
|
||||
|
||||
@@ -204,6 +204,97 @@ const ADMIN_EVIDENCE = Object.freeze({
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* Nominatim address types whose meaning is stable enough for the area consumer's
|
||||
* country/admin1/admin2 contract. `region`, `district` and numeric admin levels
|
||||
* vary by country, so they remain unverified instead of being guessed.
|
||||
*/
|
||||
const VERIFIED_ADMIN_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
state: 'admin1',
|
||||
province: 'admin1',
|
||||
county: 'admin2',
|
||||
state_district: 'admin2',
|
||||
});
|
||||
|
||||
const EXPLICIT_NAME_LEVEL = Object.freeze({
|
||||
country: 'country',
|
||||
nation: 'country',
|
||||
province: 'admin1',
|
||||
state: 'admin1',
|
||||
region: 'admin1',
|
||||
oblast: 'admin1',
|
||||
prefecture: 'admin1',
|
||||
canton: 'admin1',
|
||||
governorate: 'admin1',
|
||||
county: 'admin2',
|
||||
district: 'admin2',
|
||||
department: 'admin2',
|
||||
zone: 'retired',
|
||||
});
|
||||
|
||||
function explicitNameEvidence(value) {
|
||||
const normalized = String(value || '')
|
||||
.normalize('NFKC')
|
||||
.toLocaleLowerCase()
|
||||
.replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
|
||||
.trim();
|
||||
const prefix = normalized.match(
|
||||
/^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/,
|
||||
)?.[1];
|
||||
const suffix = normalized.match(
|
||||
/\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/,
|
||||
)?.[1];
|
||||
const levels = new Set(
|
||||
[prefix, suffix]
|
||||
.filter(Boolean)
|
||||
.map((type) => EXPLICIT_NAME_LEVEL[type])
|
||||
.filter(Boolean),
|
||||
);
|
||||
if (levels.size > 1) return { contradictory: true, level: null };
|
||||
return { contradictory: false, level: levels.values().next().value || null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verified administrative identity carried by the selected result itself.
|
||||
* Never infer it from the request, desired kind, rank, or a separate geocoder
|
||||
* anchor. A retired name is preserved without a level so downstream policy can
|
||||
* reject it explicitly; every other contradiction stays wholly unverified.
|
||||
*/
|
||||
function selectedAdminIdentity(kind, row) {
|
||||
if (kind !== 'admin') return null;
|
||||
const adminLevel =
|
||||
VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()];
|
||||
const authoritativeAdminArea = String(row?.name || '').trim();
|
||||
if (!adminLevel || !authoritativeAdminArea) return null;
|
||||
// Validate the complete authoritative name. `adminArea` is truncated only
|
||||
// for display after every prefix/suffix signal has been considered.
|
||||
const { contradictory, level: nameLevel } = explicitNameEvidence(
|
||||
authoritativeAdminArea,
|
||||
);
|
||||
if (contradictory) return null;
|
||||
const adminArea = authoritativeAdminArea.slice(0, 160);
|
||||
if (nameLevel === 'retired') {
|
||||
// Nepal replaced its zones in 2015, but zones are current units elsewhere
|
||||
// (Ethiopia). Without a country the name stays retired.
|
||||
if (!zoneIsCurrentIn(row)) return { adminArea };
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
if (nameLevel && nameLevel !== adminLevel) return null;
|
||||
return { adminArea, adminLevel };
|
||||
}
|
||||
|
||||
/** Whether the row names a country (last display_name part) other than Nepal. */
|
||||
function zoneIsCurrentIn(row) {
|
||||
const country = String(row?.display_name || '')
|
||||
.split(',')
|
||||
.slice(1)
|
||||
.pop()
|
||||
?.trim()
|
||||
.toLocaleLowerCase();
|
||||
return Boolean(country) && country !== 'nepal';
|
||||
}
|
||||
|
||||
/** Whether supplied administrative evidence contradicts the accepted identity. */
|
||||
function adminLevelFits(kind, row) {
|
||||
const evidence = ADMIN_EVIDENCE[kind];
|
||||
@@ -584,6 +675,7 @@ export function selectOutlineResult(rows, { kind, query }) {
|
||||
row.osm_type && row.osm_id != null
|
||||
? `${row.osm_type}/${row.osm_id}`
|
||||
: null,
|
||||
...(selectedAdminIdentity(kind, row) || {}),
|
||||
polygons,
|
||||
},
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user