benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-04 10:24:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(outlines): restore voice outlines without public Overpass

Browse Source 
use bundled place boundaries and shared OpenFreeMap tiles before a guarded Nominatim fallback

preserve multipart shapes, holes and cancellation,
and keep pending annotations visible while ground sampling finishes across frames

keep non-Latin name distinctions when selecting provider results,
and retain an outline anchor only when it lies in filled geometry
This commit is contained in:
manjunath committed 2026-10-27 13:24:02 +00:00
2 parents 41b2475362 15de9d738c
commit d4434aa6d7
1489 files changed
+196442 -12353
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/mcp/panelKey.js
+117
View File
@@ -0,0 +1,117 @@
import fs from 'node:fs';
import path from 'node:path';
import { randomBytes, randomUUID } from 'node:crypto';
import { fileURLToPath } from 'node:url';
/** Where the install's panel key lives: beside the other local state, which
* git ignores, at the checkout root rather than the client's working
* directory (an MCP client may start the server anywhere). */
export const DEFAULT_PANEL_KEY_FILE = fileURLToPath(
new URL('../../.gev-cache/mcp-panel-key', import.meta.url),
);
const KEY_PATTERN = /^[A-Za-z0-9_-]{43}$/;
const newKey = () => randomBytes(32).toString('base64url');
const notShared = (error) =>
`panel key not shared (${error.code || error.message}); using a key of this process`;
function readKey(file) {
const text = fs.readFileSync(file, 'utf8').trim();
return KEY_PATTERN.test(text) ? text : null;
}
/** Serialize malformed-file repair; ordinary first creation still uses link(). */
function repairKey(file, tmp, key) {
const lock = `${file}.repair-lock`;
const deadline = Date.now() + 2000;
const pause = new Int32Array(new SharedArrayBuffer(4));
let fd;
for (;;) {
try {
fd = fs.openSync(lock, 'wx', 0o600);
break;
} catch (error) {
if (error.code !== 'EEXIST') throw error;
// Another repair may already have published a usable key.
const winner = readKey(file);
if (winner) return winner;
if (Date.now() >= deadline) {
throw Object.assign(new Error('panel key repair is busy'), {
code: 'ETIMEDOUT',
});
}
Atomics.wait(pause, 0, 0, 10);
}
}
try {
// A repair completed between our earlier read and acquiring the lock.
const winner = readKey(file);
if (winner) return winner;
fs.renameSync(tmp, file);
return key;
} finally {
fs.closeSync(fd);
fs.rmSync(lock, { force: true });
}
}
/**
* The panel key every stdio server of this install shares.
*
* A client can start more than one server process for one connection: Claude
* Desktop was reported to start two processes; serving the panel's page from
* one and sending its `panel_request` calls to the other reproduces #927.
* Per-process keys then mismatch, and the panel refuses the request.
* The first process to run creates the key, without replacing one
* another process created at the same moment; the rest read it. When the file
* cannot be read or written, or repair stays busy for two seconds, the process
* falls back to its own key, as every process did before.
*
* As before, the key keeps `panel_request` from clients that only list it; it
* is not access control.
*
* @param {{ file?: string, log?: (line: string) => void }} [options]
* @returns {string}
*/
export function sharedPanelKey({
file = DEFAULT_PANEL_KEY_FILE,
log = () => {},
} = {}) {
try {
const existing = readKey(file);
if (existing) return existing;
// Present but not a key: replace it below.
} catch (error) {
if (error.code !== 'ENOENT') {
log(notShared(error));
return newKey();
}
}
const key = newKey();
let tmp = null;
try {
fs.mkdirSync(path.dirname(file), { recursive: true });
tmp = path.join(
path.dirname(file),
`.${path.basename(file)}.${randomUUID().slice(0, 8)}.tmp`,
);
fs.writeFileSync(tmp, `${key}\n`, { flag: 'wx', mode: 0o600 });
try {
// link() publishes the whole file at once and fails if another process
// got there first, so a reader never sees a half-written key and two
// processes starting together cannot end up with different keys.
fs.linkSync(tmp, file);
return key;
} catch (error) {
if (error.code !== 'EEXIST') throw error;
const winner = readKey(file);
if (winner) return winner;
return repairKey(file, tmp, key);
}
} catch (error) {
log(notShared(error));
return key;
} finally {
if (tmp) fs.rmSync(tmp, { force: true });
}
}
server/mcp/server.js
+6 -4
View File
@@ -27,14 +27,16 @@ const INSTRUCTIONS =
export function createLocalMcpServer({
apiBase = DEFAULT_API_BASE,
fetchImpl,
panelKey = randomBytes(32).toString('base64url'),
} = {}) {
const { version } = JSON.parse(
readFileSync(new URL('../../package.json', import.meta.url), 'utf8'),
);
// Each server makes a new key for its panel. The panel's page carries it
// and panel_request requires it; any client may read the page, so it keeps
// the tool from clients that only list it, and is not access control.
const panelKey = randomBytes(32).toString('base64url');
// Unless given one, each server makes a new key for its panel. The panel's
// page carries it and panel_request requires it; any client may read the
// page, so it keeps the tool from clients that only list it, and is not
// access control. The stdio entry passes the install's shared key
// (panelKey.js), so separate processes of one client agree.
return createMcpServer({
catalog: catalogForSurface(
composeCatalog({
server/mcp/stdio.js
+5 -1
View File
@@ -11,6 +11,7 @@
import { createInterface } from 'node:readline';
import { fileURLToPath } from 'node:url';
import { parseErrorResponse } from '../../src/tools/mcp/index.js';
import { sharedPanelKey } from './panelKey.js';
import { createLocalMcpServer } from './server.js';
import { DEFAULT_API_BASE } from './services.js';
@@ -97,7 +98,10 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
console.error(
`God's Eye View MCP server reading ${options.apiBase || DEFAULT_API_BASE}`,
);
await serveStdio(createLocalMcpServer(options), {
// A client may start several of these for one connection; they share
// the install's panel key so the panel works whichever one it reaches.
const panelKey = sharedPanelKey({ log: (line) => console.error(line) });
await serveStdio(createLocalMcpServer({ ...options, panelKey }), {
input: process.stdin,
output: process.stdout,
log: (line) => console.error(`<- ${line}`),
server/providers/aircraft/adsb-lol.js
+82 -45
View File
@@ -1,7 +1,12 @@
import {
coalesceProxyRequest,
readResponseTextCapped,
} from '../common/http.js';
/**
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
*
* Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. When upstream
* Serves GET /api/military from https://api.adsb.lol/v2/mil. When upstream
* fails — a thrown fetch OR a non-OK status such as 429 — the proxy serves its
* cached body as STALE instead of relaying the failure, and it backs off from
* upstream for the Retry-After period (bounded) so a rate limit is never
@@ -17,6 +22,7 @@
export function adsbLolProxy() {
/** @type {string|null} Cached upstream JSON body. */
let _cache = null;
const inFlight = new Map();
/** @type {number} Epoch-ms when the cache was populated. */
let _cacheAt = 0;
/** @type {number} Epoch-ms until which upstream is not contacted. */
@@ -25,6 +31,10 @@ export function adsbLolProxy() {
let _cooldownStatus = 0;
/** Response cache TTL (ms). */
const CACHE_MS = 12000;
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
const UPSTREAM_TIMEOUT_MS = 12000;
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
const RATE_LIMIT_COOLDOWN_MS = 30000;
/** Cooldown after a 5xx (ms). */
@@ -55,10 +65,11 @@ export function adsbLolProxy() {
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
'X-ADS-B-Cache': cacheStatus,
'X-Feed-Source': 'adsb.lol',
'X-Feed-Cache': cacheStatus,
...(['HIT', 'STALE'].includes(cacheStatus)
? {
'X-ADS-B-Cache-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)),
'X-Feed-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)),
}
: {}),
...extra,
@@ -67,8 +78,11 @@ export function adsbLolProxy() {
}
const installMiddleware = (server) => {
server.middlewares.use('/api/adsblol/mil', async (req, res) => {
server.middlewares.use('/api/military', async (req, res, next) => {
try {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
const now = Date.now();
if (_cache && now - _cacheAt < CACHE_MS) {
serve(res, 200, _cache, 'HIT');
@@ -94,50 +108,73 @@ export function adsbLolProxy() {
);
return;
}
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
});
if (upstream.ok) {
const body = await upstream.text();
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
serve(res, 200, body, 'MISS');
return;
}
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
serve(res, 200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
});
return;
const refresh = coalesceProxyRequest(inFlight, 'global', async () => {
const result = (status, body, cacheStatus, extra = {}) => ({
status,
body,
cacheStatus,
extra,
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
return result(200, body, 'MISS');
}
}
const body = await upstream.text();
serve(
res,
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
return result(200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
});
}
}
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
return result(
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
);
});
const response = await refresh.promise;
serve(
res,
response.status,
response.body,
response.cacheStatus,
response.extra,
);
} catch (e) {
console.error('[adsb.lol Proxy]', e.message);
server/providers/aircraft/enrichment.js
+15 -6
View File
@@ -1,10 +1,15 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseJsonCapped } from '../common/http.js';
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
/**
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
* airports) and hex → aircraft type/registration. Free community API — cached
* aggressively: ONE upstream request per new key ever (404s negative-cached),
* persisted to disk so restarts don't re-hammer it. Adapted from skylight
* aggressively: successes and 404s are kept per key for 24 hours (TTL_MS) and
* persisted to disk so restarts don't re-hammer it; other failures are not
* cached. Adapted from skylight
* (MIT) server/src/enrich/routes.ts.
*/
export function adsbdbProxy() {
@@ -80,12 +85,16 @@ export function adsbdbProxy() {
kind === 'route'
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
const signal = AbortSignal.timeout(8000);
const res = await fetch(url, { signal });
if (res.ok) {
const payload = await readResponseJsonCapped(
res,
ADSBDB_MAX_RESPONSE_BYTES,
signal,
);
const data =
kind === 'route'
? parseRoute(await res.json())
: parseAircraft(await res.json());
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
store[key] = { at: Date.now(), data }; // data may be null — negative cache
dirty = true;
return data;
server/providers/aircraft/opensky.js
+224 -149
View File
@@ -1,7 +1,10 @@
import fs from 'node:fs';
import os from 'node:os';
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
import {
coalesceProxyRequest,
readResponseJsonCapped,
readResponseTextCapped,
} from '../common/http.js';
import { requiredFiniteQueryNumber } from '../common/query.js';
// ---------------------------------------------------------------------------
@@ -29,11 +32,34 @@ const OPENSKY_CACHE_MS = 9000;
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
const OPENSKY_STATES_URL =
'https://opensky-network.org/api/states/all?extended=1';
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/**
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
* 30 s … 30 min, or 2 min when the header is absent.
* @param {Response} upstream Rate-limited response.
* @returns {number} Cooldown (ms).
*/
function openSkyRateLimitCooldownMs(upstream) {
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
const retryAfterSec =
header == null || header.trim() === '' ? NaN : Number(header);
return Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
}
/**
* Fetch and read the global snapshot, giving each attempt
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
* the connection fails. A second failure throws, which the proxy answers
* the connection fails. An oversized body is not retried: it would fail again
* and spend more credits. A second failure throws, which the proxy answers
* from its stale cache or the regional fallback.
* @param {Record<string, string>} headers Request headers.
* @returns {Promise<{upstream: Response, body: string}>}
@@ -41,13 +67,36 @@ const OPENSKY_STATES_URL =
async function fetchOpenSkyStates(headers) {
for (let attempt = 1; ; attempt++) {
try {
const upstream = await fetch(OPENSKY_STATES_URL, {
headers,
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
});
return { upstream, body: await upstream.text() };
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
if (upstream.status === 429 || upstream.status >= 500) {
// Start the cooldown from the headers and drop the rejected body
// unread, so a stalled or oversized 429 body cannot skip it.
_openskyCooldownUntil =
Date.now() +
(upstream.status === 429
? openSkyRateLimitCooldownMs(upstream)
: 15000);
_openskyCooldownStatus = upstream.status;
void upstream.body?.cancel().catch(() => {});
return {
upstream,
body: JSON.stringify({
error:
upstream.status === 429
? 'OpenSky rate limited.'
: 'OpenSky upstream unavailable.',
}),
};
}
const body = await readResponseTextCapped(
upstream,
OPENSKY_STATES_MAX_RESPONSE_BYTES,
signal,
);
return { upstream, body };
} catch (error) {
if (attempt >= 2) throw error;
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
console.warn(
'[OpenSky] global snapshot attempt failed, retrying:',
error?.message || error,
@@ -71,6 +120,8 @@ async function fetchOpenSkyStates(headers) {
let _openskyTtlMs = OPENSKY_CACHE_MS;
/** @type {number} Epoch-ms before which no upstream fetch is attempted. */
let _openskyCooldownUntil = 0;
let _openskyCooldownStatus = 429;
const _openskySnapshotInFlight = new Map();
/**
* Picks the cache TTL from the remaining daily credit budget.
* Client polls every 30 s, so tiers ≤30 s cost the same 480 credits/h; the
@@ -88,6 +139,10 @@ function openskyAdaptiveTtlMs(remaining) {
}
/** @type {boolean} Guards duplicate auth-failure warnings in logs. */
let _openskyAuthWarned = false;
/** One warning per process when OPENSKY_CREDENTIALS_FILE cannot be read. */
let _openskyCredentialsFileWarned = false;
/** One warning per process when OPENSKY_AUTH_MODE=basic is requested. */
let _openskyBasicRetiredWarned = false;
/** @type {boolean} Guards duplicate invalid-auth-mode warnings. */
let _openskyAuthModeWarned = false;
/** Default auth mode when OPENSKY_AUTH_MODE env is unset. */
@@ -107,6 +162,48 @@ const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
// current instead of coasting a stale worldwide frame indefinitely.
const OPENSKY_SOURCE_STALE_MS = 120_000;
/**
* The OpenSky API client for OAuth: OPENSKY_CLIENT_ID and
* OPENSKY_CLIENT_SECRET, or else the credentials.json that OpenSky's account
* page downloads, named by OPENSKY_CREDENTIALS_FILE ("~" is the home
* directory). The file is read on each token refresh, so a replaced file is
* picked up without a restart.
*
* @param {Record<string, string|undefined>} [env]
* @param {(path: string) => string} [readFile]
* @returns {{ clientId: string, clientSecret: string } | null}
*/
export function openSkyClientCredentials(
env = process.env,
readFile = (path) => fs.readFileSync(path, 'utf8'),
) {
const clientId = String(env.OPENSKY_CLIENT_ID || '').trim();
const clientSecret = String(env.OPENSKY_CLIENT_SECRET || '').trim();
if (clientId && clientSecret) return { clientId, clientSecret };
const file = String(env.OPENSKY_CREDENTIALS_FILE || '').trim();
if (!file) return null;
const path = file.replace(/^~(?=$|[\\/])/, os.homedir());
let raw;
try {
raw = JSON.parse(readFile(path));
} catch {
if (!_openskyCredentialsFileWarned) {
console.warn(
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
);
_openskyCredentialsFileWarned = true;
}
return null;
}
const fileId = String(raw?.clientId ?? raw?.client_id ?? '').trim();
const fileSecret = String(
raw?.clientSecret ?? raw?.client_secret ?? '',
).trim();
return fileId && fileSecret
? { clientId: fileId, clientSecret: fileSecret }
: null;
}
/**
* Obtain a valid OpenSky OAuth2 bearer token, refreshing if needed.
*
@@ -125,9 +222,9 @@ export async function getOpenSkyToken() {
// return the same promise instead of issuing a duplicate token request
if (_openskyTokenPromise) return _openskyTokenPromise;
const clientId = process.env.OPENSKY_CLIENT_ID;
const clientSecret = process.env.OPENSKY_CLIENT_SECRET;
if (!clientId || !clientSecret) return null;
const credentials = openSkyClientCredentials();
if (!credentials) return null;
const { clientId, clientSecret } = credentials;
// Wrap the async token fetch in a shared promise stored in _openskyTokenPromise
_openskyTokenPromise = (async () => {
@@ -368,8 +465,10 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
*
* Supports four auth modes controlled by OPENSKY_AUTH_MODE env:
* - 'oauth' (default) — client_credentials bearer token
* - 'basic' — HTTP Basic with OPENSKY_USERNAME / OPENSKY_PASSWORD
* - 'auto' — try OAuth first, fall back to Basic, then anon
* - 'basic' — retired: OpenSky stopped accepting HTTP Basic (username and
* password) on 2026-03-18, so this requests anonymously and
* warns once
* - 'auto' — try OAuth first, then anon
* - 'anon' — no credentials
*
* Successful responses are cached for OPENSKY_CACHE_MS (~9 s). On
@@ -380,8 +479,12 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
*/
export function openSkyProxy() {
const installMiddleware = (server) => {
server.middlewares.use('/api/opensky', async (req, res) => {
server.middlewares.use('/api/flights', async (req, res, next) => {
try {
// This mount also matches its /track route, served by its own handler.
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
return next();
res.setHeader('X-Feed-Source', 'OpenSky Network');
const requestedMode = normalizeOpenSkyAuthMode(
process.env.OPENSKY_AUTH_MODE,
);
@@ -418,7 +521,9 @@ export function openSkyProxy() {
requestedMode: cachedMeta.requestedMode || requestedMode,
usedMode: cachedMeta.usedMode || 'unknown',
reason: isStale
? 'rate_limited_serving_stale'
? _openskyCooldownStatus === 429
? 'rate_limited_serving_stale'
: `upstream_http_${_openskyCooldownStatus}_serving_stale`
: cachedMeta.reason || 'cached',
staleSeconds: isStale
? (now - _openskyCacheTime) / 1000
@@ -445,80 +550,114 @@ export function openSkyProxy() {
)
return;
res.writeHead(
429,
_openskyCooldownStatus,
buildOpenSkyHeaders({
cacheStatus: 'COOLDOWN',
requestedMode,
usedMode: 'none',
reason: 'rate_limited',
reason:
_openskyCooldownStatus === 429
? 'rate_limited'
: `upstream_http_${_openskyCooldownStatus}`,
retryAfterSeconds: (_openskyCooldownUntil - now) / 1000,
}),
);
res.end(
JSON.stringify({
error: 'OpenSky rate limited; proxy cooling down.',
error: 'OpenSky upstream unavailable; proxy cooling down.',
}),
);
return;
}
const basicUser = process.env.OPENSKY_USERNAME || '';
const basicPass = process.env.OPENSKY_PASSWORD || '';
const hasBasicCreds = Boolean(basicUser && basicPass);
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
const refresh = coalesceProxyRequest(
_openskySnapshotInFlight,
'global',
async () => {
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
if (requestedMode === 'basic') {
if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'basic_credentials';
} else {
reason = 'missing_basic_creds';
}
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'oauth_unavailable_fallback_basic';
} else {
reason = 'missing_oauth_and_basic_creds';
}
}
if (requestedMode === 'basic') {
// OpenSky no longer accepts a Basic header, so sending
// OPENSKY_USERNAME / OPENSKY_PASSWORD would only fail; ask
// anonymously and say once what to configure instead.
if (!_openskyBasicRetiredWarned) {
console.warn(
'[OpenSky] OPENSKY_AUTH_MODE=basic: OpenSky retired username/password sign-in on 2026-03-18, so flights load anonymously (rate-limited). Use OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
);
_openskyBasicRetiredWarned = true;
}
reason = 'basic_retired';
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'missing_oauth_creds';
}
}
let { upstream, body } = await fetchOpenSkyStates(headers);
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
if (
(upstream.status === 401 || upstream.status === 403) &&
requestedMode === 'auto' &&
usedMode === 'oauth' &&
hasBasicCreds
) {
const retryHeaders = {
Accept: 'application/json',
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
};
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
usedMode = 'basic';
reason = 'oauth_rejected_fallback_basic';
}
const { upstream, body } = await fetchOpenSkyStates(headers);
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
const sourceEpochMs = upstream.ok
? openSkySourceEpochMs(body)
: null;
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
return {
upstream,
body,
usedMode,
reason,
sourceEpochMs,
};
},
);
let { upstream, body, usedMode, reason, sourceEpochMs } =
await refresh.promise;
if (
upstream.ok &&
openSkySourceIsStale(sourceEpochMs, now) &&
@@ -529,30 +668,16 @@ export function openSkyProxy() {
'opensky_snapshot_stale_regional_fallback',
))
) {
// Keep the last global snapshot available as a fail-soft cache,
// but do not label or render it as a fresh live result.
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = { requestedMode, usedMode, reason };
return;
}
if (upstream.status === 429) {
reason = 'rate_limited';
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
// 2 min when the header is absent) — no upstream attempts until then.
const retryAfterSec = Number(
upstream.headers.get('x-rate-limit-retry-after-seconds'),
);
const cooldownMs = Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
_openskyCooldownUntil = now + cooldownMs;
if (upstream.status === 429 || upstream.status >= 500) {
reason =
upstream.status === 429
? 'rate_limited'
: `upstream_http_${upstream.status}`;
// Credit governor: fetchOpenSkyStates already started the cooldown
// from OpenSky's retry-after — no upstream attempts until then.
const cooldownMs = Math.max(0, _openskyCooldownUntil - Date.now());
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
@@ -562,7 +687,7 @@ export function openSkyProxy() {
cacheStatus: 'STALE',
requestedMode,
usedMode,
reason: 'rate_limited_serving_stale',
reason: `${reason}_serving_stale`,
staleSeconds: (now - _openskyCacheTime) / 1000,
retryAfterSeconds: cooldownMs / 1000,
}),
@@ -583,35 +708,24 @@ export function openSkyProxy() {
}
if (upstream.status === 401 || upstream.status === 403) {
if (requestedMode === 'basic' && !hasBasicCreds) {
if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth missing. Basic mode requires OPENSKY_USERNAME and OPENSKY_PASSWORD.',
});
reason = 'missing_basic_creds';
} else if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET.',
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'oauth_invalid_or_missing';
} else if (usedMode === 'basic') {
body = JSON.stringify({
error: 'OpenSky auth invalid. Username/password were rejected.',
});
reason = 'basic_invalid_credentials';
} else if (usedMode === 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth client credentials were rejected.',
});
reason = 'oauth_invalid_credentials';
} else if (requestedMode === 'auto' && !hasBasicCreds) {
} else if (requestedMode === 'auto') {
body = JSON.stringify({
error:
'OpenSky auth missing. Provide basic credentials or valid OAuth client credentials.',
'OpenSky auth missing. Provide OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'missing_oauth_and_basic_creds';
reason = 'missing_oauth_creds';
} else {
body = JSON.stringify({
error: 'OpenSky auth required.',
@@ -620,45 +734,6 @@ export function openSkyProxy() {
}
}
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'basic' &&
reason === 'basic_credentials'
) {
reason = 'basic_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
res.writeHead(
upstream.status,
buildOpenSkyHeaders({
server/providers/aircraft/openskyCredentials.test.mjs
+81
View File
@@ -0,0 +1,81 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import os from 'node:os';
import path from 'node:path';
import { openSkyClientCredentials } from './opensky.js';
const file = (contents) => (filePath) => {
file.read = filePath;
if (contents instanceof Error) throw contents;
return contents;
};
test('OpenSky client env vars come first', () => {
const read = () => assert.fail('the file is not read when env vars are set');
assert.deepEqual(
openSkyClientCredentials(
{
OPENSKY_CLIENT_ID: ' id ',
OPENSKY_CLIENT_SECRET: 'secret',
OPENSKY_CREDENTIALS_FILE: 'credentials.json',
},
read,
),
{ clientId: 'id', clientSecret: 'secret' },
);
});
test('OPENSKY_CREDENTIALS_FILE supplies the client when env vars are empty', () => {
const env = {
OPENSKY_CLIENT_ID: '',
OPENSKY_CREDENTIALS_FILE: '~/Downloads/credentials.json',
};
// The file OpenSky's account page downloads uses camelCase keys.
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"clientId":"cid","clientSecret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
assert.equal(
path.normalize(file.read),
path.normalize(path.join(os.homedir(), 'Downloads/credentials.json')),
);
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"client_id":"cid","client_secret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
});
test('a missing, unreadable or incomplete credentials file means no client', () => {
assert.equal(openSkyClientCredentials({}, file('{}')), null);
const env = { OPENSKY_CREDENTIALS_FILE: 'credentials.json' };
const missing = Object.assign(new Error('nope'), { code: 'ENOENT' });
assert.equal(openSkyClientCredentials(env, file(missing)), null);
assert.equal(openSkyClientCredentials(env, file('not json')), null);
assert.equal(openSkyClientCredentials(env, file('{"clientId":"cid"}')), null);
});
test('credential file warnings never expose file contents or read errors', async (t) => {
const warnings = [];
t.mock.method(console, 'warn', (message) => warnings.push(message));
for (const contents of [
'{"clientId":"id","clientSecret":FAKE_SECRET_123}',
new Error('could not read FAKE_SECRET_123'),
]) {
const { openSkyClientCredentials: resolve } = await import(
`./opensky.js?warning=${warnings.length}`
);
const env = { OPENSKY_CREDENTIALS_FILE: '/synthetic/credentials.json' };
assert.equal(resolve(env, file(contents)), null);
assert.equal(resolve(env, file(contents)), null);
}
assert.deepEqual(warnings, [
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
]);
});
server/providers/aircraft/tracks.js
+16 -11
View File
@@ -3,14 +3,13 @@ import { readCappedResponseText } from '../common/http.js';
/**
* Vite plugin: aircraft track-history backfill proxies (PRD WS-F F1/F2).
*
* /api/opensky-track?icao24=<hex6> — OpenSky GET /tracks/all (experimental;
* /api/flights/track?icao24=<hex6> — OpenSky GET /tracks/all (experimental;
* own credit bucket, 4 credits per call on the free tier). OAuth via the
* shared coalesced token. 60s per-icao cache; 404/429 forwarded so the
* client can fall back to its accumulated trail silently.
* /api/adsblol/trace?hex=<hex> — adsb.lol tar1090 readsb trace
* (undocumented but live; no browser CORS, hence this proxy). Up to ~24h
* of real history per aircraft. Treat as best-effort; data is ODbL —
* credit "adsb.lol (ODbL)" in the UI.
* /api/military/track?hex=<hex> — compatibility refusal for the retired,
* undocumented ADSB.lol trace route. The standalone military source uses
* locally accumulated trails; provider-supplied history remains optional.
*/
export function trackBackfillProxies() {
const TRACK_CACHE_MS = 60000;
@@ -65,7 +64,7 @@ export function trackBackfillProxies() {
}
function install(middlewares) {
middlewares.use('/api/opensky-track', async (req, res) => {
middlewares.use('/api/flights/track', async (req, res) => {
try {
const incoming = new URL(req.url || '', 'http://localhost');
const icao24 = String(incoming.searchParams.get('icao24') || '')
@@ -93,7 +92,7 @@ export function trackBackfillProxies() {
}
});
middlewares.use('/api/adsblol/trace', async (req, res) => {
middlewares.use('/api/military/track', async (req, res) => {
try {
const incoming = new URL(req.url || '', 'http://localhost');
const hex = String(incoming.searchParams.get('hex') || '')
@@ -107,10 +106,16 @@ export function trackBackfillProxies() {
);
return;
}
await proxyJson(
res,
`lol:${hex}`,
`https://adsb.lol/data/traces/${hex.slice(-2)}/trace_full_${hex}.json`,
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Retry-After', '1800');
res.end(
JSON.stringify({
error:
'Military history unavailable; locally accumulated trails remain available',
code: 'history_unavailable',
}),
);
} catch (error) {
res.statusCode = 502;
server/providers/cctv.js
+81 -34
View File
@@ -1,11 +1,9 @@
import { createCctvCatalog } from './cctv/catalog.js';
import {
normalizeFeedType,
isVideoFeedType,
toFiniteNumber,
} from './cctv/normalize.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
cctvLiveMediaType,
cctvRasterImageType,
proxyMediaResponse,
fetchCctvImageFromUpstream,
fetchTxdotSnapshot,
@@ -17,6 +15,8 @@ import {
CCTV_MAX_SOURCES_CEILING,
} from './cctv/constants.js';
import { sanitizeCctvRangeHeader } from './cctv/range.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
import { createHlsPuller } from './cctv/stream.js';
import { googleServerApiKey } from './places/google-key.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
@@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
* GET /api/cctv/sources — list all registered camera sources
* GET /api/cctv/health — per-camera health/status report
* GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera
* GET /api/cctv/media/:id — proxy live video/image media from upstream
* GET /api/cctv/media/:id — proxy live video media from upstream
* GET /api/cctv/frame/:id — single frame with fallback chain
*
* @returns {import('vite').Plugin}
@@ -116,8 +116,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' },
signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS),
});
const svType = svResp.headers.get('content-type') || '';
if (!svResp.ok || !svType.startsWith('image/')) return null;
const svType = cctvRasterImageType(svResp.headers.get('content-type'));
if (!svResp.ok || !svType) {
try {
await svResp.body?.cancel();
} catch {
/* no-op */
}
return null;
}
return {
ok: true,
@@ -135,15 +142,19 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
});
server.middlewares.use('/api/cctv', async (req, res) => {
try {
const url = new URL(req.url || '/', 'http://localhost');
// Refuse cross-site requests before any catalog or upstream work. The
// keyless gate keeps the routes reachable behind a reverse proxy; the
// key-bearing Street View fallback below has its own check.
if (admitSameSiteKeyless(req, res)) return;
const sources = await getCctvSources();
const sourceById = new Map(
sources.map((source) => [source.id, source]),
);
const url = new URL(req.url || '/', 'http://localhost');
if (url.pathname === '/sources') {
const body = {
trimmedPacks: getCctvSources.trimmedPacks?.() ?? [],
trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [],
sources: sources.map((source) => ({
id: source.id,
pack: source.pack,
@@ -164,6 +175,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
poseSource: source.poseSource,
model: source.model || '',
manufacturer: source.manufacturer || '',
license: source.license,
credit: source.credit || '',
code: source.code || '',
@@ -214,6 +227,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const source = sourceById.get(cameraId);
const mediaUrl = source?.url || '';
const feedType = normalizeFeedType(source?.feedType || 'image');
// Live feeds only. A still camera is served by /frame, whose fetch
// keeps redirects on the registered origin, accepts only raster
// images and caps the body; relaying it here would bypass all three.
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
res.writeHead(404, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(
JSON.stringify({
error: 'No live media for this camera; use the frame endpoint',
}),
);
return;
}
const leaseId = url.searchParams.get('lease');
if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) {
res.writeHead(400);
@@ -365,7 +393,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
}
return;
}
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok) {
try {
await upstream.body?.cancel();
@@ -390,31 +417,42 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return;
}
if (
isVideoFeedType(feedType) &&
!(
contentType.startsWith('video/') ||
contentType.includes('mpegurl')
)
) {
// The relayed type is the feed kind's own, checked before the
// camera counts as healthy and before any byte is relayed.
const contentType = cctvLiveMediaType(
feedType,
upstream.headers.get('content-type'),
);
if (!contentType) {
try {
await upstream.body?.cancel();
} catch {
/* already closed */
}
setHealth(cameraId, {
status: 'degraded',
sourceKind: 'upstream',
label: source?.provider || 'Configured source',
message: `Unexpected media type ${contentType || 'unknown'}`,
message: 'Unexpected media type',
});
} else {
setHealth(cameraId, {
status: 'ok',
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
label: source?.provider || 'Configured source',
message: isVideoFeedType(feedType)
? 'Live stream connected'
: 'Snapshot feed connected',
res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'Media proxy failed' }));
return;
}
setHealth(cameraId, {
status: 'ok',
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
label: source?.provider || 'Configured source',
message: isVideoFeedType(feedType)
? 'Live stream connected'
: 'Snapshot feed connected',
});
await proxyMediaResponse(res, upstream, {
contentType,
sourceHeader: isVideoFeedType(feedType)
? 'live-media'
: 'upstream-image',
@@ -507,13 +545,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return;
}
const sv = await streetViewFallback({
lat,
lon,
heading,
fov,
pitch,
});
// The Street View fallback spends the Google key, so a forwarded
// request (whose client this server cannot see) gets the placeholder,
// as if no key were configured. Any forwarding header counts, even an
// empty one.
const forwarded = PROXY_SIGNALS.some(
(name) => req.headers?.[name] !== undefined,
);
const sv = forwarded
? null
: await streetViewFallback({
lat,
lon,
heading,
fov,
pitch,
});
if (sv?.ok) {
setHealth(cameraId, {
status: 'degraded',
server/providers/cctv/catalog.js
+114 -15
View File
@@ -4,6 +4,8 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import { loadTaiwanSources } from './taiwan.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -17,18 +19,32 @@ import {
loadWarendorfSourcesFromCatalog,
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadHkSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
loadWsdotSourcesFromOpenData,
loadVegvesenSourcesFromOpenData,
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
loadFloridaSourcesFromOpenData,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
/** Env opt-in: only "1" enables. For packs whose feed access needs an
* operator agreement or whose terms are unconfirmed, so an unset flag never
* reaches the upstream. */
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
/**
* Live open-data packs, in merge order. Adding a region is one entry here
* plus its loader in sources.js; the catalog cap is shared across entries
* round-robin (cap.js), so a new pack never silently evicts an older one.
* plus its loader (in sources.js or its own module); the catalog cap is
* shared across entries round-robin (cap.js), so a new pack never silently
* evicts an older one.
* Each pack fails independently (allSettled) and is gated by its own env
* kill switch.
* kill switch, or by an opt-in flag where access needs an agreement or the
* terms are unconfirmed.
*/
const LIVE_PACKS = [
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
@@ -87,11 +103,68 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
load: loadCalgarySourcesFromOpenData,
},
{
name: 'catalonia',
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'hk',
enabled: () => envEnabled('CCTV_HK_ENABLED'),
load: loadHkSourcesFromOpenData,
},
{
name: 'deldot',
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
load: loadDelDOTSourcesFromOpenData,
},
{
name: 'kingcounty',
enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'),
load: loadKingCountySourcesFromOpenData,
},
{
name: 'wsdot',
enabled: () => envEnabled('CCTV_WSDOT_ENABLED'),
load: loadWsdotSourcesFromOpenData,
},
{
name: 'vegvesen',
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
load: loadVegvesenSourcesFromOpenData,
},
{
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
// off unless the operator opts in (see DATA_SOURCES.md).
name: 'nyc',
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
load: loadNycSourcesFromOpenData,
},
{
name: 'toronto',
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
load: loadTorontoSourcesFromOpenData,
},
{
// Opt-in until the Via Lietuva open-data terms are confirmed to cover the
// camera photos (see DATA_SOURCES.md).
name: 'lithuania',
enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
load: loadLithuaniaSources,
},
{
// FL511's terms limit its content to individual, personal use (other use
// needs FDOT's written consent), so the pack is off unless the operator
// opts in (see DATA_SOURCES.md).
name: 'florida',
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
load: loadFloridaSourcesFromOpenData,
},
{
name: 'taiwan',
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
load: loadTaiwanSources,
},
];
/**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
@@ -143,16 +216,19 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
/** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
* callers so a post-TTL burst launches ONE refetch, not one per request. */
let _cctvSourceInflight = null;
/** @type {Array<{pack: string, available: number, served: number}>} Packs
* that serve fewer cameras than they offer. */
let _cctvTrimmedPacks = [];
/** @type {WeakMap<Array<object>, Array<{pack: string, available: number, served: number}>>}
* Packs that serve fewer cameras than they offer, keyed by the source list
* they describe, so a request served the old list during a background
* refresh never pairs it with the new list's trim notes. */
const _cctvTrimmedPacks = new WeakMap();
/**
* Assemble and cache the merged CCTV source list.
*
* Merges every source pack (live open-data packs, local file, env
* variable), deduplicates by ID, shares the catalog cap fairly across
* packs, and caches for CCTV_SOURCE_CACHE_MS.
* packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is
* returned while a background refresh replaces it; only a first load waits.
*
* @returns {Promise<Array<object>>} Deduplicated, capped source list.
*/
@@ -167,11 +243,31 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
// Single-flight: a burst of requests arriving past the TTL shares ONE refresh
// instead of each launching the full multi-provider refetch. The `.finally`
// clears the ref so the next post-TTL cycle starts fresh.
if (_cctvSourceInflight) return _cctvSourceInflight;
_cctvSourceInflight = refreshCctvSources().finally(() => {
_cctvSourceInflight = null;
});
return _cctvSourceInflight;
if (!_cctvSourceInflight) {
_cctvSourceInflight = refreshCctvSources().finally(() => {
_cctvSourceInflight = null;
});
// A background refresh has no caller waiting on it: report its failure
// here, once per refresh rather than once per request it serves. Keep
// the old catalog for another TTL, as an empty refresh does, so a refresh
// that keeps failing is retried once per TTL, not back to back.
if (_cctvSourceCache.length) {
_cctvSourceInflight.catch((error) => {
_cctvSourceCacheAt = Date.now();
console.warn(
'[CCTV] background catalog refresh failed; serving',
_cctvSourceCache.length,
'cameras from the last catalog:',
error?.message || error,
);
});
}
}
// With a catalog in hand, answer with it while the refresh runs: the
// refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up
// to its 30 s timeout), and every /api/cctv request would otherwise wait
// for it. Only the first load, with nothing to serve yet, waits.
return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight;
}
/**
@@ -261,7 +357,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
}
if (capped.length > 0 || _cctvSourceCache.length === 0) {
_cctvSourceCache = capped;
_cctvTrimmedPacks = trimmedPacks;
_cctvTrimmedPacks.set(capped, trimmedPacks);
} else {
// Every source came back empty (all live packs timed out / upstream outage)
// but a good catalog is already cached — serve it stale rather than blanking
@@ -275,7 +371,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
return _cctvSourceCache;
}
/** Packs serving fewer cameras than they offer, as of the last refresh. */
getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
/** Packs serving fewer cameras than they offer in `sources` (by default the
* current catalog). Pass the list a request was served to keep the two in
* step while a background refresh swaps the catalog. */
getCctvSources.trimmedPacks = (sources = _cctvSourceCache) =>
_cctvTrimmedPacks.get(sources) ?? [];
return getCctvSources;
}
server/providers/cctv/constants.js
+292 -4
View File
@@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. Sized above the sum of the
* default per-pack caps so a default install never trims.
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
@@ -18,7 +18,7 @@ export const CCTV_MAX_SOURCES_CEILING = 5000;
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
const TXDOT_ORIGIN = 'https://its.txdot.gov';
export const TXDOT_CCTV_STATUS_URL = (district) =>
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
@@ -113,13 +113,34 @@ export const CALTRANS_ANCHORS = [
{ lat: 32.7157, lon: -117.1611 }, // San Diego
{ lat: 38.5816, lon: -121.4944 }, // Sacramento
];
export const FLORIDA_CCTV_URL =
'https://services.arcgis.com/3wFbqsFPLeKqOlIK/arcgis/rest/services/FL511_Traffic_Cameras/FeatureServer/0/query';
export const FLORIDA_IMAGE_HOSTS = [
'images-dis.divas.cloud',
'snapshots.divas.cloud',
];
export const DEFAULT_FLORIDA_MAX_SOURCES = 500;
/** Byte cap per catalog page (2,000 rows, about 650 KiB measured). */
export const FLORIDA_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Catalog page ceiling: 40 pages covers 4,000+ cameras even at 100 rows a page. */
export const FLORIDA_MAX_CATALOG_PAGES = 40;
export const FLORIDA_ANCHORS = [
{ lat: 25.7617, lon: -80.1918 }, // Miami
{ lat: 28.5384, lon: -81.3789 }, // Orlando
{ lat: 27.9506, lon: -82.4572 }, // Tampa
{ lat: 30.3322, lon: -81.6557 }, // Jacksonville
{ lat: 30.4383, lon: -84.2807 }, // Tallahassee
{ lat: 26.1224, lon: -80.1373 }, // Fort Lauderdale
{ lat: 26.7153, lon: -80.0534 }, // West Palm Beach
{ lat: 30.4213, lon: -87.2169 }, // Pensacola (panhandle)
];
/** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */
export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam';
export const TFL_IMAGE_ORIGIN =
'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/';
export const DEFAULT_TFL_MAX_SOURCES = 250;
export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 };
/** Ontario 511: keyless CARS/511 camera catalog; frame URLs are still images. */
/** Ontario 511: CARS/511 camera catalog, which needs a developer key (ONTARIO_511_API_KEY); frame URLs are still images. */
export const ONTARIO_511_CAMERAS_URL =
'https://511on.ca/api/v2/get/cameras?format=json&lang=en';
export const ONTARIO_511_IMAGE_ORIGIN = 'https://511on.ca/map/Cctv/';
@@ -188,6 +209,8 @@ export const TARKTEE_IMAGES_URL =
'https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages';
export const TARKTEE_IMAGE_ORIGIN = 'https://tarktee.transpordiamet.ee/images/';
export const DEFAULT_TARKTEE_MAX_SOURCES = 179;
/** Hard ceiling per Tarktee DATEX XML document (~120–165 KB live). */
export const TARKTEE_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
export const TARKTEE_ANCHORS = [
{ lat: 59.437, lon: 24.753 }, // Tallinn
{ lat: 58.378, lon: 26.729 }, // Tartu
@@ -236,6 +259,190 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 };
* body cannot be buffered without limit. */
export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/**
* Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML
* XML list (verified 2026-09-15, hourly catalog refresh), fetched over https
* (verified 2026-09-27: same 200 body, no redirect). The feed carries a
* `font` field naming the actual publisher per camera — SCT's own highway
* cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are
* kept, each confirmed to return a real frame. Frame URLs are checked
* against a host allowlist rather than one pinned origin, since each
* publisher serves from its own host.
*/
export const CATALONIA_CAMERAS_URL =
'https://www.gencat.cat/transit/opendata/cameres.xml';
/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27),
* so a runaway body cannot be buffered without limit. */
export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024;
/** Allowed image hosts, one per kept `font` publisher in the feed, each
* mapped to the one directory its frame links live in (every link in the
* 2026-10 feed shares it); a link must name a single file directly inside. */
export const CATALONIA_IMAGE_HOSTS = Object.freeze(
new Map([
['mct.gencat.cat', '/mct2bo/'], // SCT
['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona)
['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa
]),
);
/** SCT `sctidcam` values: a bare image filename, which also keys the camera
* id (the feed's `fid` embeds an upstream server-start stamp). */
export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i;
/** Per-`font` partner credit, shown beside the provider in the CCTV panel
* (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's
* own publisher and carries no separate credit. */
export const CATALONIA_FONT_CREDIT = Object.freeze({
IMI: 'Ajuntament de Barcelona',
Terrassa: 'Ajuntament de Terrassa',
});
/** Ground-elevation priors in metres, by `font`: the feed carries no
* elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau
* (~270 m). The client's ground snap corrects these on 3D-tile stacks. */
export const CATALONIA_FONT_ELEVATION_M = Object.freeze({
SCT: 80,
IMI: 20,
Terrassa: 270,
});
export const DEFAULT_CATALONIA_MAX_SOURCES = 150;
/** Prioritization anchors: Barcelona and Terrassa. */
export const CATALONIA_ANCHORS = [
{ lat: 41.3874, lon: 2.1686 }, // Barcelona
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
export const DEFAULT_HK_CAMERAS_URL =
'https://static.data.gov.hk/td/traffic-snapshot-images/code/Traffic_Camera_Locations_En.xml';
/** The only origin HK camera frames may come from. Verified 2026-10-11: still
* URLs answer 200 on this host with no redirect hop. */
export const HK_IMAGE_ORIGIN = 'https://tdcctv.data.one.gov.hk/';
/** Above the full TD English catalog (~1,013 cameras) so a default install
* keeps nearly all of them. Catalog-wide round-robin still applies when the
* sum of packs exceeds DEFAULT_CCTV_MAX_SOURCES (Norway-style). */
export const DEFAULT_HK_MAX_SOURCES = 1100;
/** Prioritization anchors across Hong Kong Island, Kowloon and the New
* Territories so a lowered cap keeps territory-wide coverage rather than
* only the harbour corridor. */
export const HK_ANCHORS = [
{ lat: 22.2819, lon: 114.158 }, // Central
{ lat: 22.2976, lon: 114.1722 }, // Tsim Sha Tsui
{ lat: 22.3827, lon: 114.19 }, // Sha Tin
{ lat: 22.3701, lon: 114.114 }, // Tsuen Wan
{ lat: 22.4503, lon: 114.1688 }, // Tai Po
{ lat: 22.493, lon: 114.139 }, // Fanling
{ lat: 22.391, lon: 113.977 }, // Tuen Mun
{ lat: 22.4445, lon: 114.022 }, // Yuen Long
{ lat: 22.307, lon: 114.26 }, // Tseung Kwan O
];
/** Hard ceiling on the locations XML body. The live English list is ~400 KB
* for ~1,000 cameras; this only exists so an unbounded upstream cannot be
* buffered without limit. */
export const HK_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
* country (~900 cameras). The raw DATEX node needs an account; this view does
* not. Frames are stills on the agency's camera host. */
export const DEFAULT_VEGVESEN_CCTV_URL =
'https://ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items?f=application/geo%2Bjson&limit=5000';
/** The only origin Vegvesen camera frames may come from. */
export const VEGVESEN_IMAGE_ORIGIN =
'https://kamera.atlas.vegvesen.no/api/images/';
/** Live HLS for the ~140 cameras that publish video; only this exact
* per-camera manifest path is registered. */
export const VEGVESEN_VIDEO_URL = (cameraId) =>
`https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`;
/** Above the ~850 working cameras so the pack keeps them all on its own. The
* catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is
* filled round-robin, so a full default install may thin this pack. */
export const DEFAULT_VEGVESEN_MAX_SOURCES = 900;
/** Hard ceiling on the catalog body. The whole country is ~800 KB. */
export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024;
/** Prioritization anchors: the largest cities from south to north, so a cap
* keeps national coverage rather than just the capital region. */
export const NORWAY_ANCHORS = [
{ lat: 59.9139, lon: 10.7522 }, // Oslo
{ lat: 60.3913, lon: 5.3221 }, // Bergen
{ lat: 58.97, lon: 5.7331 }, // Stavanger
{ lat: 63.4305, lon: 10.3951 }, // Trondheim
{ lat: 58.1467, lon: 7.9956 }, // Kristiansand
{ lat: 67.2804, lon: 14.4049 }, // Bodø
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
];
/** Kartverket's keyless point-height API (CC BY 4.0): terrain height above
* sea level (NN2000), used as the Vegvesen cameras' ground prior because the
* camera feed's points are 2D. */
export const KARTVERKET_HEIGHT_URL =
'https://ws.geonorge.no/hoydedata/v1/punkt';
/** Points per request. The API takes up to 50, but above ~20 it answers the
* extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not
* at all; at 20 nearly every camera resolves from the 1 m terrain model. */
export const KARTVERKET_MAX_POINTS = 20;
/** Requests in flight at once. The whole country is ~36 requests of
* 0.6-9.5 s each, so a cold lookup takes ~6-13 s. */
export const KARTVERKET_CONCURRENCY = 8;
/** Deadline for one complete lookup, shared by every request. A lookup cut
* short by it is resumed on the next refresh, without a backoff. */
export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000;
/** One request that takes longer than this counts as a failing service. */
export const KARTVERKET_REQUEST_TIMEOUT_MS = 10 * 1000;
/** The Vegvesen lane's budget, measured from the start of the loader (catalog
* fetch included), for waiting on heights. Under the 15 s per-provider fetch
* timeout so this pack is never the catalog's slowest lane. A lookup still
* running after it keeps filling the cache for the next refresh. */
export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000;
/** The lane only waits when more than this share of its cameras has no
* answer yet (a cold cache). A few new cameras or expired misses are looked
* up in the background instead of holding the refresh. */
export const KARTVERKET_WAIT_MISSING_SHARE = 0.1;
/** A point Kartverket has no height for (some ferry quays and bridges) is not
* asked about again for this long. */
export const KARTVERKET_NULL_TTL_MS = 6 * 60 * 60 * 1000;
/** After a failing request (408, 429, 5xx, a network error, a request
* timeout or an unreadable body), no lookup starts for this long, or for the
* Retry-After the service sent, bounded to [1 min, 1 h]. Longer than the
* 15-minute catalog refresh (CCTV_SOURCE_CACHE_MS), so a failing service
* costs one attempt per backoff, not one per refresh. */
export const KARTVERKET_FAILURE_BACKOFF_MS = 30 * 60 * 1000;
export const KARTVERKET_MAX_BACKOFF_MS = 60 * 60 * 1000;
export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024;
/** Plausible Norwegian terrain: below sea level only at the shore, and no road
* camera sits above ~1,500 m. Anything outside is treated as no height. */
export const KARTVERKET_HEIGHT_RANGE_M = Object.freeze({ min: -20, max: 2500 });
/** Same identity the other CCTV upstream requests send. */
export const KARTVERKET_USER_AGENT = 'gods-eye-view-cctv-proxy/1.0';
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
* five boroughs (~980 cameras). Frames are stills synthesized from each
* camera UUID on the city's webcam host — never taken from the payload. */
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
* is ignored; the registered path is always this origin plus a validated
* UUID plus `/image`. */
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
export const NYC_CAMERA_ID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const DEFAULT_NYC_MAX_SOURCES = 250;
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
* this only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
export const NYC_GROUND_ELEVATION_M = 10;
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
* rather than Midtown alone. */
export const NYC_ANCHORS = [
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
{ lat: 40.7282, lon: -73.7949 }, // Queens
{ lat: 40.8448, lon: -73.8648 }, // Bronx
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
@@ -244,6 +451,83 @@ export const DELDOT_ANCHORS = [
{ lat: 39.1582, lon: -75.5244 }, // Dover (Kent)
{ lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex)
];
/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer
* ("King County and WSDOT Traffic Cameras", King County DOT Road Services).
* GeoJSON with WGS84 point geometry. `outFields` is an explicit list on
* purpose: the layer also carries asset-administration columns (including
* staff names in `ModifiedBy`) that must never be fetched. Uniquely among the
* packs, the county publishes each camera's hardware `Manufacturer`/`Model`
* (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the
* source's `model`. */
export const KING_COUNTY_CAMERAS_URL =
'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson';
/** Frames come only from the county's own image handler host. */
export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/';
/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by
* default. */
export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125;
/** Prioritization anchor: Renton, roughly central to the county road network. */
export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }];
/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows
* and well under 100 KB; this only exists so an upstream that streams an
* unbounded body cannot be buffered without limit. */
export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024;
/**
* City of Toronto RESCU traffic cameras: the municipal network the province's
* 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen
* and Lake Shore. One keyless list for the whole city (~336 cameras), from the
* City's open data portal. It is fetched on every catalog refresh, but the
* upstream file is not updated often and may lag the live camera network.
*
* The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the
* portal publishes the same data as .xml and .csv beside it.
*/
export const TORONTO_RESCU_LIST_URL =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json';
/**
* The only origin Toronto camera frames may come from. The list carries no URL
* field: frame URLs are BUILT from this origin and a strictly-digits camera
* number (`loc####.jpg`, the convention the dataset documents), so no upstream
* field can steer the frame proxy off-host.
*/
export const TORONTO_RESCU_IMAGE_ORIGIN =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/';
/** The whole city fits under this; the cap exists to stay tunable, not to trim. */
export const DEFAULT_TORONTO_MAX_SOURCES = 336;
/** Yonge / Queen: the prioritization anchor. */
export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 };
/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single
* number fits: this is the mean across the 336 camera sites. The ground-height
* sidecar has no Toronto entries yet, so this prior seeds placement; a camera
* that gains a sidecar entry uses it instead, and the client's ground snap
* corrects the prior wherever 3D tiles are loaded. */
export const TORONTO_GROUND_ELEVATION_M = 116;
/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under
* 60 KB; this only exists so an upstream that streams an unbounded body cannot
* be buffered without limit. */
export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the
* official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator),
* ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use;
* the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */
export const WSDOT_CAMERAS_URL =
'https://data.wsdot.wa.gov/travelcenter/Cameras.json';
/** Only frames on WSDOT's own image host are registered; the catalog also
* lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports)
* whose own terms apply. */
export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/';
export const DEFAULT_WSDOT_MAX_SOURCES = 250;
/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this
* only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Prioritization anchors: downtown Seattle and Spokane. */
export const WSDOT_ANCHORS = [
{ lat: 47.6062, lon: -122.3321 }, // Seattle
{ lat: 47.6588, lon: -117.426 }, // Spokane
];
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */
export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one
@@ -251,6 +535,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
* pending forever — a hung fetch aborts, the loader returns [], and
* serve-stale/other packs take over. */
export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000;
/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across
* ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian
* pack on about a third of loads. Both must arrive for any camera to load. */
export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000;
/** Individual CCTV image fetches must settle before the active 10-second
* client refresh cadence. A bounded miss can fall through to Street View or
* the synthetic frame instead of leaving the browser preview pending. */
server/providers/cctv/groundHeights.js
+1 -1
View File
@@ -3,7 +3,7 @@ import path from 'node:path';
import { poseHash } from '../../../src/data/cctvFootprint.js';
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
export const DEFAULT_GROUND_HEIGHTS_FILE =
const DEFAULT_GROUND_HEIGHTS_FILE =
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
server/providers/cctv/lithuania.js
+154
View File
@@ -0,0 +1,154 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseJsonCapped } from '../common/http.js';
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
// Without lks=true this endpoint returns [latitude, longitude]. The camera
// table's x/y fields use LKS-94 metres and must never be treated as degrees.
export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`;
export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last';
const ANCHORS = [
{ lat: 54.6872, lon: 25.2797 }, // Vilnius
{ lat: 54.8985, lon: 23.9036 }, // Kaunas
{ lat: 55.7033, lon: 21.1443 }, // Klaipeda
{ lat: 55.9349, lon: 23.3137 }, // Siauliai
{ lat: 55.7348, lon: 24.3575 }, // Panevezys
];
/** Join public catalogs by ID; register only the official still endpoint. */
export function lithuaniaCameraToSource(row, feature) {
const id = String(row?.id ?? '');
if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null;
const point = feature?.points?.[0]?.point;
if (!Array.isArray(point) || point.length !== 2) return null;
const [lat, lon] = point;
if (
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 53.8 ||
lat > 56.5 ||
lon < 20.8 ||
lon > 26.9
)
return null;
try {
const url = new URL(row.image);
if (
url.origin !== 'https://eismoinfo.lt' ||
url.username ||
url.password ||
url.pathname !== FRAME_PATH ||
url.searchParams.get('id') !== id
)
return null;
} catch {
return null;
}
// Reconstruct so extra query parameters cannot alter the request.
const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`;
const cameraId = `lithuania-${id}`;
const name = String(row.name || feature.name || `Lietuva ${id}`)
.trim()
.slice(0, 160);
return {
id: cameraId,
name,
city: 'Lithuania / Lietuva',
cityId: 'lithuania',
provider: 'Via Lietuva / Eismoinfo',
lat,
lon,
// No calibrated pose is published; use the app's low-confidence prior.
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'eismoinfo-public',
credit:
'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills',
code: cameraDisplayCode(name.toUpperCase()),
};
}
async function readCatalog(url, sibling) {
const signal = AbortSignal.any([
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
sibling.signal,
]);
try {
const response = await fetch(url, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `Catalog HTTP ${response.status}`,
);
}
return await readResponseJsonCapped(
response,
LITHUANIA_MAX_CATALOG_BYTES,
signal,
);
} catch (error) {
// The join needs both catalogs; release the other request at once
// rather than leaving its body open until the timeout.
sibling.abort();
throw error;
}
}
/** Two bounded requests per shared catalog refresh, no API key required. */
export async function loadLithuaniaSources() {
try {
const sibling = new AbortController();
const [rows, layers] = await Promise.all([
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
]);
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
if (!Array.isArray(features)) return [];
const locations = new Map(
features.filter(Boolean).map((feature) => [String(feature.id), feature]),
);
const cameras = new Map();
for (const row of rows) {
const camera = lithuaniaCameraToSource(
row,
locations.get(String(row?.id)),
);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400);
const cap = Number.isFinite(rawCap)
? Math.max(1, Math.min(1000, Math.floor(rawCap)))
: 400;
const sources = prioritizeSources([...cameras.values()], cap, ANCHORS);
console.log(
`[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`,
);
return sources;
} catch (error) {
console.warn(
'[CCTV] Lithuania camera download error:',
error?.message || error,
);
return [];
}
}
server/providers/cctv/media.js
+131 -10
View File
@@ -1,13 +1,21 @@
import { Readable } from 'node:stream';
import { hashSeed, escapeXml } from './normalize.js';
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
import {
CALGARY_IMAGE_ORIGIN,
CCTV_FRAME_FETCH_TIMEOUT_MS,
CCTV_FRAME_MAX_BODY_BYTES,
CCTV_MEDIA_FETCH_TIMEOUT_MS,
CCTV_MEDIA_IDLE_TIMEOUT_MS,
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
} from './constants.js';
/**
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
@@ -84,7 +92,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
* @param {ReadableStream|NodeJS.ReadableStream|null} body
* @returns {import('stream').Readable|null}
*/
export function toReadable(body) {
function toReadable(body) {
if (!body) return null;
if (typeof body.pipe === 'function') return body;
if (typeof body.getReader === 'function') {
@@ -93,6 +101,73 @@ export function toReadable(body) {
return null;
}
/**
* Split a Content-Type value into its lower-case essence and parameters.
*
* @param {string|null|undefined} value
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
* value is missing or is not a `type/subtype` media type.
*/
function parseMediaType(value) {
const [head, ...rest] = String(value ?? '').split(';');
const essence = head.trim().toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
const params = new Map();
for (const part of rest) {
const eq = part.indexOf('=');
if (eq <= 0) continue;
const name = part.slice(0, eq).trim().toLowerCase();
let paramValue = part.slice(eq + 1).trim();
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
if (name && !params.has(name)) params.set(name, paramValue);
}
return { essence, params };
}
/** Still-image types a camera frame may be served as. */
const CCTV_RASTER_IMAGE_TYPES = new Set([
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'image/avif',
]);
/**
* The canonical type of a raster camera frame, or null for any other type.
*
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvRasterImageType(contentType) {
const essence = parseMediaType(contentType)?.essence;
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
}
/**
* The type to relay a live feed as, or null when the upstream type does not
* match the feed kind. MJPEG keeps its part boundary, without which the body
* cannot be split into frames.
*
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvLiveMediaType(feedType, contentType) {
const parsed = parseMediaType(contentType);
if (!parsed) return null;
if (feedType === 'mp4' || feedType === 'webm')
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
return null;
const boundary = parsed.params.get('boundary') || '';
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
return null;
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
? `multipart/x-mixed-replace; boundary=${boundary}`
: `multipart/x-mixed-replace; boundary="${boundary}"`;
}
/**
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
*
@@ -104,6 +179,8 @@ export function toReadable(body) {
* @param {Response} upstream - fetch() Response object.
* @param {object} [opts]
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
* @param {string} [opts.contentType] - Validated type to send instead of the
* upstream's own.
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
* allowed between upstream chunks before the stream is released. Injectable
* only to keep the deadline unit-testable.
@@ -114,10 +191,10 @@ export async function proxyMediaResponse(
{
sourceHeader = 'upstream',
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
contentType = upstream.headers.get('content-type') ||
'application/octet-stream',
} = {},
) {
const contentType =
upstream.headers.get('content-type') || 'application/octet-stream';
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
const contentLength = upstream.headers.get('content-length');
const contentRange = upstream.headers.get('content-range');
@@ -340,7 +417,14 @@ export async function fetchCctvMediaUpstream(
if (downstream?.aborted) controller.abort();
else downstream?.addEventListener?.('abort', onDownstreamAbort);
try {
return await fetchImpl(url, { headers, signal: controller.signal });
// Redirects stay on the registered origin, as for frames.
const upstream = await fetchWithinHost(
url,
{ headers, signal: controller.signal },
fetchImpl,
);
if (!upstream) throw new Error('Media redirect refused');
return upstream;
} finally {
clearTimeout(timeoutId);
downstream?.removeEventListener?.('abort', onDownstreamAbort);
@@ -428,9 +512,40 @@ export async function fetchTxdotSnapshot(
/** Redirect hops the frame path will follow, and only within the same host. */
const MAX_SAME_HOST_REDIRECTS = 2;
/**
* Directories camera catalogs are pinned to, by host. A redirect on one of
* these hosts must stay inside the pinned directory too, so a same-origin
* redirect cannot leave it (another bucket on a shared storage host).
*/
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
prefix,
]);
}
/** Whether a redirect target stays inside its host's pinned directory. */
function withinPinnedPrefix(url) {
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
}
/**
* Fetch a registered frame URL following redirects ONLY within the original
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
* on a pinned camera host, within its pinned directory.
* Default redirect-following would let an upstream steer a host-pinned
* request, and its host-specific headers, to any origin, another port, or a
* plaintext downgrade.
@@ -471,7 +586,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
} catch {
return null;
}
if (next.origin !== origin) return null;
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
current = next;
}
return null;
@@ -543,9 +658,15 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok || !contentType.startsWith('image/')) {
controller.abort();
const contentType = cctvRasterImageType(
upstream.headers.get('content-type'),
);
if (!upstream.ok || !contentType) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
server/providers/cctv/normalize.js
+155 -3
View File
@@ -39,11 +39,21 @@ export function escapeXml(text) {
/**
* Coerce a value to a finite number, returning fallback if NaN/Infinity.
*
* Absent, blank and boolean values take the fallback too: `Number` reads
* null, '' and false as 0, which would turn a missing coordinate, heading or
* elevation into a real zero.
*
* @param {*} value
* @param {number} [fallback=NaN]
* @returns {number}
*/
export function toFiniteNumber(value, fallback = NaN) {
if (
value == null ||
typeof value === 'boolean' ||
(typeof value === 'string' && !value.trim())
)
return fallback;
const num = Number(value);
return Number.isFinite(num) ? num : fallback;
}
@@ -54,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
* @param {string} text
* @returns {string}
*/
export function normalizeKey(text) {
function normalizeKey(text) {
return String(text || '')
.trim()
.toLowerCase()
@@ -70,7 +80,7 @@ export function normalizeKey(text) {
* @param {string} value
* @returns {{lat:number, lon:number}}
*/
export function parsePointString(value) {
function parsePointString(value) {
const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
@@ -90,7 +100,7 @@ export function parsePointString(value) {
* @param {string|object|null} value
* @returns {{lat:number, lon:number}}
*/
export function coerceLatLon(value) {
function coerceLatLon(value) {
if (!value) return { lat: NaN, lon: NaN };
if (typeof value === 'string') {
@@ -367,11 +377,93 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
);
}
/** Florida statewide, Key West to Georgia state line, with slack at each edge */
export function isLikelyFloridaCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 24.3 &&
lat <= 31.1 &&
lon >= -87.7 &&
lon <= -79.8
);
}
/** New York City's five boroughs, with slack for the harbor and city line. */
export function isLikelyNycCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.45 &&
lat <= 40.95 &&
lon >= -74.3 &&
lon <= -73.65
);
}
/** Hong Kong SAR extent, with slack for the outlying islands. */
export function isLikelyHongKongCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 22.15 &&
lat <= 22.6 &&
lon >= 113.8 &&
lon <= 114.45
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 57.9 &&
lat <= 71.3 &&
lon >= 4.4 &&
lon <= 31.3
);
}
export function isLikelyFinlandCoordinate(lat, lon) {
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
}
/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */
export function isLikelyCataloniaCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.3 &&
lat <= 42.9 &&
lon >= 0.0 &&
lon <= 3.4
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
* place names, plus the handful of named entities XML/HTML text commonly
* carries. A reference that names no valid character (NUL, a surrogate, or
* anything past U+10FFFF) is left as written rather than thrown on, so one
* bad field cannot fail a whole catalog.
*
* @param {string} text
* @returns {string}
*/
export function decodeNumericEntities(text) {
return String(text || '')
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => {
const code = hex ? Number.parseInt(hex, 16) : Number(dec);
const valid =
code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);
return valid ? String.fromCodePoint(code) : match;
})
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.trim();
}
/**
* Human label for one Fintraffic preset (camera view).
*
@@ -540,5 +632,65 @@ export function normalizeSourceItem(item) {
// badge can distinguish them from raw automated priors (e.g. Austin Open
// Data, which never sets this field). Passed through as-is to the client.
poseSource: item.poseSource === 'curated' ? 'curated' : undefined,
// Optional hardware-model passthrough (additive, like poseSource): packs
// whose catalog publishes the camera's hardware model — King County's
// layer carries Manufacturer/Model per camera — pass it through so
// identified-hardware consumers can use it. Absent everywhere else.
model:
typeof item.model === 'string' && item.model.trim()
? item.model.trim()
: undefined,
// Bare model strings collide across vendors (e.g. "3950"), so the
// publishing pack's manufacturer rides along for disambiguation.
manufacturer:
typeof item.manufacturer === 'string' && item.manufacturer.trim()
? item.manufacturer.trim()
: undefined,
};
}
/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */
export function isLikelyTorontoCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 43.55 &&
lat <= 43.9 &&
lon >= -79.7 &&
lon <= -79.1
);
}
/**
* A camera URL under a pinned directory, by parsed URL rather than string
* prefix: the same scheme and exact host as `prefix`, no credentials, the
* default port, and a normalized path inside the prefix's directory. Dot
* segments (including encoded ones) and backslashes are resolved before the
* path is compared, and an encoded slash or backslash is refused.
*
* @param {string} raw - Camera URL from a catalog or redirect.
* @param {string} prefix - Pinned origin and directory, ending in `/`.
* @returns {string|null} The normalized URL, or null when it is not pinned.
*/
export function pinnedCameraUrl(raw, prefix) {
let url;
let pin;
try {
url = new URL(String(raw ?? '').trim());
pin = new URL(prefix);
} catch {
return null;
}
if (
url.protocol !== pin.protocol ||
url.hostname !== pin.hostname ||
url.username ||
url.password ||
url.port ||
!url.pathname.startsWith(
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
) ||
/%2f|%5c/i.test(url.pathname)
)
return null;
return url.href;
}
server/providers/cctv/ontario511.test.mjs
+129
View File
@@ -0,0 +1,129 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
_resetOntarioKeyWarningForTest,
loadOntarioSourcesFromOpenData,
} from './sources.js';
const ROW = {
Id: '42',
Latitude: 43.6532,
Longitude: -79.3832,
Location: 'Hwy 401 near Yonge St',
Direction: 'Eastbound',
Views: [
{
Status: 'Enabled',
Url: 'https://511on.ca/map/Cctv/4201',
Description: 'Eastbound',
},
],
};
function captureConsole(t) {
const lines = [];
const capture = (...args) => lines.push(args.join(' '));
t.mock.method(console, 'warn', capture);
t.mock.method(console, 'log', capture);
return lines;
}
test.beforeEach(() => _resetOntarioKeyWarningForTest());
test('without ONTARIO_511_API_KEY the Ontario pack is skipped, logged once', async (t) => {
const lines = captureConsole(t);
let fetches = 0;
t.mock.method(globalThis, 'fetch', async () => {
fetches += 1;
return Response.json([ROW]);
});
assert.deepEqual(await loadOntarioSourcesFromOpenData({}, {}), []);
assert.deepEqual(
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: ' ' }),
[],
);
assert.equal(fetches, 0);
assert.equal(lines.length, 1);
assert.match(lines[0], /ONTARIO_511_API_KEY/);
});
test('with a key the camera list is requested with it and parsed', async (t) => {
const lines = captureConsole(t);
const requested = [];
t.mock.method(globalThis, 'fetch', async (url) => {
requested.push(String(url));
return Response.json([ROW]);
});
const cameras = await loadOntarioSourcesFromOpenData(
{ sourceRoot: '/ignored' },
{ ONTARIO_511_API_KEY: ' test-key ' },
);
assert.equal(requested.length, 1);
const url = new URL(requested[0]);
assert.equal(url.host, '511on.ca');
assert.equal(url.searchParams.get('key'), 'test-key');
assert.equal(url.searchParams.get('format'), 'json');
assert.equal(cameras.length, 1);
assert.equal(cameras[0].id, 'on-42');
assert.equal(cameras[0].url, 'https://511on.ca/map/Cctv/4201');
// The records go to the browser through /api/cctv; the key must stay on
// the server-side catalog request.
assert.doesNotMatch(JSON.stringify(cameras), /test-key/);
assert.doesNotMatch(lines.join('\n'), /test-key/);
});
test('the source cap is read from the same environment as the key', async (t) => {
captureConsole(t);
const rows = Array.from({ length: 12 }, (_, i) => ({
...ROW,
Id: String(100 + i),
Latitude: ROW.Latitude + i * 0.01,
Views: [{ ...ROW.Views[0], Url: `https://511on.ca/map/Cctv/${100 + i}` }],
}));
t.mock.method(globalThis, 'fetch', async () => Response.json(rows));
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'test-key', CCTV_ONTARIO_MAX_SOURCES: '8' },
);
assert.equal(cameras.length, 8);
});
for (const status of [400, 401, 403]) {
test(`a ${status} points at the key by name, never by value`, async (t) => {
const lines = captureConsole(t);
t.mock.method(globalThis, 'fetch', async () =>
Response.json({ Message: 'Invalid Key' }, { status }),
);
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'secret-key-value' },
);
assert.deepEqual(cameras, []);
assert.match(
lines.join('\n'),
new RegExp(`failed: ${status} \\(check ONTARIO_511_API_KEY\\)`),
);
assert.doesNotMatch(lines.join('\n'), /secret-key-value/);
});
}
for (const status of [429, 500, 503]) {
test(`a ${status} is reported without blaming the key`, async (t) => {
const lines = captureConsole(t);
t.mock.method(globalThis, 'fetch', async () =>
Response.json({}, { status }),
);
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'secret-key-value' },
);
assert.deepEqual(cameras, []);
assert.equal(lines.length, 1);
assert.equal(
lines[0],
`[CCTV] Ontario 511 camera download failed: ${status}`,
);
assert.doesNotMatch(lines[0], /secret-key-value/);
});
}
server/providers/cctv/sources.js
+1983 -39
View File
File diff suppressed because it is too large. Load diff
server/providers/cctv/stream.js
+1 -1
View File
@@ -70,7 +70,7 @@ export async function fetchHlsBytes(
}
}
export function sameOriginHlsUrl(value, base) {
function sameOriginHlsUrl(value, base) {
const url = new URL(value, base);
if (
!['https:', 'http:'].includes(url.protocol) ||
server/providers/cctv/taiwan.js
+303
View File
@@ -0,0 +1,303 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseTextCapped } from '../common/http.js';
/** Freeway Bureau (MOTC) CCTV static table in the national MOTC 2.0 XML
* standard, published keyless and refreshed daily (~1,800 cameras). */
export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
/** After a failed download, wait this long before asking again. The catalog
* host does not answer from some networks outside Taiwan, and every attempt
* there holds the shared catalog refresh until the connect timeout. */
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
/** The table is republished daily; a good copy stands in for a failed
* download for up to this long. */
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// The table publishes each camera's MJPEG stream. The same camera servers
// answer a single JPEG for the same camera ID on a sibling path, which is what
// the frame proxy can relay; only these exact pairs are ever registered.
const STREAM_TO_FRAME = new Map([
['cctvn.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
['cctvn5.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
['cctvc.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
[
'cctvs.freeway.gov.tw',
['/live-view/mjpg/video.cgi', '/live-view/jpg/image.cgi'],
],
]);
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
/** Prioritization anchors, north to south, so a cap keeps the whole island
* rather than only greater Taipei. */
const ANCHORS = [
{ lat: 25.033, lon: 121.5654 }, // Taipei
{ lat: 24.9936, lon: 121.301 }, // Taoyuan
{ lat: 24.8138, lon: 120.9675 }, // Hsinchu
{ lat: 24.7021, lon: 121.7378 }, // Yilan
{ lat: 24.1477, lon: 120.6736 }, // Taichung
{ lat: 22.9999, lon: 120.227 }, // Tainan
{ lat: 22.6273, lon: 120.3014 }, // Kaohsiung
];
/** Taiwan main island with slack for the coast; the freeway network has no
* offshore-island cameras. */
function isLikelyTaiwanCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= 21.8 &&
lat <= 25.4 &&
lon >= 119.9 &&
lon <= 122.1
);
}
const XML_ENTITIES = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
};
function codePoint(value, radix) {
const code = parseInt(value, radix);
return Number.isInteger(code) && code > 0 && code <= 0x10ffff
? String.fromCodePoint(code)
: '';
}
function decodeXml(text) {
return String(text ?? '')
.replace(
/&(?:(amp|lt|gt|quot|apos)|#(\d{1,7})|#x([0-9a-fA-F]{1,6}));/g,
(_, name, dec, hex) =>
name ? XML_ENTITIES[name] : codePoint(dec ?? hex, dec ? 10 : 16),
)
.trim();
}
function tag(block, name) {
const match = new RegExp(`<${name}>([^<]*)</${name}>`).exec(block);
return match ? decodeXml(match[1]) : '';
}
/**
* Parse the MOTC 2.0 `CCTVList` document into flat rows. Regex-based like the
* Catalonia and Tarktee parsers: the schema is flat, and only the fields used
* below are read.
*
* @param {string} xml
* @returns {Array<object>}
*/
export function parseTaiwanCctvXml(xml) {
const rows = [];
const blockRe = /<CCTV>([\s\S]*?)<\/CCTV>/g;
let match;
while ((match = blockRe.exec(String(xml || ''))) !== null) {
const block = match[1];
rows.push({
cctvId: tag(block, 'CCTVID'),
streamUrl: tag(block, 'VideoStreamURL'),
lat: Number(tag(block, 'PositionLat')),
lon: Number(tag(block, 'PositionLon')),
roadName: tag(block, 'RoadName'),
direction: tag(block, 'RoadDirection'),
sectionStart: tag(block, 'Start'),
sectionEnd: tag(block, 'End'),
mile: tag(block, 'LocationMile'),
});
}
return rows;
}
/**
* Published MJPEG stream URL -> the same camera's still, plus the host and
* camera number that identify it, or null.
*
* @param {string} streamUrl
* @returns {?{frameUrl: string, host: string, camera: string}}
*/
export function taiwanFrame(streamUrl) {
let url;
try {
url = new URL(String(streamUrl ?? ''));
} catch {
return null;
}
if (url.protocol !== 'https:' || url.username || url.password || url.port)
return null;
const paths = STREAM_TO_FRAME.get(url.hostname);
if (!paths || url.pathname !== paths[0]) return null;
const camera = url.searchParams.get('camera') || '';
if (!/^[A-Za-z0-9-]{1,40}$/.test(camera)) return null;
return {
// Reconstruct so extra query parameters cannot alter the request.
frameUrl: `https://${url.hostname}${paths[1]}?camera=${camera}`,
host: url.hostname.split('.')[0],
camera,
};
}
/**
* One parsed Freeway Bureau row -> one catalog source, or null.
*
* @param {object} row - Output of parseTaiwanCctvXml.
* @returns {?object}
*/
export function taiwanCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const { lat, lon } = row;
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
const frame = taiwanFrame(row.streamUrl);
if (!frame) return null;
const imageUrl = frame.frameUrl;
// CCTVID carries Chinese place names and a few IDs repeat across distinct
// cameras, so the stable key is the camera server plus its camera number.
const rawId = String(row.cctvId ?? '')
.trim()
.slice(0, 80);
const cameraId = `tw-freeway-${frame.host}-${frame.camera.toLowerCase()}`;
const road = String(row.roadName ?? '').trim();
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
const mile = String(row.mile ?? '').trim();
const section = [row.sectionStart, row.sectionEnd]
.map((part) => String(part ?? '').trim())
.filter(Boolean)
.join('→');
// A bare carriageway letter is not a name; fall back to the published ID.
const name =
(road || mile || section
? [road, direction, mile, section].filter(Boolean).join(' ')
: '') ||
rawId ||
`Freeway camera ${frame.camera}`;
return {
id: cameraId,
name: name.slice(0, 160),
// One island-wide category in the camera picker, like Finland and Norway.
city: 'Taiwan / 臺灣',
cityId: 'taiwan',
provider: 'Freeway Bureau, MOTC',
lat,
lon,
// The table gives the carriageway direction, not the camera's bearing,
// so the pose uses the shared low-confidence prior.
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'tw-freeway-motc',
license:
'Freeway Bureau, MOTC open data, Open Government Data License, Taiwan (Version 1.0)',
code: cameraDisplayCode(
`${road} ${mile}`.trim().toUpperCase() || frame.camera,
),
};
}
let lastGood = null;
let lastGoodAt = 0;
let retryAfter = 0;
/** Forget the last good table and any failure backoff (tests only). */
export function resetTaiwanSourcesCache() {
lastGood = null;
lastGoodAt = 0;
retryAfter = 0;
}
function fallbackSources() {
return lastGood && Date.now() - lastGoodAt <= TAIWAN_LAST_GOOD_MAX_AGE_MS
? lastGood
: [];
}
function taiwanSourceCap() {
const rawCap = Number(
process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES,
);
return Number.isFinite(rawCap)
? Math.max(1, Math.min(2000, Math.floor(rawCap)))
: DEFAULT_TAIWAN_MAX_SOURCES;
}
/**
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
* One bounded request per shared catalog refresh. A failed download serves the
* last good table (up to a day old) and is not retried for an hour, so a host
* that cannot reach the Freeway Bureau does not wait on it every refresh.
*
* @returns {Promise<Array<object>>}
*/
export async function loadTaiwanSources() {
if (Date.now() < retryAfter) return fallbackSources();
try {
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
const response = await fetch(
process.env.CCTV_TAIWAN_URL || TAIWAN_FREEWAY_CCTV_URL,
{
headers: { Accept: 'application/xml,text/xml' },
redirect: 'manual',
signal,
},
);
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `HTTP ${response.status}`,
);
}
const xml = await readResponseTextCapped(
response,
TAIWAN_MAX_CATALOG_BYTES,
signal,
);
const cameras = new Map();
for (const row of parseTaiwanCctvXml(xml)) {
const camera = taiwanCameraToSource(row);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
if (!cameras.size) throw new Error('no usable cameras in the table');
const sources = prioritizeSources(
[...cameras.values()],
taiwanSourceCap(),
ANCHORS,
);
console.log(
`[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`,
);
lastGood = sources;
lastGoodAt = Date.now();
retryAfter = 0;
return sources;
} catch (error) {
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
const fallback = fallbackSources();
console.warn(
`[CCTV] Taiwan camera download failed (${error?.message || error}); ` +
(fallback.length
? `serving the last good ${fallback.length} cameras`
: 'no Taiwan cameras') +
', next attempt in 1 h',
);
return fallback;
}
}
server/providers/common/request.js
+53 -8
View File
@@ -14,16 +14,34 @@ async function readRequestBodyCapped(req, maxBytes) {
return Buffer.concat(chunks);
}
function readRequestBody(req, maxBytes = 1024 * 1024) {
/** Optional cancellation discards buffered data while the request drains. */
function readRequestBody(req, maxBytes = 1024 * 1024, signal) {
return new Promise((resolve, reject) => {
let total = 0;
let bodyTooLarge = false;
let settled = false;
const chunks = [];
req.on('data', (chunk) => {
const stopReading = () => {
if (!signal) return;
req.off('data', data);
req.off('end', end);
signal.removeEventListener('abort', abort);
};
const fail = (error) => {
if (signal) {
if (settled) return;
settled = true;
chunks.length = 0;
signal.removeEventListener('abort', abort);
}
reject(error);
};
const abort = () => fail(signal.reason);
const data = (chunk) => {
// Keep draining after the cap without retaining more bytes. Destroying
// the request also destroys its socket, so the handler cannot deliver
// the fixed error response it constructs for BODY_TOO_LARGE.
if (bodyTooLarge) return;
if (bodyTooLarge || settled) return;
total += chunk.length;
if (total > maxBytes) {
// Same marker readRequestBodyCapped sets, so a caller can tell an
@@ -32,15 +50,42 @@ function readRequestBody(req, maxBytes = 1024 * 1024) {
chunks.length = 0;
const error = new Error(`Request body exceeds ${maxBytes} bytes`);
error.code = 'BODY_TOO_LARGE';
reject(error);
fail(error);
return;
}
chunks.push(chunk);
});
req.on('end', () => {
};
const end = () => {
if (settled) {
stopReading();
return;
}
if (!bodyTooLarge) resolve(Buffer.concat(chunks).toString('utf8'));
});
req.on('error', reject);
if (signal) {
settled = true;
chunks.length = 0;
stopReading();
}
};
const close = () => {
fail(
new DOMException('Request closed before its body ended', 'AbortError'),
);
stopReading();
req.off('error', fail);
req.off('close', close);
};
if (signal) {
// Abort/overflow release retained bytes but keep draining. Retain an
// error handler until close, including a late error after end/abort.
req.on('close', close);
signal.addEventListener('abort', abort, { once: true });
if (signal.aborted) abort();
}
req.on('data', data);
req.on('end', end);
req.on('error', fail);
if (signal && req.destroyed) close();
});
}
server/providers/common/request.test.mjs
+137
View File
@@ -0,0 +1,137 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { EventEmitter, getEventListeners } from 'node:events';
import { readRequestBody } from './request.js';
function fixture() {
const req = new EventEmitter();
req.pause = () => assert.fail('The reader must keep draining');
req.destroy = () =>
assert.fail('The reader must preserve the response socket');
const owner = new AbortController();
return { req, owner };
}
function finish(req, owner) {
assert.doesNotThrow(() => req.emit('error', new Error('late stream error')));
req.emit('end');
assert.doesNotThrow(() => req.emit('error', new Error('error before close')));
req.emit('close');
for (const event of ['data', 'end', 'error', 'close'])
assert.equal(req.listenerCount(event), 0, event);
assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
}
test('default and cancellable readers preserve byte caps and UTF-8 across chunks', async () => {
for (const useSignal of [false, true]) {
for (const text of ['', 'hello', 'é']) {
const { req, owner } = fixture();
const bytes = Buffer.from(text);
const body = readRequestBody(
req,
bytes.length,
useSignal ? owner.signal : undefined,
);
for (const byte of bytes) req.emit('data', Buffer.from([byte]));
req.emit('end');
assert.equal(await body, text);
if (useSignal) finish(req, owner);
}
}
});
test('default overflow rejects before end and drains without pausing or destroying', async () => {
const { req } = fixture();
const rejection = assert.rejects(readRequestBody(req, 2), {
code: 'BODY_TOO_LARGE',
});
req.emit('data', Buffer.from('é'));
req.emit('data', Buffer.from('!'));
await rejection;
req.emit('data', Buffer.alloc(4096));
assert.doesNotThrow(() => req.emit('error', new Error('late')));
req.emit('end');
});
test('partial abort settles before end, discards late data and guards errors until close', async () => {
const { req, owner } = fixture();
const reason = new Error('request deadline');
const rejection = assert.rejects(
readRequestBody(req, 1024, owner.signal),
(error) => error === reason,
);
req.emit('data', Buffer.from('partial'));
owner.abort(reason);
await rejection;
assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
req.emit('data', Buffer.alloc(4096));
finish(req, owner);
});
test('already-aborted and already-closed requests settle and release their listeners', async () => {
for (const closed of [false, true]) {
const { req, owner } = fixture();
if (closed) req.destroyed = true;
else owner.abort();
await assert.rejects(readRequestBody(req, 1024, owner.signal), {
name: 'AbortError',
});
if (!closed) finish(req, owner);
for (const event of ['data', 'end', 'error', 'close'])
assert.equal(req.listenerCount(event), 0);
assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
}
});
test('overflow keeps its first failure through later abort and stream errors', async () => {
const { req, owner } = fixture();
const rejection = assert.rejects(readRequestBody(req, 2, owner.signal), {
code: 'BODY_TOO_LARGE',
});
req.emit('data', Buffer.from('é!'));
await rejection;
owner.abort(new Error('later deadline'));
req.emit('data', Buffer.alloc(4096));
finish(req, owner);
});
test('early close and request error reject once; successful end wins over later abort', async () => {
for (const event of ['close', 'error', 'end']) {
const { req, owner } = fixture();
const reason = new Error('request failed');
const body = readRequestBody(req, 32, owner.signal);
const observed =
event === 'end'
? body.then((value) => assert.equal(value, 'ok'))
: assert.rejects(
body,
event === 'close'
? { name: 'AbortError' }
: (error) => error === reason,
);
req.emit('data', Buffer.from('ok'));
req.emit(event, reason);
owner.abort(new Error('later abort'));
await observed;
if (event !== 'close') finish(req, owner);
assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
}
});
test('a prior data observer can abort synchronously without retaining or resolving that chunk', async () => {
const { req, owner } = fixture();
const reason = new Error('observer stopped request');
const external = () => owner.abort(reason);
req.on('data', external);
const rejection = assert.rejects(
readRequestBody(req, 1024, owner.signal),
(error) => error === reason,
);
req.emit('data', Buffer.from('must be discarded'));
await rejection;
req.emit('end');
req.emit('close');
assert.deepEqual(req.listeners('data'), [external]);
assert.equal(getEventListeners(owner.signal, 'abort').length, 0);
req.off('data', external);
});
server/providers/common/same-site.js
+45
View File
@@ -41,3 +41,48 @@ export function sameSiteGated(handler) {
return handler(req, res, next);
};
}
/**
* Proxy header values arrive client-controlled; read one only when it is a
* single scheme token, never a comma-separated chain.
* @param {import('node:http').IncomingMessage} req
* @returns {'http:'|'https:'}
*/
function forwardedProtocol(req) {
if (req.socket?.encrypted) return 'https:';
const proto = String(req.headers?.['x-forwarded-proto'] || '')
.trim()
.toLowerCase();
return proto === 'https' ? 'https:' : 'http:';
}
/**
* Cross-site gate for keyless routes the map needs in order to render
* (`/api/terrain/heights`). It refuses the same cross-site browser shapes as
* `admitSameSite` (a `Sec-Fetch-Site` other than same-origin/none, a foreign
* or opaque `Origin`) but does not refuse reverse-proxy forwarding headers:
* those protect credentials and spend, which a keyless route has neither of,
* and refusing them would break rendering for an instance served behind the
* reverse proxy the README documents. A single `X-Forwarded-Proto` supplies
* the scheme an `Origin` is compared against. That header cannot be attached
* to a cross-site browser request without a CORS preflight this server never
* answers, and `Sec-Fetch-Site` is still enforced regardless.
* @param {import('node:http').IncomingMessage} req
* @param {import('node:http').ServerResponse} res
* @returns {boolean} True when a 403 was sent.
*/
export function admitSameSiteKeyless(req, res) {
const verdict = admitSameSiteRequest({
hostHeader: req.headers?.host,
protocol: forwardedProtocol(req),
origin: req.headers?.origin,
secFetchSite: req.headers?.['sec-fetch-site'],
proxyHeaders: {},
});
if (verdict.ok) return false;
res.statusCode = verdict.status;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify({ error: verdict.error }));
return true;
}
server/providers/cyclones.js
+12 -2
View File
@@ -4,6 +4,8 @@ const STATUS_URL = 'https://www.nhc.noaa.gov/CurrentStorms.json';
const GIS =
'https://mapservices.weather.noaa.gov/tropical/rest/services/tropical/NHC_tropical_weather_summary/MapServer';
const HOUR = 3600_000;
// Data, not UI copy: src/layers/cyclones/index.js keeps the same English in
// msg() and translates it when shown (likewise 'Cyclone data unavailable').
const COVERAGE =
'Atlantic and eastern/central North Pacific; not worldwide cyclone coverage.';
const LAYERS = [
@@ -41,6 +43,14 @@ function number(value, min, max) {
const n = Number(value);
return Number.isFinite(n) && n >= min && n <= max ? n : null;
}
// CurrentStorms.json reports movementSpeed in miles per hour (intensity is in
// knots); convert so `speedKt` means what it says.
function knotsFromMph(mph) {
return mph === null ? null : Math.round(mph * 0.868976);
}
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
// up to about half an hour before; allow that, but not a clock far ahead.
const EARLY_ADVISORY_MS = 90 * 60_000;
function iso(value, now) {
if (
typeof value !== 'string' ||
@@ -50,7 +60,7 @@ function iso(value, now) {
const n = Date.parse(value);
if (
!Number.isFinite(n) ||
n > now + 5 * 60_000 ||
n > now + EARLY_ADVISORY_MS ||
now - n > 12 * HOUR ||
new Date(n).toISOString().replace('.000Z', 'Z') !==
value.replace('.000Z', 'Z')
@@ -115,7 +125,7 @@ export function parseCycloneStatus(payload, now = Date.now()) {
pressureHpa: number(raw.pressure, 800, 1100),
movement: {
directionDegrees: number(raw.movementDir, 0, 360),
speedKt: number(raw.movementSpeed, 0, 200),
speedKt: knotsFromMph(number(raw.movementSpeed, 0, 200)),
},
advisoryUrl: officialLink(forecast.url),
outlookUrl: raw.id.startsWith('al')
server/providers/firms.js
+12 -2
View File
@@ -2,6 +2,7 @@ import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import { readResponseTextCapped } from './common/http.js';
/**
* NASA FIRMS live active-fire proxy with a memory + disk cache.
@@ -35,6 +36,12 @@ export function firmsProxy() {
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
* bound, not a budget.
*/
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
@@ -83,9 +90,12 @@ export function firmsProxy() {
*/
async function fetchSource(key, source) {
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
const signal = AbortSignal.timeout(60_000);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const records = parseFirmsCsv(await res.text());
const records = parseFirmsCsv(
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
);
if (records === null) throw new Error('non-CSV upstream response');
return records;
}
server/providers/gbfs.js
+57 -4
View File
@@ -1,8 +1,10 @@
import { readResponseTextCapped } from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import {
isAllowedGbfsHost,
isAllowedGbfsPath,
gbfsCacheControl,
gbfsUpstreamHeaders,
} from '../../src/data/gbfsSource.js';
// ---------------------------------------------------------------------------
@@ -11,8 +13,24 @@ import {
/** Upstream fetch timeout for GBFS requests (ms). */
const GBFS_PROXY_TIMEOUT_MS = 12000;
/** Value of the Client-Identifier header sent with every upstream request. */
export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view';
export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
/**
* Requests/minute/IP admitted by the proxy.
*
* The bikeshare layer's own demand is small and known: `src/layers/bikeshare/
* registry.js` catalogues 15 systems, station information is cached per city
* after its first fetch, and station status refreshes once per city every
* `STATUS_POLL_MS` (60 s) with in-flight requests deduplicated. Even with every
* catalogued system in camera range at once that is about 15 requests a minute,
* so 120 is eight times a maximal real session while still capping a caller
* using this proxy as an open relay against a third-party GBFS host.
*/
export const GBFS_RATE_PER_MIN = 120;
function gbfsRedirectHost(location, requestUrl) {
if (!location) return '';
try {
@@ -60,6 +78,10 @@ export async function fetchGbfsUpstream(
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
// Urban Sharing (the Norwegian city-bike feeds) asks every client to
// name itself in this header; other operators ignore it.
'Client-Identifier': GBFS_CLIENT_IDENTIFIER,
...gbfsUpstreamHeaders(new URL(url).hostname),
},
redirect: 'manual',
signal: controller.signal,
@@ -113,7 +135,12 @@ export async function fetchGbfsUpstream(
*
* @returns {import('vite').Plugin}
*/
export function gbfsProxy() {
export function gbfsProxy({ ratePerMinute = GBFS_RATE_PER_MIN } = {}) {
const allow = makeRateLimiter({
windowMs: 60_000,
max: ratePerMinute,
globalMax: ratePerMinute * 3,
});
const installMiddleware = (server) => {
server.middlewares.use('/api/gbfs', async (req, res) => {
try {
@@ -126,6 +153,19 @@ export function gbfsProxy() {
return;
}
// Frequency, checked before the target is even parsed: the allowlist
// already says WHERE this proxy may go, nothing said HOW OFTEN. A
// refused request performs no outbound fetch and starts no timer.
if (!allow(clientKey(req))) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
'Retry-After': '10',
});
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
const url = new URL(req.url || '/', 'http://localhost');
const encodedTarget = url.pathname.replace(/^\/+/, '');
if (!encodedTarget) {
@@ -181,7 +221,7 @@ export function gbfsProxy() {
return;
}
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) {
res.writeHead(400, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
@@ -196,13 +236,26 @@ export function gbfsProxy() {
}
const upstream = await fetchGbfsUpstream(upstreamUrl.toString());
// Only JSON is relayed, re-serialized under a fixed type: neither the
// upstream's text nor its declared type reaches the browser.
let payload;
try {
payload = JSON.parse(upstream.body);
} catch {
res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'GBFS upstream returned non-JSON' }));
return;
}
res.writeHead(upstream.status, {
'Content-Type': upstream.contentType,
'Content-Type': 'application/json; charset=utf-8',
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
'X-GBFS-Upstream': upstreamUrl.hostname,
'X-GBFS-Cache': 'MISS',
});
res.end(upstream.body);
res.end(JSON.stringify(payload));
} catch (error) {
if (error?.name === 'AbortError') {
res.writeHead(504, {
server/providers/gemini.js
+22
View File
@@ -0,0 +1,22 @@
import { createGeminiTokenHandler } from './gemini/realtime.js';
/** Mint constrained Live API credentials; the permanent key stays on Node. */
function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
function install(middlewares) {
middlewares.use(
'/api/gemini/token',
createGeminiTokenHandler({ ...realtime, annotationGuidance }),
);
}
return {
name: 'gemini-live-proxy',
configureServer(server) {
install(server.middlewares);
},
configurePreviewServer(server) {
install(server.middlewares);
},
};
}
export { geminiLiveProxy };
server/providers/gemini/config.js
+53
View File
@@ -0,0 +1,53 @@
import { realtimeInstructions } from '../openai/instructions.js';
import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
export const GEMINI_TOKEN_ENDPOINT =
'https://generativelanguage.googleapis.com/v1beta/auth_tokens';
export const GEMINI_LIVE_WEBSOCKET_URL =
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
export function normalizeGeminiModel(value) {
const model = String(value ?? GEMINI_LIVE_MODEL_DEFAULT)
.trim()
.replace(/^models\//, '');
return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
}
/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
export function createGeminiLiveConfig(
annotationGuidance,
inputMode = 'open-mic',
tools = GEV_REALTIME_TOOLS,
) {
if (!['open-mic', 'push-to-talk'].includes(inputMode))
throw new Error('Invalid Gemini input mode');
const config = {
generationConfig: { responseModalities: ['AUDIO'] },
systemInstruction: {
parts: [{ text: realtimeInstructions(annotationGuidance) }],
},
tools: [
{
functionDeclarations: tools.map(
({ name, description, parameters }) => ({
name,
description,
// GEV's canonical schemas are JSON Schema, not Google's Schema.
parametersJsonSchema: structuredClone(parameters),
// App actions must finish before the model confirms their result.
behavior: 'BLOCKING',
}),
),
},
],
inputAudioTranscription: {},
outputAudioTranscription: {},
contextWindowCompression: { slidingWindow: {} },
};
if (inputMode === 'push-to-talk')
config.realtimeInputConfig = {
automaticActivityDetection: { disabled: true },
};
return config;
}
server/providers/gemini/realtime.js
+208
View File
@@ -0,0 +1,208 @@
import { waitForSignal } from '../../../src/sources/requestDeadline.js';
import { clientKey, makeCostRateLimiter } from '../common/rate-limit.js';
import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
import {
GEMINI_TOKEN_ENDPOINT,
GEMINI_LIVE_WEBSOCKET_URL,
GEMINI_LIVE_MODEL_DEFAULT,
createGeminiLiveConfig,
normalizeGeminiModel,
} from './config.js';
import {
isGeminiRequestOriginAllowed,
readGeminiTokenRequest,
} from './request.js';
const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
export function createGeminiTokenHandler({
annotationGuidance,
tools,
fetchImpl = (...args) => fetch(...args),
resolveApiKey = () => process.env.GEMINI_API_KEY,
resolveModel = () =>
process.env.GEMINI_LIVE_MODEL || GEMINI_LIVE_MODEL_DEFAULT,
resolveHost = () => process.env.HOST,
resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
resolveRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN ?? '30',
now = Date.now,
timeoutMs = 15_000,
} = {}) {
let limiter;
return async (req, res) => {
if (req.aborted || res.destroyed || res.writableEnded) return;
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('X-Content-Type-Options', 'nosniff');
const reply = (status, errorOrPayload) => {
if (res.destroyed || res.writableEnded) return;
res.statusCode = status;
if (status >= 400) res.setHeader('Connection', 'close');
res.end(
JSON.stringify(
typeof errorOrPayload === 'string'
? { error: errorOrPayload }
: errorOrPayload,
),
);
};
if (req.method !== 'POST') {
res.setHeader('Allow', 'POST');
return reply(405, 'Method not allowed');
}
let apiKey;
let model;
try {
if (
!isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
)
return reply(403, 'Gemini voice requires a same-origin app request');
apiKey = String(resolveApiKey() || '').trim();
if (!apiKey)
return reply(503, 'Add a Gemini API key in Provider Settings');
model = normalizeGeminiModel(resolveModel());
if (!model)
return reply(503, 'The Gemini Live model configuration is invalid');
if (limiter === undefined)
limiter = makeCostRateLimiter(resolveRateLimit(), 30);
if (limiter && !limiter(clientKey(req))) {
res.setHeader('Retry-After', '60');
return reply(
429,
'Gemini voice is starting too often. Try again shortly.',
);
}
} catch {
// Connect does not observe the handler's returned promise. A synchronous
// configuration resolver must fail with a bounded, fixed response.
return reply(503, MINT_FAILURE);
}
const lifetime = new AbortController();
let disconnected = false;
const disconnect = () => {
disconnected = true;
lifetime.abort(new DOMException('Request closed', 'AbortError'));
};
const responseClosed = () => {
if (!res.writableEnded) disconnect();
};
req.once('aborted', disconnect);
res.once('close', responseClosed);
const deadline = setTimeout(() => {
lifetime.abort(
new DOMException('Token request timed out', 'TimeoutError'),
);
}, timeoutMs);
deadline.unref?.();
let acquiredResponse;
try {
const inputMode = await readGeminiTokenRequest(req, lifetime.signal);
lifetime.signal.throwIfAborted();
const config = createGeminiLiveConfig(
annotationGuidance,
inputMode,
tools,
);
const issuedAt = now();
// REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
// input shape, converted before transport. With no fieldMask the complete
// setup is locked to this server-owned configuration.
const request = {
uses: 1,
expireTime: new Date(issuedAt + 30 * 60_000).toISOString(),
newSessionExpireTime: new Date(issuedAt + 60_000).toISOString(),
bidiGenerateContentSetup: { model: `models/${model}`, ...config },
};
const acquisition = Promise.resolve(
fetchImpl(GEMINI_TOKEN_ENDPOINT, {
method: 'POST',
redirect: 'error',
signal: lifetime.signal,
headers: {
'x-goog-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify(request),
}),
);
// A fetch implementation can ignore abort. Bound this caller's wait and
// discard a response that arrives after timeout/disconnection.
void acquisition.then(
(late) => {
acquiredResponse = late;
if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
},
() => {},
);
const response = await waitForSignal(acquisition, lifetime.signal);
lifetime.signal.throwIfAborted();
if (
response.redirected ||
(response.url && response.url !== GEMINI_TOKEN_ENDPOINT)
) {
void response.body?.cancel().catch(() => {});
throw new Error('Unexpected token response destination');
}
if (!response.ok) {
// Never forward upstream bodies, headers or exception messages.
void response.body?.cancel().catch(() => {});
if (response.status === 429) {
res.setHeader('Retry-After', '60');
return reply(429, 'Gemini quota is unavailable. Try again later.');
}
if (response.status === 401 || response.status === 403)
return reply(
503,
'Gemini access was refused. Check the API key and model access.',
);
if (response.status === 400 || response.status === 404)
return reply(
502,
'Gemini Live is unavailable for the configured model.',
);
return reply(502, MINT_FAILURE);
}
const data = await readResponseJsonCapped(
response,
UPSTREAM_RESPONSE_BYTES,
lifetime.signal,
);
lifetime.signal.throwIfAborted();
if (
!data ||
Array.isArray(data) ||
typeof data !== 'object' ||
typeof data.name !== 'string' ||
data.name.length > 8192 ||
!/^auth_tokens\/[A-Za-z0-9._~+/=-]+$/.test(data.name) ||
data.name.includes(apiKey)
)
throw new Error('Invalid ephemeral token');
reply(200, {
token: data.name,
model,
config,
websocketUrl: GEMINI_LIVE_WEBSOCKET_URL,
});
} catch (error) {
if (disconnected || req.aborted) return;
if (error?.code === 'BODY_TOO_LARGE')
return reply(413, 'Token request is too large');
if (error?.code === 'BAD_CONTENT_TYPE')
return reply(415, 'Use an empty JSON token request');
if (error?.code === 'BAD_BODY')
return reply(400, 'Use an empty JSON token request');
reply(lifetime.signal.aborted ? 504 : 502, MINT_FAILURE);
} finally {
// Abort can happen after acquisition resolves but before this handler
// resumes and passes ownership to the body reader.
if (lifetime.signal.aborted)
void acquiredResponse?.body?.cancel().catch(() => {});
clearTimeout(deadline);
req.off('aborted', disconnect);
res.off('close', responseClosed);
}
};
}
server/providers/gemini/request.js
+79
View File
@@ -0,0 +1,79 @@
import { readRequestBody } from '../common/request.js';
import { admitSameSiteRequest } from '../../../src/localRequestGate.mjs';
import {
isAllowedHost,
resolveAllowedHosts,
} from '../../../build/allowedHosts.js';
/** Use the app's Host and request policies, with a required browser Origin. */
export function isGeminiRequestOriginAllowed(
req,
configuredHost = '',
configuredAllowedHosts,
) {
const host = req.headers?.host;
const origin = req.headers?.origin;
if (
typeof host !== 'string' ||
typeof origin !== 'string' ||
!host ||
!origin ||
/[\s/?#@\\]/.test(host)
)
return false;
const protocol = req.socket?.encrypted ? 'https:' : 'http:';
if (
!admitSameSiteRequest({
hostHeader: host,
protocol,
origin,
secFetchSite: req.headers?.['sec-fetch-site'],
proxyHeaders: req.headers || {},
}).ok
)
return false;
// The shared gate compares a present Origin exactly with this authority;
// this route also requires the Origin and an allowed Host.
const boundHost = String(configuredHost || '')
.trim()
.toLowerCase();
return isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [
boundHost,
]);
}
/** Admit only the existing input gesture; callers cannot supply model/setup. */
export async function readGeminiTokenRequest(req, signal, maxBytes = 1024) {
const declared = Number(req.headers?.['content-length']);
const fail = (code) =>
Object.assign(new Error('Invalid token request'), { code });
const body = await readRequestBody(
req,
maxBytes,
Number.isFinite(declared) && declared > maxBytes
? AbortSignal.abort(fail('BODY_TOO_LARGE'))
: signal,
);
if (!body.trim()) return 'open-mic';
if (
!/^application\/json(?:\s*;|$)/i.test(req.headers?.['content-type'] || '')
)
throw fail('BAD_CONTENT_TYPE');
let parsed;
try {
parsed = JSON.parse(body);
} catch {
throw fail('BAD_BODY');
}
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object')
throw fail('BAD_BODY');
const keys = Object.keys(parsed);
if (!keys.length) return 'open-mic';
if (
keys.length === 1 &&
keys[0] === 'inputMode' &&
parsed.inputMode === 'push-to-talk'
)
return 'push-to-talk';
throw fail('BAD_BODY');
}
server/providers/ghostMaps.js
+437
View File
@@ -0,0 +1,437 @@
import fsp from 'node:fs/promises';
import path from 'node:path';
import {
GHOSTMAPS_MAX_FILE_BYTES,
GHOSTMAPS_REPO,
buildGhostMapsCatalog,
ghostMapsRawUrl,
parseGitRefAdvertisement,
} from '../../src/layers/ghostMaps/catalog.js';
import {
readResponseBytesCapped,
readResponseJsonCapped,
readResponseTextCapped,
} from './common/http.js';
// S2 Underground GhostMaps (public GitHub repository of KMZ intel products).
// The repository is not ours, so there is no webhook: the watcher reads the
// branch head through git's smart-HTTP ref advertisement (~0.5 KB, outside the
// REST API rate limit) every few seconds, and only a new commit costs REST
// calls (tree + commit). Browsers hear about a new commit over server-sent
// events and fetch only the products whose blob changed. Files are fetched by
// commit-pinned raw URLs and kept on disk by blob sha.
const DEFAULT_POLL_MS = 15_000;
const MIN_POLL_MS = 5_000;
const FAILURE_BACKOFF_MS = [15_000, 30_000, 60_000, 120_000, 300_000];
const TIMEOUT_MS = 15_000;
const FILE_TIMEOUT_MS = 120_000;
const HEARTBEAT_MS = 25_000;
const MAX_SUBSCRIBERS = 64;
const MAX_REF_BYTES = 64 * 1024;
const MAX_TREE_BYTES = 16 * 1024 * 1024;
const MAX_COMMIT_BYTES = 2 * 1024 * 1024;
const USER_AGENT = 'gods-eye-view-ghostmaps/1.0';
const REPO_PATH = `${GHOSTMAPS_REPO.owner}/${GHOSTMAPS_REPO.name}`;
/** Poll interval from GHOSTMAPS_POLL_SECONDS, bounded below at 5 s. */
export function ghostMapsPollMs(raw = process.env.GHOSTMAPS_POLL_SECONDS) {
const seconds = Number(raw);
if (!raw || !Number.isFinite(seconds)) return DEFAULT_POLL_MS;
return Math.max(MIN_POLL_MS, Math.round(seconds * 1000));
}
/**
* The repository watcher and file cache, independent of HTTP routing.
* @returns {{start: Function, stop: Function, catalog: Function,
* checkNow: Function, file: Function, subscribe: Function}}
*/
export function createGhostMapsWatcher({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
pollMs = ghostMapsPollMs(),
cacheDir = path.join(process.cwd(), '.gev-cache', 'ghostmaps'),
token = () => String(process.env.GITHUB_TOKEN || '').trim(),
log = (...args) => console.log(...args),
warn = (...args) => console.warn(...args),
} = {}) {
let state = {
commit: null,
committedAt: null,
message: null,
products: [],
checkedAt: null,
changedAt: null,
error: null,
};
let timer = null;
let running = false;
let checking = null;
let failures = 0;
let diskLoaded = false;
const listeners = new Set();
const downloads = new Map();
const apiHeaders = () => ({
Accept: 'application/vnd.github+json',
'User-Agent': USER_AGENT,
...(token() ? { Authorization: `Bearer ${token()}` } : {}),
});
async function get(url, headers, timeoutMs = TIMEOUT_MS) {
const response = await fetchImpl(url, {
headers,
redirect: 'follow',
signal: AbortSignal.timeout(timeoutMs),
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw Object.assign(new Error(`HTTP ${response.status} from ${url}`), {
status: response.status,
});
}
return response;
}
async function readHead() {
const response = await get(
`${GHOSTMAPS_REPO.url}.git/info/refs?service=git-upload-pack`,
{ 'User-Agent': USER_AGENT },
);
const sha = parseGitRefAdvertisement(
await readResponseTextCapped(response, MAX_REF_BYTES),
);
if (!sha) throw new Error('GhostMaps branch head not advertised');
return sha;
}
async function readCommit(sha) {
const api = `https://api.github.com/repos/${REPO_PATH}`;
const tree = await readResponseJsonCapped(
await get(`${api}/git/trees/${sha}?recursive=1`, apiHeaders()),
MAX_TREE_BYTES,
);
if (!Array.isArray(tree?.tree)) throw new Error('GhostMaps tree malformed');
if (tree.truncated) warn('[ghostmaps] repository tree listing truncated');
let committedAt = null;
let message = null;
try {
const commit = await readResponseJsonCapped(
await get(`${api}/commits/${sha}`, apiHeaders()),
MAX_COMMIT_BYTES,
);
committedAt = Date.parse(commit?.commit?.committer?.date) || null;
message = String(commit?.commit?.message || '')
.split('\n')[0]
.slice(0, 200);
} catch {
// The commit date is decoration; the tree is what matters.
}
return { products: buildGhostMapsCatalog(tree.tree), committedAt, message };
}
async function loadDisk() {
if (diskLoaded) return;
diskLoaded = true;
try {
const saved = JSON.parse(
await fsp.readFile(path.join(cacheDir, 'catalog.json'), 'utf8'),
);
if (/^[0-9a-f]{40}$/.test(saved?.commit) && Array.isArray(saved.products))
state = { ...state, ...saved, error: null };
} catch {
/* first run */
}
}
async function saveDisk() {
try {
await fsp.mkdir(cacheDir, { recursive: true });
const { commit, committedAt, message, products, changedAt } = state;
await fsp.writeFile(
path.join(cacheDir, 'catalog.json'),
JSON.stringify({ commit, committedAt, message, products, changedAt }),
);
} catch (error) {
warn('[ghostmaps] could not save catalog:', error.message);
}
}
function emit() {
const event = publicCatalog();
for (const listener of listeners) {
try {
listener(event);
} catch {
/* a broken subscriber must not stop the rest */
}
}
}
async function check() {
await loadDisk();
const head = await readHead();
state.checkedAt = now();
if (head === state.commit) {
if (state.error !== null) {
state.error = null;
emit();
}
return false;
}
const next = await readCommit(head);
const previous = new Map(state.products.map((p) => [p.id, p.version]));
const changed = next.products
.filter((p) => previous.get(p.id) !== p.version)
.map((p) => p.id);
state = {
...state,
...next,
commit: head,
changedAt: now(),
error: null,
};
log(
`[ghostmaps] commit ${head.slice(0, 8)}${
next.message ? ` "${next.message}"` : ''
}; updated: ${changed.join(', ') || 'none'}`,
);
await saveDisk();
emit();
// Warm the default products so the first browser load is local.
for (const product of state.products)
if (product.defaultOn) file(product.id).catch(() => {});
return true;
}
/** Run one check now, coalescing with one already in flight. */
function checkNow() {
if (!checking)
checking = check()
.then(
(changed) => {
failures = 0;
return changed;
},
(error) => {
failures += 1;
state.error = error.message;
state.checkedAt = now();
warn('[ghostmaps] check failed:', error.message);
emit();
return false;
},
)
.finally(() => {
checking = null;
});
return checking;
}
function schedule() {
clearTimeout(timer);
if (!running) return;
const delay = failures
? FAILURE_BACKOFF_MS[Math.min(failures, FAILURE_BACKOFF_MS.length) - 1]
: pollMs;
timer = setTimeout(async () => {
await checkNow();
schedule();
}, delay);
timer.unref?.();
}
/** Start watching (idempotent); the first check runs immediately. */
function start() {
if (running) return checking || Promise.resolve(false);
running = true;
const first = checkNow();
first.then(schedule);
return first;
}
function stop() {
running = false;
clearTimeout(timer);
timer = null;
}
function publicCatalog() {
return {
repo: GHOSTMAPS_REPO.url,
branch: GHOSTMAPS_REPO.branch,
commit: state.commit,
committedAt: state.committedAt,
message: state.message,
checkedAt: state.checkedAt,
changedAt: state.changedAt,
pollMs,
error: state.error,
products: state.products.map(
({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}) => ({
id,
title,
defaultOn,
fileName,
fileTitle,
version,
size,
dateMs,
}),
),
};
}
/**
* The bytes of one product's current edition, from disk or a commit-pinned
* download. Only catalog paths are ever fetched.
*/
async function file(id) {
const product = state.products.find((p) => p.id === id);
if (!product) return null;
const target = path.join(cacheDir, 'files', `${product.version}.kmz`);
try {
return { product, bytes: await fsp.readFile(target) };
} catch {
/* not cached yet */
}
const commit = state.commit;
let pending = downloads.get(product.version);
if (!pending) {
pending = (async () => {
const response = await get(
ghostMapsRawUrl(commit, product.path),
{ 'User-Agent': USER_AGENT },
FILE_TIMEOUT_MS,
);
const bytes = Buffer.from(
await readResponseBytesCapped(response, GHOSTMAPS_MAX_FILE_BYTES),
);
await fsp.mkdir(path.dirname(target), { recursive: true });
const temp = `${target}.${process.pid}.tmp`;
await fsp.writeFile(temp, bytes);
await fsp.rename(temp, target);
await pruneFiles();
return bytes;
})().finally(() => downloads.delete(product.version));
downloads.set(product.version, pending);
}
return { product, bytes: await pending };
}
/** Keep only files that a current product still points to. */
async function pruneFiles() {
const keep = new Set(state.products.map((p) => `${p.version}.kmz`));
const dir = path.join(cacheDir, 'files');
for (const name of await fsp.readdir(dir).catch(() => [])) {
if (!keep.has(name) && !name.endsWith('.tmp'))
await fsp.rm(path.join(dir, name), { force: true }).catch(() => {});
}
}
function subscribe(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
}
return {
start,
stop,
checkNow,
catalog: publicCatalog,
file,
subscribe,
subscribers: () => listeners.size,
};
}
/** Same-origin GhostMaps routes for dev and preview. */
export function ghostMapsProxy({ watcher = createGhostMapsWatcher() } = {}) {
const json = (res, status, value) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
});
res.end(JSON.stringify(value));
};
function events(req, res) {
if (watcher.subscribers() >= MAX_SUBSCRIBERS)
return json(res, 429, { error: 'too_many_subscribers' });
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-store',
Connection: 'keep-alive',
'X-Accel-Buffering': 'no',
});
const send = (catalog) =>
res.write(`event: catalog\ndata: ${JSON.stringify(catalog)}\n\n`);
res.write('retry: 5000\n\n');
send(watcher.catalog());
const unsubscribe = watcher.subscribe(send);
const heartbeat = setInterval(() => res.write(': ping\n\n'), HEARTBEAT_MS);
heartbeat.unref?.();
const close = () => {
clearInterval(heartbeat);
unsubscribe();
};
req.on('close', close);
res.on('error', close);
}
async function handler(req, res) {
if (req.method !== 'GET')
return json(res, 405, { error: 'method_not_allowed' });
const url = new URL(req.url || '/', 'http://local');
const match = /^\/file\/([a-z0-9-]{1,40})$/.exec(url.pathname);
if (!['/catalog', '/events'].includes(url.pathname) && !match)
return json(res, 404, { error: 'unknown_route' });
// Watching starts with the first request, so an install that never turns
// the layer on never contacts GitHub.
const first = watcher.start();
if (url.pathname === '/catalog') {
if (!watcher.catalog().commit) await first;
return json(res, 200, watcher.catalog());
}
if (url.pathname === '/events') return events(req, res);
try {
if (!watcher.catalog().commit) await first;
const found = await watcher.file(match[1]);
if (!found) return json(res, 404, { error: 'unknown_product' });
const pinned = url.searchParams.get('v') === found.product.version;
res.writeHead(200, {
'Content-Type': 'application/vnd.google-earth.kmz',
'Content-Length': found.bytes.length,
'X-GhostMaps-Version': found.product.version,
// A versioned URL names one immutable blob.
'Cache-Control': pinned
? 'private, max-age=31536000, immutable'
: 'no-store',
});
return res.end(found.bytes);
} catch {
return json(res, 502, { error: 'ghostmaps_unavailable' });
}
}
return {
name: 'ghostmaps',
configureServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/ghostmaps', handler);
},
};
}
server/providers/ghostMaps.test.mjs
+294
View File
@@ -0,0 +1,294 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdtemp, readdir, rm } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import {
createGhostMapsWatcher,
ghostMapsPollMs,
ghostMapsProxy,
} from './ghostMaps.js';
const MASTER =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/Master Database/';
const KINETIC =
'ArcGIS Data for ATAK (KMZs)/Common Intelligence Picture/30-Day Pulls/Kinetic_Activities_Last_30_Days.kmz';
const sha = (c) => c.repeat(40);
/** A fake GitHub: ref advertisement, tree, commit and raw files. */
function fakeGitHub() {
const repo = { head: sha('1'), trees: new Map(), calls: [] };
repo.trees.set(sha('1'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_15_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('a'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
const body = (text, status = 200) =>
new Response(text, {
status,
headers: { 'content-length': String(Buffer.byteLength(text)) },
});
const fetchImpl = async (url) => {
repo.calls.push(url);
if (url.endsWith('/info/refs?service=git-upload-pack'))
return body(
`001e# service=git-upload-pack\n0000\n003d${repo.head} refs/heads/main\n0000`,
);
const tree = /git\/trees\/([0-9a-f]{40})/.exec(url);
if (tree) return body(JSON.stringify({ tree: repo.trees.get(tree[1]) }));
if (/\/commits\//.test(url))
return body(
JSON.stringify({
commit: {
message: 'Oct 18 Update\nmore',
committer: { date: '2026-10-18T00:00:00Z' },
},
}),
);
if (url.startsWith('https://raw.githubusercontent.com/'))
return body('KMZ!');
return body('nope', 404);
};
return { repo, fetchImpl };
}
async function withCache(fn) {
const dir = await mkdtemp(path.join(os.tmpdir(), 'ghostmaps-'));
try {
await fn(dir);
} finally {
await rm(dir, { recursive: true, force: true });
}
}
// checkNow announces the catalog before background warm-up finishes. Join the
// same file reads before changing editions or removing the temporary cache.
async function warmDefaults(watcher) {
await Promise.all(
watcher
.catalog()
.products.filter((product) => product.defaultOn)
.map((product) => watcher.file(product.id)),
);
}
test('a new commit is announced once, naming only the products that changed', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
const first = watcher.catalog();
assert.equal(first.commit, sha('1'));
assert.equal(first.message, 'Oct 18 Update');
assert.deepEqual(
first.products.map((p) => p.id),
['cip', 'kinetic-30d'],
);
assert.equal(events.length, 1);
// Same head: one ref read, no REST calls, no event.
repo.calls.length = 0;
assert.equal(await watcher.checkNow(), false);
assert.equal(repo.calls.length, 1);
assert.equal(events.length, 1);
// A new upload: the CIP gets a newer edition, the 30-day pull is untouched.
repo.head = sha('2');
repo.trees.set(sha('2'), [
...repo.trees.get(sha('1')),
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
]);
assert.equal(await watcher.checkNow(), true);
await warmDefaults(watcher);
assert.equal(events.length, 2);
const cip = events[1].products.find((p) => p.id === 'cip');
assert.equal(cip.version, sha('c'));
assert.match(cip.fileName, /October_18/);
assert.equal(
events[1].products.find((p) => p.id === 'kinetic-30d').version,
sha('b'),
);
// Repository paths stay on the server.
assert.equal('path' in cip, false);
watcher.stop();
}));
test('files download by commit-pinned URL, cache by blob, and stale blobs are pruned', () =>
withCache(async (cacheDir) => {
const { repo, fetchImpl } = fakeGitHub();
const watcher = createGhostMapsWatcher({
fetchImpl,
cacheDir,
log: () => {},
warn: () => {},
});
await watcher.checkNow();
const found = await watcher.file('kinetic-30d');
assert.equal(found.bytes.toString(), 'KMZ!');
const raw = repo.calls.filter((url) =>
url.includes('raw.githubusercontent'),
);
assert.ok(raw.every((url) => url.includes(`/GhostMaps/${sha('1')}/`)));
assert.ok(
raw.some((url) => url.endsWith('/Kinetic_Activities_Last_30_Days.kmz')),
);
const before = repo.calls.length;
await watcher.file('kinetic-30d');
assert.equal(repo.calls.length, before, 'second read comes from disk');
assert.equal(await watcher.file('not-a-product'), null);
// When the edition changes, the old blob leaves the disk cache.
await watcher.file('cip');
repo.head = sha('2');
repo.trees.set(sha('2'), [
{
path: `${MASTER}S2U_Common_Intelligence_Picture_October_18_2026_Export_1.kmz`,
type: 'blob',
size: 4,
sha: sha('c'),
},
{ path: KINETIC, type: 'blob', size: 4, sha: sha('b') },
]);
await watcher.checkNow();
await watcher.file('cip');
const files = (await readdir(path.join(cacheDir, 'files'))).sort();
assert.deepEqual(files, [`${sha('b')}.kmz`, `${sha('c')}.kmz`]);
watcher.stop();
}));
test('the catalog survives a restart from disk, and upstream failures are reported', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
const quiet = { log: () => {}, warn: () => {} };
const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
await online.checkNow();
await warmDefaults(online);
const offline = createGhostMapsWatcher({
fetchImpl: async () => new Response('down', { status: 503 }),
cacheDir,
...quiet,
});
assert.equal(await offline.checkNow(), false);
const catalog = offline.catalog();
assert.equal(catalog.commit, sha('1'));
assert.equal(catalog.products.length, 2);
assert.match(catalog.error, /HTTP 503/);
}));
test('an unchanged head clears an upstream outage and announces recovery once', () =>
withCache(async (cacheDir) => {
const { fetchImpl } = fakeGitHub();
let unavailable = false;
let checkedAt = 1000;
const requests = [];
const watcher = createGhostMapsWatcher({
cacheDir,
now: () => checkedAt,
fetchImpl: async (url, options) => {
requests.push(url);
if (unavailable) return new Response('down', { status: 503 });
return fetchImpl(url, options);
},
log: () => {},
warn: () => {},
});
const events = [];
watcher.subscribe((catalog) => events.push(catalog));
await watcher.checkNow();
await Promise.all([watcher.file('cip'), watcher.file('kinetic-30d')]);
const healthy = watcher.catalog();
assert.equal(events.length, 1);
unavailable = true;
checkedAt = 2000;
assert.equal(await watcher.checkNow(), false);
assert.match(watcher.catalog().error, /HTTP 503/);
assert.equal(events.length, 2);
unavailable = false;
checkedAt = 3000;
requests.length = 0;
assert.equal(await watcher.checkNow(), false, 'no new edition arrived');
const recovered = watcher.catalog();
assert.equal(recovered.error, null);
assert.equal(recovered.checkedAt, checkedAt);
assert.equal(recovered.commit, healthy.commit);
assert.equal(recovered.changedAt, healthy.changedAt);
assert.deepEqual(recovered.products, healthy.products);
assert.equal(requests.length, 1, 'recovery needs only the ref read');
assert.match(requests[0], /\/info\/refs\?service=git-upload-pack$/);
assert.equal(events.length, 3, 'subscribers see the recovered catalog');
assert.equal(events[2].error, null);
assert.equal(events[2].checkedAt, checkedAt);
checkedAt = 4000;
assert.equal(await watcher.checkNow(), false);
assert.equal(events.length, 3, 'healthy unchanged polls stay silent');
watcher.stop();
}));
test('poll interval is configurable with a 5 s floor', () => {
assert.equal(ghostMapsPollMs(undefined), 15000);
assert.equal(ghostMapsPollMs('30'), 30000);
assert.equal(ghostMapsPollMs('1'), 5000);
assert.equal(ghostMapsPollMs('soon'), 15000);
});
test('routes: unknown paths and methods are refused without contacting GitHub', async () => {
let started = 0;
const watcher = {
start: () => {
started += 1;
return Promise.resolve(false);
},
catalog: () => ({ commit: sha('1'), products: [] }),
file: async () => null,
subscribe: () => () => {},
subscribers: () => 0,
};
let handler;
ghostMapsProxy({ watcher }).configureServer({
middlewares: { use: (_route, fn) => (handler = fn) },
});
const call = async (method, url) => {
const res = {
destroyed: false,
writeHead(status, headers) {
this.status = status;
this.headers = headers;
},
end(body) {
this.body = String(body ?? '');
},
};
await handler({ method, url }, res);
return res;
};
assert.equal((await call('GET', '/nope')).status, 404);
assert.equal((await call('POST', '/catalog')).status, 405);
assert.equal((await call('GET', '/file/../../etc')).status, 404);
assert.equal(started, 0);
assert.equal((await call('GET', '/file/cip')).status, 404);
const catalog = await call('GET', '/catalog');
assert.equal(catalog.status, 200);
assert.equal(JSON.parse(catalog.body).commit, sha('1'));
assert.equal(started, 2);
});
server/providers/gnssIntegrity.js
+238
View File
@@ -0,0 +1,238 @@
import {
GNSS_CLASSIFIER,
GNSS_WINDOW_MS,
normalizeGnssAircraft,
} from '../../src/layers/gnss/records.js';
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
// This is a separate read from the flights fallback on purpose: that point
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
// whole-degree anchors held for a minute.
const RADIUS_NM = 250;
const CACHE_MS = 60_000;
const CACHE_MAX = 64;
/** Default 429 cooldown when upstream sends no usable Retry-After. */
const COOLDOWN_MS = 60_000;
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
const COOLDOWN_MIN_MS = 5_000;
const COOLDOWN_MAX_MS = 60 * 60_000;
const FAILURE_COOLDOWN_MS = 15_000;
const MAX_IN_FLIGHT = 8;
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
const TIMEOUT_MS = 10_000;
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
export function gnssIntegrityAnchor(url) {
const anchor = adsbLolFallbackAnchor({ url });
if (!anchor) return null;
return {
lat: Math.round(anchor.latitude),
lon: Math.round(anchor.longitude),
};
}
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
export function gnssRetryCooldownMs(raw, nowMs) {
if (raw) {
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
const ms = seconds * 1000;
if (Number.isFinite(seconds) && seconds > 0)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
const at =
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
if (Number.isFinite(at) && at > nowMs)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
}
return COOLDOWN_MS;
}
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
export function gnssIntegrityProxy({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
monotonicNow = () => performance.now(),
warn = (...args) => console.warn(...args),
} = {}) {
const cache = new Map();
const inFlight = new Map();
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
let cooldownUntil = 0;
let cooldownStatus = 429;
async function load({ lat, lon }) {
if (monotonicNow() < cooldownUntil)
throw Object.assign(new Error('cooling_down'), {
status: cooldownStatus,
});
const signal = AbortSignal.timeout(TIMEOUT_MS);
try {
const response = await fetchImpl(
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
{
signal,
redirect: 'error',
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
},
},
);
if (!response.ok) {
await response.body?.cancel();
if (response.status === 429) {
const at = monotonicNow();
const raw = response.headers.get('retry-after');
const delay = gnssRetryCooldownMs(raw, now());
if (delay === COOLDOWN_MAX_MS)
warn('[gnss-integrity] upstream cooldown capped at one hour');
cooldownStatus = 429;
cooldownUntil = Math.max(cooldownUntil, at + delay);
throw Object.assign(new Error('upstream_rate_limited'), {
status: 429,
});
}
throw new Error('upstream_unavailable');
}
const payload = await readResponseJsonCapped(
response,
MAX_RESPONSE_BYTES,
signal,
);
const rows = normalizeGnssAircraft(payload);
if (!rows) throw new Error('invalid_snapshot');
const sourceNow = payload.now;
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
throw new Error('undated_snapshot');
const fetchedAt = Math.min(
now(),
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
);
if (now() - fetchedAt > GNSS_WINDOW_MS)
throw new Error('expired_snapshot');
return {
fetchedAt,
anchor: { lat, lon },
radiusNm: RADIUS_NM,
// `degraded` on each row is this classifier's verdict, not gpsjam's.
classifier: GNSS_CLASSIFIER.id,
rows,
};
} catch (error) {
if (error.status !== 429) {
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
if (until > cooldownUntil) {
cooldownUntil = until;
cooldownStatus = 502;
}
}
throw error;
}
}
function age(entry) {
if (!entry) return Infinity;
const elapsed = monotonicNow() - entry.savedAt;
// A backwards injected clock cannot rejuvenate a snapshot.
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
}
async function acquire(anchor, peer) {
const key = `${anchor.lat},${anchor.lon}`;
const previous = cache.get(key);
if (age(previous) < CACHE_MS)
return { value: previous.value, ageMs: age(previous), stale: false };
try {
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
throw Object.assign(new Error('busy'), { status: 429, local: true });
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
if (monotonicNow() >= cooldownUntil && !allow(peer))
throw Object.assign(new Error('rate_limited'), {
status: 429,
local: true,
});
const value = await load(anchor);
const entry = {
value,
savedAt: monotonicNow(),
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
};
cache.delete(key);
cache.set(key, entry);
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
return entry;
});
const entry = await promise;
const ageMs = age(entry);
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
} catch (error) {
if (age(previous) <= GNSS_WINDOW_MS)
return { value: previous.value, ageMs: age(previous), stale: true };
throw error;
}
}
async function handler(req, res) {
const json = (status, value, stale = false, retryAfterS = 60) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...([429, 502].includes(status)
? { 'Retry-After': String(retryAfterS) }
: {}),
...(stale ? { 'X-Data-Stale': 'true' } : {}),
});
res.end(JSON.stringify(value));
};
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
if (path !== '/' && path !== '')
return json(404, { error: 'unknown_route' });
const anchor = gnssIntegrityAnchor(req.url);
if (!anchor) return json(400, { error: 'invalid_anchor' });
try {
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
// ageMs is measured on this server's clock, so the browser can date
// the rows without comparing its clock to ours.
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
} catch (error) {
if (error.status === 429) {
// Tell the browser how long the shared upstream cooldown has left.
const retryAfterS = Math.max(
1,
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
);
return json(
429,
{
error: error.local ? error.message : 'gnss_integrity_rate_limited',
},
false,
retryAfterS,
);
}
json(
502,
{ error: 'gnss_integrity_unavailable' },
false,
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
);
}
}
return {
name: 'gnss-integrity',
configureServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
};
}
server/providers/local-receivers.js
+9 -9
View File
@@ -27,13 +27,13 @@ import { normalizeDump1090Aircraft } from '../../src/sources/adsbRecords.js';
*/
export const LOCAL_RECEIVERS_ROUTE = '/api/local-receivers/aircraft';
export const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
export const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
export const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
export const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
export const LOCAL_RECEIVER_CACHE_MS = 1_000;
const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
const LOCAL_RECEIVER_CACHE_MS = 1_000;
/** A feed whose own `now` is older than this is reported stale. */
export const LOCAL_RECEIVER_STALE_MS = 10_000;
const LOCAL_RECEIVER_STALE_MS = 10_000;
const MAX_FEEDS = 8;
const LOG_PREFIX = '[local-receivers]';
@@ -139,7 +139,7 @@ class FeedError extends Error {
* @param {string} address Resolved IP address.
* @returns {boolean}
*/
export function isLocalReceiverAddress(address) {
function isLocalReceiverAddress(address) {
const value = String(address || '').toLowerCase();
if (value === '::1') return true;
const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(value);
@@ -152,7 +152,7 @@ export function isLocalReceiverAddress(address) {
* @param {typeof dnsLookup} lookupImpl `dns.promises.lookup`-compatible.
* @returns {Promise<Array<{address:string, family:number}>>}
*/
export async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
let resolved;
try {
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
@@ -274,7 +274,7 @@ function fetchLocalReceiverFeed(url, options) {
* @param {object} options
* @returns {Promise<object>} Parsed JSON document.
*/
export async function fetchLocalReceiverDocument(
async function fetchLocalReceiverDocument(
url,
{
fetchImpl = fetchLocalReceiverFeed,
server/providers/local.js
+27 -3
View File
@@ -1,3 +1,4 @@
import path from 'node:path';
import { openSkyProxy } from './aircraft/opensky.js';
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
import { tomtomProxy } from './traffic.js';
@@ -19,18 +20,34 @@ import { adsbLolProxy } from './aircraft/adsb-lol.js';
import { aisLiveProxy } from './vessels/ais-live.js';
import { trackBackfillProxies } from './aircraft/tracks.js';
import { openAiRealtimeProxy } from './openai.js';
import { geminiLiveProxy } from './gemini.js';
import { googlePlacesContextProxy } from './places.js';
import { mapillaryProxy } from './mapillary.js';
import { keySetupEndpoint } from '../standalone/key-setup.js';
import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { ghostMapsProxy } from './ghostMaps.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
/**
* Construct the local provider plugins in their established order.
* `realtime` configures the voice session token endpoint.
* `realtime` configures OpenAI; `gemini` explicitly opts into Gemini Live.
* `nominatimStorageDir` binds an owned absolute directory before construction;
* omission retains the durable per-source cache and policy state.
*/
function localProviderPlugins({ realtime } = {}) {
function localProviderPlugins({
realtime,
gemini,
nominatimStorageDir = NOMINATIM_CACHE_DIR,
} = {}) {
if (
typeof nominatimStorageDir !== 'string' ||
!nominatimStorageDir.trim() ||
!path.isAbsolute(nominatimStorageDir)
)
throw new TypeError('nominatimStorageDir must be an absolute directory');
return [
openSkyProxy(),
celestrakProxy(),
@@ -42,7 +59,7 @@ function localProviderPlugins({ realtime } = {}) {
overpassProxy(),
militaryInstallationsProxy(),
regionalBriefProxy(),
geocodeProxy({ storageDir: NOMINATIM_CACHE_DIR }),
geocodeProxy({ storageDir: nominatimStorageDir }),
weatherEffectsProxy(),
cctvProxy({ sourceRoot: defaultSourceRoot }),
radioBrowserProxy(),
@@ -53,11 +70,15 @@ function localProviderPlugins({ realtime } = {}) {
aisLiveProxy(),
trackBackfillProxies(),
openAiRealtimeProxy({ realtime }),
...(gemini ? [geminiLiveProxy({ realtime: gemini })] : []),
googlePlacesContextProxy(),
windProxy(),
weatherProxy(),
cycloneProxy(),
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
ghostMapsProxy(),
keySetupEndpoint(),
];
}
@@ -79,6 +100,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
export { googlePlacesContextProxy } from './places.js';
export { googleServerApiKey } from './places.js';
export { keylessGooglePlacesResponse } from './places.js';
export { keylessGoogleGeocodeResponse } from './places.js';
export { validateGeocodeAddress } from './places.js';
export { geocodeBounds } from './places.js';
export { adsbLolFallbackAnchor } from './aircraft/opensky.js';
export { readResponseTextCapped } from './common/http.js';
export { readResponseJsonCapped } from './common/http.js';
server/providers/mapillary.js
+106
View File
@@ -0,0 +1,106 @@
import { sameSiteGated } from './common/same-site.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import {
fetchTile,
mapillaryToken,
parseTilePath,
retryAfterWait,
TileUpstreamError,
} from './mapillary/tiles.js';
/**
* Tile requests per client IP per minute, cache hits included: over 60 new
* 9-tile views a minute, more than a person flying the camera reaches.
*/
export const TILE_ROUTE_MAX_PER_MIN = 600;
function sendJson(res, status, payload) {
res.statusCode = status;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify(payload));
}
/** GET /api/mapillary/status — whether a token is configured, never its value. */
function handleStatus(req, res) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
}
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — trimmed protobuf tile. */
async function handleTile(req, res, allow) {
if (req.method !== 'GET')
return sendJson(res, 405, { error: 'Method not allowed' });
if (!allow(clientKey(req))) {
res.setHeader('Retry-After', '5');
return sendJson(res, 429, {
error: 'Too many tile requests',
retryAfter: 5,
});
}
const address = parseTilePath((req.url || '').split('?')[0]);
if (!address)
return sendJson(res, 400, {
error: 'Tile path must be /coverage/{z}/{x}/{y} with z 11–14',
});
if (!mapillaryToken())
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
// A client that leaves (a tile the camera moved past) cancels its fetch.
const left = new AbortController();
res.on?.('close', () => {
if (!res.writableEnded) left.abort();
});
try {
const { bytes, source } = await fetchTile(address, { signal: left.signal });
res.statusCode = bytes.length ? 200 : 204;
res.setHeader('Content-Type', 'application/x-protobuf');
res.setHeader('Cache-Control', 'public, max-age=3600');
res.setHeader('X-Gev-Cache', source);
res.end(bytes.length ? bytes : undefined);
} catch (error) {
if (left.signal.aborted) return;
const status = error instanceof TileUpstreamError ? error.status : 0;
// A rejected token is a key problem the panel can name, not a fault.
if (status === 401 || status === 403)
return sendJson(res, 403, {
error: 'Mapillary rejected the access token',
keyRejected: true,
});
if (status === 429) {
const retryAfter = retryAfterWait(error.retryAfter);
res.setHeader('Retry-After', String(retryAfter));
return sendJson(res, 429, {
error: 'Mapillary is rate-limiting tile requests',
retryAfter,
});
}
// An upstream 400 must not read as a malformed request to this proxy.
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
}
}
/**
* Vite plugin: Mapillary coverage tiles with the token added server-side, and
* a status route. Both refuse cross-site requests so another page cannot
* spend the token.
*/
export function mapillaryProxy() {
const install = (middlewares) => {
const allow = makeRateLimiter({
windowMs: 60_000,
max: TILE_ROUTE_MAX_PER_MIN,
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
});
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
middlewares.use(
'/api/mapillary/tiles',
sameSiteGated((req, res) => handleTile(req, res, allow)),
);
};
return {
name: 'mapillary-proxy',
configureServer: (server) => install(server.middlewares),
configurePreviewServer: (server) => install(server.middlewares),
};
}
server/providers/mapillary/tiles.js
+265
View File
@@ -0,0 +1,265 @@
import { readResponseBytesCapped } from '../common/http.js';
import { stripTileLayers } from './trim.js';
/** Mapillary vector tiles, /{layer}/2/{z}/{x}/{y}: the only host the token goes to. */
const TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
/** Only the street zooms the app draws are proxied. */
const MIN_ZOOM = 11;
const MAX_ZOOM = 14;
/**
* The z14 `image` layer is ~98% of a tile (12 MB over a dense city) and unused
* (positions come from the Graph API), so it is dropped in transit.
*/
const DROP_LAYERS = ['image'];
/** A z14 tile with its image layer is ~12 MB; anything past this is wrong. */
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
const FETCH_TIMEOUT_MS = 60_000;
/** Upstream fetches at once; more wait their turn. */
export const TILE_UPSTREAM_CONCURRENCY = 6;
/** Tiles change only when new imagery is processed. */
export const TILE_TTL_MS = 24 * 60 * 60 * 1000;
/** Trimmed tiles are ~80 KB: a dozen 9-tile views, well under 100 MB. */
export const TILE_MEMORY_MAX_TILES = 120;
/** A trimmed tile is ~80 KB; a body this big failed to trim and is not kept. */
export const TILE_MEMORY_MAX_TILE_BYTES = 4 * 1024 * 1024;
/** The longest upstream Retry-After honoured, in seconds. */
export const MAX_RETRY_AFTER_SEC = 600;
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
export function mapillaryToken() {
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
}
/** Mapillary answered with an error status (or a size this proxy refuses). */
export class TileUpstreamError extends Error {
constructor(status, message, retryAfter = null) {
super(message || `Mapillary tiles HTTP ${status}`);
this.name = 'TileUpstreamError';
this.status = status;
/** The upstream Retry-After header, as sent. */
this.retryAfter = retryAfter;
}
}
/** Seconds a Retry-After header asks for (an integer or an HTTP date), else null. */
function retryAfterSeconds(header, now = Date.now()) {
const text = String(header ?? '').trim();
if (!text) return null;
const seconds = /^\d+$/.test(text)
? Number(text)
: (Date.parse(text) - now) / 1000;
return Number.isFinite(seconds) && seconds > 0 ? Math.ceil(seconds) : null;
}
/** The wait a 429 asks for, bounded; a missing or malformed header means a minute. */
export function retryAfterWait(header, now = Date.now()) {
return Math.min(retryAfterSeconds(header, now) ?? 60, MAX_RETRY_AFTER_SEC);
}
/**
* While Mapillary rate-limits, tile misses are refused here until the wait is
* over, so the rest of a view (and the fetches queued behind the first 429)
* do not go on asking. Scoped to the token that was refused.
* @type {{until: number, token: string}|null}
*/
let refusal = null;
/** The refusal still in force for `token`, as the error to answer with. */
function refusedNow(token) {
if (!refusal) return null;
const left = refusal.until - Date.now();
if (left <= 0 || refusal.token !== token) {
refusal = null;
return null;
}
return new TileUpstreamError(429, undefined, String(Math.ceil(left / 1000)));
}
/**
* Parse `/coverage/{z}/{x}/{y}`; null unless z is 11–14 and x/y are on the grid.
* @returns {{z: number, x: number, y: number, key: string}|null}
*/
export function parseTilePath(pathname) {
const match = /^\/coverage\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(pathname);
if (!match) return null;
const [z, x, y] = match.slice(1).map(Number);
if (z < MIN_ZOOM || z > MAX_ZOOM || x >= 2 ** z || y >= 2 ** z) return null;
return { z, x, y, key: `${z}/${x}/${y}` };
}
function trim(bytes) {
try {
return stripTileLayers(bytes, DROP_LAYERS);
} catch (error) {
console.warn('[Mapillary Proxy] tile trim failed:', error?.message);
return bytes;
}
}
/** One tile from Mapillary, trimmed; empty bytes mean no coverage. */
async function fetchUpstream({ z, x, y }, signal) {
const token = mapillaryToken();
const refused = refusedNow(token);
if (refused) throw refused;
const response = await fetch(
`${TILE_HOST}/mly1_public/2/${z}/${x}/${y}?access_token=${encodeURIComponent(token)}`,
{
signal: AbortSignal.any([signal, AbortSignal.timeout(FETCH_TIMEOUT_MS)]),
// The token is in the query: a redirect is never followed (a 3xx is a 502).
redirect: 'manual',
headers: { Accept: 'application/x-protobuf' },
},
);
if (!response.ok || response.status === 204) {
await response.body?.cancel().catch(() => {});
if (response.status === 204 || response.status === 404)
return Buffer.alloc(0);
const retryAfter = response.headers.get('retry-after');
if (response.status === 429)
refusal = {
token,
until: Date.now() + retryAfterWait(retryAfter) * 1000,
};
throw new TileUpstreamError(response.status, undefined, retryAfter);
}
try {
const bytes = await readResponseBytesCapped(response, TILE_MAX_BYTES);
return trim(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.length));
} catch (error) {
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
}
}
let upstreamActive = 0;
/** @type {Array<() => void>} fetches waiting for a slot, oldest first */
const upstreamWaiting = [];
/**
* Run `task` in an upstream slot, waiting for one unless `signal` aborts
* first. A freed slot passes straight to the oldest waiter, still counted, so
* a request arriving meanwhile cannot take it first.
*/
async function withUpstreamSlot(signal, task) {
if (upstreamActive < TILE_UPSTREAM_CONCURRENCY) upstreamActive++;
else
await new Promise((resolve, reject) => {
const wake = () => {
signal.removeEventListener('abort', leave);
resolve();
};
const leave = () => {
const at = upstreamWaiting.indexOf(wake);
if (at !== -1) upstreamWaiting.splice(at, 1);
reject(signal.reason);
};
upstreamWaiting.push(wake);
signal.addEventListener('abort', leave, { once: true });
});
try {
// Handed a slot after leaving: it goes on to the next waiter.
signal.throwIfAborted();
return await task();
} finally {
const next = upstreamWaiting.shift();
if (next) next();
else upstreamActive--;
}
}
/** `promise`, or a rejection as soon as `signal` aborts. */
function unlessAborted(promise, signal) {
if (!signal) return promise;
signal.throwIfAborted();
return new Promise((resolve, reject) => {
const leave = () => reject(signal.reason);
signal.addEventListener('abort', leave, { once: true });
promise
.then(resolve, reject)
.finally(() => signal.removeEventListener('abort', leave));
});
}
/** @type {Map<string, {bytes: Buffer, at: number}>} least recently used first */
const memory = new Map();
/**
* Upstream fetches shared per tile, with the requests still waiting on each.
* @type {Map<string, {promise: Promise<Buffer>, controller: AbortController, waiters: number}>}
*/
const inFlight = new Map();
function remember(key, bytes, at = Date.now()) {
if (bytes.length > TILE_MEMORY_MAX_TILE_BYTES) return;
memory.delete(key);
memory.set(key, { bytes, at });
for (const oldest of memory.keys()) {
if (memory.size <= TILE_MEMORY_MAX_TILES) break;
memory.delete(oldest);
}
}
function recall(key) {
const hit = memory.get(key);
if (!hit) return null;
if (Date.now() - hit.at > TILE_TTL_MS) {
memory.delete(key);
return null;
}
remember(key, hit.bytes, hit.at); // most recently used again
return hit.bytes;
}
/** Start the upstream fetch for a tile, shared until its last request leaves. */
function startFlight(address) {
const controller = new AbortController();
const flight = { controller, waiters: 0 };
flight.promise = withUpstreamSlot(controller.signal, () =>
fetchUpstream(address, controller.signal),
)
.then((bytes) => {
remember(address.key, bytes);
return bytes;
})
.finally(() => {
if (inFlight.get(address.key) === flight) inFlight.delete(address.key);
});
// Every request may have left; the rejection is then nobody's.
flight.promise.catch(() => {});
inFlight.set(address.key, flight);
return flight;
}
/**
* One tile from memory, a shared in-flight fetch, or Mapillary. `signal`
* aborts when the request's client leaves; the fetch stops once all have.
* @param {{z: number, x: number, y: number, key: string}} address from parseTilePath
* @param {{signal?: AbortSignal}} [options]
* @returns {Promise<{bytes: Buffer, source: 'memory'|'inflight'|'upstream'}>}
*/
export async function fetchTile(address, { signal } = {}) {
const cached = recall(address.key);
if (cached) return { bytes: cached, source: 'memory' };
let flight = inFlight.get(address.key);
const shared = Boolean(flight);
flight ||= startFlight(address);
flight.waiters++;
try {
const bytes = await unlessAborted(flight.promise, signal);
return { bytes, source: shared ? 'inflight' : 'upstream' };
} finally {
if (--flight.waiters === 0) {
// A later request for the tile must not join a cancelled fetch.
if (inFlight.get(address.key) === flight) inFlight.delete(address.key);
flight.controller.abort();
}
}
}
/** Test seam: forget every cached and in-flight tile. */
export function _resetTileCacheForTest() {
memory.clear();
inFlight.clear();
refusal = null;
upstreamActive = 0;
upstreamWaiting.length = 0;
}
server/providers/mapillary/trim.js
+43
View File
@@ -0,0 +1,43 @@
import { PbfReader, PbfWriter } from 'pbf';
/** Field number of `layers` in a Mapbox Vector Tile, and of `name` in a layer. */
const TILE_LAYER_FIELD = 3;
const LAYER_NAME_FIELD = 1;
/** Read a layer's name without touching its features. */
function layerName(layerBytes) {
const pbf = new PbfReader(layerBytes);
let name = '';
pbf.readFields((tag, _result, reader) => {
if (tag !== LAYER_NAME_FIELD) return;
name = reader.readString();
reader.pos = reader.length; // the name is all we need
}, null);
return name;
}
/**
* Drop whole layers from a vector tile without decoding kept layers, so a
* 12 MB tile trims in milliseconds. Returns the input when nothing was dropped.
* @param {Buffer|Uint8Array} bytes
* @param {Iterable<string>} dropNames
*/
export function stripTileLayers(bytes, dropNames) {
const drop = new Set(dropNames || []);
if (!bytes?.length || !drop.size) return bytes;
const reader = new PbfReader(bytes);
const writer = new PbfWriter();
let dropped = false;
reader.readFields((tag, _result, pbf) => {
if (tag !== TILE_LAYER_FIELD) return;
const end = pbf.readVarint() + pbf.pos;
const layerBytes = bytes.subarray(pbf.pos, end);
pbf.pos = end;
if (drop.has(layerName(layerBytes))) {
dropped = true;
return;
}
writer.writeBytesField(TILE_LAYER_FIELD, layerBytes);
}, null);
return dropped ? Buffer.from(writer.finish()) : bytes;
}
server/providers/military-installations.js
+11 -2
View File
@@ -9,7 +9,10 @@ import {
fetchOverpassPayload,
overpassNotConfigured,
} from './overpass/transport.js';
import { resolveOverpassUpstreams } from './overpass/constants.js';
import {
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
} from './overpass/constants.js';
import {
_militaryInstallationCache,
trimMilitaryInstallationCache,
@@ -74,7 +77,13 @@ function militaryInstallationsProxy() {
retrievedAt: new Date().toISOString(),
status: 'ready',
};
const entry = { payload, cachedAt: Date.now() };
// Tagged with the configured upstream set so the disk layer does not serve
// this answer back after the operator repoints (see overpass/cache.js).
const entry = {
payload,
cachedAt: Date.now(),
upstreams: resolveOverpassUpstreamTag(),
};
_militaryInstallationCache.set(key, entry);
trimMilitaryInstallationCache();
writeMilitaryInstallationDisk(key, entry);
server/providers/military-installations/cache.js
+4
View File
@@ -5,6 +5,7 @@ import {
MILITARY_INSTALLATION_DISK_DIR,
MILITARY_INSTALLATION_MAX_CACHE,
} from './constants.js';
import { cachedByConfiguredUpstream } from '../overpass/constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
@@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk(
const entry = JSON.parse(
await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'),
);
// Same reason as the Overpass disk cache: the key is the bbox alone, so an
// entry written against a different OVERPASS_UPSTREAMS is otherwise a hit.
if (!cachedByConfiguredUpstream(entry)) return null;
if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null;
return migrateMilitaryInstallationEntry(entry);
} catch {
server/providers/openai.js
+22 -3
View File
@@ -1,20 +1,26 @@
import { defaultSourceRoot } from './common/source-root.js';
import { handleHudSummary } from './openai/hud-summary.js';
import { createDebugLogHandler } from './openai/debug-log.js';
import { createRealtimeTokenHandler } from './openai/realtime.js';
import {
createRealtimeOAuthLoginHandler,
createRealtimeOAuthStatusHandler,
createRealtimeTokenHandler,
} from './openai/realtime.js';
import { sameSiteGated } from './common/same-site.js';
import { createCodexOAuthLogin } from './openai/codex-auth.js';
/**
* Vite plugin: OpenAI Realtime ephemeral client secret.
*
* Keeps OPENAI_API_KEY server-side while the browser connects to the
* Realtime API over WebRTC with a short-lived secret.
* Keeps the selected cloud credential server-side while the browser connects
* to the Realtime API over WebRTC with a short-lived secret.
*/
function openAiRealtimeProxy({
sourceRoot = defaultSourceRoot,
annotationGuidance,
realtime = {},
} = {}) {
const oauthLogin = createCodexOAuthLogin(realtime);
function install(middlewares) {
// Cost-bearing and log endpoints refuse cross-site browser requests
// (see server/providers/common/same-site.js and SECURITY.md).
@@ -31,16 +37,29 @@ function openAiRealtimeProxy({
createRealtimeTokenHandler({ ...realtime, annotationGuidance }),
),
);
middlewares.use(
'/api/realtime/oauth-status',
sameSiteGated(createRealtimeOAuthStatusHandler({ oauthLogin })),
);
middlewares.use(
'/api/realtime/oauth-login',
sameSiteGated(createRealtimeOAuthLoginHandler({ oauthLogin })),
);
}
return {
name: 'openai-realtime-proxy',
configureServer(server) {
install(server.middlewares);
server.httpServer?.once('close', oauthLogin.dispose);
},
configurePreviewServer(server) {
install(server.middlewares);
server.httpServer?.once('close', oauthLogin.dispose);
},
closeBundle: oauthLogin.dispose,
};
}
server/providers/openai/codex-auth.js
+360
View File
@@ -0,0 +1,360 @@
import { spawn } from 'node:child_process';
import {
accessSync,
constants as fsConstants,
readFileSync,
statSync,
} from 'node:fs';
import os from 'node:os';
import path from 'node:path';
export function codexAuthPath({
environment = process.env,
home = os.homedir(),
} = {}) {
const explicit = String(environment.CODEX_AUTH_JSON || '').trim();
if (explicit) return path.resolve(explicit);
const codexHome = String(environment.CODEX_HOME || '').trim();
return path.join(
codexHome ? path.resolve(codexHome) : path.join(home, '.codex'),
'auth.json',
);
}
/** An optional operator lock that refuses the metered API-key voice lane. */
export function preferCodexOAuth(environment = process.env) {
if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false;
const value = String(environment.GEV_PREFER_CODEX_OAUTH || '')
.trim()
.toLowerCase();
if (['1', 'true', 'yes', 'on'].includes(value)) return true;
if (['0', 'false', 'no', 'off'].includes(value)) return false;
throw new Error(
'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.',
);
}
function codexOAuthError(message, code = 'CODEX_OAUTH_UNAVAILABLE') {
const error = new Error(message);
error.code = code;
return error;
}
export function readCodexOAuthAccessToken({
authPath = codexAuthPath(),
readFile = readFileSync,
statFile = statSync,
nowMs = Date.now(),
} = {}) {
const unavailable =
'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.';
let parsed;
try {
parsed = JSON.parse(readFile(authPath, 'utf8'));
} catch {
throw codexOAuthError(unavailable);
}
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw codexOAuthError(unavailable);
}
const mode = String(parsed.auth_mode || '')
.trim()
.toLowerCase();
const chatGptMode = mode
? mode === 'chatgpt' || mode === 'chatgptauthtokens'
: typeof parsed.OPENAI_API_KEY !== 'string';
const token = parsed.tokens?.access_token;
if (!chatGptMode || typeof token !== 'string' || !token.trim()) {
throw codexOAuthError(unavailable);
}
// This only checks token shape and expiry. OpenAI verifies the signature when
// minting the short-lived Realtime credential; we never modify auth.json.
const parts = token.split('.');
if (
parts.length !== 3 ||
parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part))
) {
throw codexOAuthError(unavailable);
}
let payload;
try {
payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
} catch {
throw codexOAuthError(unavailable);
}
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
throw codexOAuthError(unavailable);
}
let expiresAtMs;
if (Object.hasOwn(payload, 'exp')) {
if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) {
throw codexOAuthError(unavailable);
}
expiresAtMs = payload.exp * 1000;
} else {
try {
expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000;
} catch {
throw codexOAuthError(unavailable);
}
}
if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) {
throw codexOAuthError(
'ChatGPT sign-in has expired. Re-authenticate in Provider Settings or run codex login and try again.',
'CODEX_OAUTH_REAUTH_REQUIRED',
);
}
return token;
}
export function resolveCodexExecutable({
environment = process.env,
home = os.homedir(),
access = accessSync,
} = {}) {
const explicit = String(environment.CODEX_BIN || '').trim();
if (explicit) return explicit;
for (const candidate of [
path.join(home, '.local', 'bin', 'codex'),
path.join(
home,
'.codex',
'packages',
'standalone',
'current',
'bin',
'codex',
),
]) {
try {
access(candidate, fsConstants.X_OK);
return candidate;
} catch {
// Fall through to the next known location, then PATH.
}
}
return 'codex';
}
// What `codex login` needs to find its home, open a browser and reach OpenAI.
// The server's own environment also holds every provider secret, so the
// child receives only these names.
const CODEX_LOGIN_ENV =
/^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
function codexLoginEnvironment(environment, codexHome) {
const env = {};
for (const [name, value] of Object.entries(environment || {}))
if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
env.CODEX_HOME = codexHome;
return env;
}
export function startCodexChatGptLogin({
spawnImpl = spawn,
environment = process.env,
home = os.homedir(),
executable = resolveCodexExecutable({ environment, home }),
} = {}) {
// Codex owns its credential storage. An explicit read path must agree with
// the CLI's write path; never silently sign in to another credential store.
const configuredHome = String(environment.CODEX_HOME || '').trim();
const loginHome = configuredHome
? path.resolve(configuredHome)
: path.join(home, '.codex');
if (
codexAuthPath({ environment, home }) !== path.join(loginHome, 'auth.json')
) {
return Promise.reject(
codexOAuthError(
'CODEX_AUTH_JSON does not match the Codex login storage path. Set CODEX_HOME to the matching directory or sign in to Codex separately.',
'CODEX_OAUTH_STORAGE_MISMATCH',
),
);
}
return new Promise((resolve, reject) => {
let child;
let spawned = false;
let finish;
const completion = new Promise((done) => {
finish = done;
});
const fail = () => {
reject(
codexOAuthError(
'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
'CODEX_OAUTH_LOGIN_FAILED',
),
);
finish({ exitCode: null });
};
try {
child = spawnImpl(executable, ['login'], {
cwd: home,
detached: true,
stdio: 'ignore',
env: codexLoginEnvironment(environment, loginHome),
});
} catch {
fail();
return;
}
child.once('error', () => {
if (!spawned) fail();
else finish({ exitCode: null });
});
child.once('exit', (exitCode) => finish({ exitCode }));
child.once('spawn', () => {
spawned = true;
child.unref?.();
resolve({
completion,
cancel() {
try {
child.kill?.('SIGTERM');
} catch {
/* Already stopped. */
}
},
});
});
});
}
/** One local login owner shared by the launch and status endpoints. */
export function createCodexOAuthLogin({
resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
startOAuthLogin = () => startCodexChatGptLogin(),
loginTimeoutMs = 120_000,
setTimer = setTimeout,
clearTimer = clearTimeout,
} = {}) {
let active = null;
let failure = null;
let disposed = false;
function credentials() {
try {
if (resolveOAuthAccessToken()) return { available: true };
} catch (error) {
const code = error?.code || 'CODEX_OAUTH_UNAVAILABLE';
return {
available: false,
error: error?.message || 'ChatGPT sign-in is unavailable',
code,
reauthRequired: code === 'CODEX_OAUTH_REAUTH_REQUIRED',
};
}
return {
available: false,
error: 'ChatGPT sign-in is unavailable',
code: 'CODEX_OAUTH_UNAVAILABLE',
reauthRequired: false,
};
}
function status() {
const current = credentials();
if (current.available) return current;
if (failure) return { available: false, ...failure, loginFailed: true };
return active ? { ...current, pending: true } : current;
}
function finish(attempt, error) {
if (active !== attempt) return;
clearTimer(attempt.timer);
active = null;
failure = error;
}
async function start() {
if (disposed)
throw codexOAuthError(
'ChatGPT sign-in is unavailable because the server has stopped.',
'CODEX_OAUTH_LOGIN_FAILED',
);
if (credentials().available)
return { available: true, started: false, pending: false };
if (active) return { available: false, started: false, pending: true };
// Reserve the attempt before awaiting spawn so concurrent POSTs share it.
const attempt = { process: null, timer: null };
active = attempt;
failure = null;
attempt.timer = setTimer(() => {
finish(attempt, {
code: 'CODEX_OAUTH_LOGIN_TIMEOUT',
error:
'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.',
});
attempt.process?.cancel();
}, loginTimeoutMs);
attempt.timer?.unref?.();
try {
const login = await startOAuthLogin();
if (!login?.completion || typeof login.cancel !== 'function')
throw new Error('Invalid login process');
attempt.process = login;
if (active !== attempt) {
login.cancel();
return { ...status(), started: false, pending: false };
}
Promise.resolve(login.completion).then(
(result) => {
if (active !== attempt) return;
if (result?.exitCode !== 0) {
finish(attempt, {
code: 'CODEX_OAUTH_LOGIN_FAILED',
error:
'ChatGPT sign-in did not complete. Try again in Provider Settings, or run codex login separately.',
});
} else if (!credentials().available) {
finish(attempt, {
code: 'CODEX_OAUTH_STORAGE_UNAVAILABLE',
error:
'Codex sign-in finished, but its configured auth.json has no usable ChatGPT token. Check CODEX_HOME and Codex credential storage. GEV can read file storage only.',
});
} else finish(attempt, null);
},
() =>
finish(attempt, {
code: 'CODEX_OAUTH_LOGIN_FAILED',
error:
'ChatGPT sign-in did not complete. Try again in Provider Settings.',
}),
);
return { available: false, started: true, pending: true };
} catch (error) {
const code =
error?.code === 'CODEX_OAUTH_STORAGE_MISMATCH'
? error.code
: 'CODEX_OAUTH_LOGIN_FAILED';
finish(attempt, {
code,
error:
code === 'CODEX_OAUTH_STORAGE_MISMATCH'
? error.message
: 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
});
return { ...status(), started: false, pending: false };
}
}
return {
status,
start,
dispose() {
disposed = true;
if (!active) return;
const attempt = active;
finish(attempt, null);
attempt.process?.cancel();
},
};
}
server/providers/openai/constants.js
+4
View File
@@ -16,6 +16,9 @@ const OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT = 0.5;
const OPENAI_HUD_SUMMARY_MODEL_DEFAULT = 'gpt-5-nano';
/** Input transcription for voice captions; OPENAI_REALTIME_TRANSCRIBE_MODEL=off disables it. */
const OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT = 'gpt-4o-mini-transcribe';
export {
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
OPENAI_REALTIME_MODEL_MINI_DEFAULT,
@@ -24,4 +27,5 @@ export {
OPENAI_REALTIME_REASONING_DEFAULT,
OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
};
server/providers/openai/debug-log.js
+55 -2
View File
@@ -23,7 +23,59 @@ const REALTIME_DEBUG_LOG_MAX_FILE_BYTES = 32 * 1024 * 1024;
*/
const REALTIME_DEBUG_LOG_MAX_PER_MIN = 120;
function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
/**
* Fields that carry what people said or heard: transcripts, typed and spoken
* text, streamed deltas, tool arguments/results and the span's quoted
* replies. They are dropped from the persisted log unless content logging is explicitly enabled
* (GEV_VOICE_LOG_CONTENT=1), so timing and tool diagnostics never retain
* speech by default.
*/
const VOICE_TEXT_FIELDS = new Set([
'transcript',
'text',
'delta',
'answer_text',
'preamble_text',
]);
const VOICE_CONTENT_MAX_DEPTH = 12;
const VOICE_CONTENT_MAX_DEPTH_MARKER = '[omitted: max depth]';
/** Replace spoken or typed content and complete tool argument/result bodies. */
function omitVoiceContent(value, depth = 0) {
if (depth > VOICE_CONTENT_MAX_DEPTH) return VOICE_CONTENT_MAX_DEPTH_MARKER;
if (value === null || typeof value !== 'object') return value;
if (Array.isArray(value))
return value.map((item) => omitVoiceContent(item, depth + 1));
const output = {};
for (const [key, item] of Object.entries(value)) {
// Tool bodies have open-ended schemas: omit the whole body rather than
// chasing query, label, say, display, or future tool-specific fields.
// Protocol response.output arrays still carry useful event metadata; only
// string output is a serialized function result and must be opaque.
if (
key === 'arguments' ||
key === 'result' ||
(key === 'output' && typeof item === 'string')
) {
output[key] =
typeof item === 'string'
? `[omitted ${item.length} chars]`
: '[omitted]';
continue;
}
if (VOICE_TEXT_FIELDS.has(key) && typeof item === 'string') {
output[key] = `[omitted ${item.length} chars]`;
continue;
}
output[key] = omitVoiceContent(item, depth + 1);
}
return output;
}
function createDebugLogHandler({
sourceRoot = defaultSourceRoot,
includeContent = process.env.GEV_VOICE_LOG_CONTENT === '1',
} = {}) {
const logDir = path.join(sourceRoot, '.gev-logs');
const logFile = path.join(logDir, 'realtime-conversations.jsonl');
const allow = makeRateLimiter({
@@ -79,7 +131,8 @@ function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
try {
const body = await readRequestBody(req, REALTIME_DEBUG_LOG_MAX_BYTES);
const record = JSON.parse(body || '{}');
const parsed = JSON.parse(body || '{}');
const record = includeContent ? parsed : omitVoiceContent(parsed);
// The server's own timestamp comes last, so a record cannot supply one.
await append(
`${JSON.stringify({
server/providers/openai/instructions.js
+19 -26
View File
@@ -1,5 +1,5 @@
const DEFAULT_ANNOTATION_GUIDANCE =
'Use a single annotate_map call with several annotations when you are describing multiple related places at once. Set flyTo true only when the user is not already looking at the place; if every mark in a call lands off-screen the app auto-frames them, so when unsure leave flyTo false. Do NOT say out loud that you are drawing, highlighting, or annotating — just speak naturally about the places while the marks appear. ANNOTATIONS ACCUMULATE AND PERSIST — keep adding marks as you explore; you can fly around, change topic, and jump between far-apart places and the marks STAY, so the user can build up the map and show people things. Do NOT clear on your own initiative: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
function realtimeInstructions(
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
@@ -10,30 +10,27 @@ function realtimeInstructions(
'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
'Only control the app by calling the provided tools. Never invent tool names or arguments.',
'Call tools only for clear GEV control, navigation, visual-style, layer, or app-state requests. For ordinary conversation, answer normally without tools.',
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
'Panels: set_panel_open or show_data_layers_menu; "Open Context" is only set_panel_open{panelId:"global-context-panel",open:true}; "Open Contacts" is set_context_mode contacts (expands Context itself). Unnamed settings: ask "Voice settings or Provider settings?" first; named panels apply directly. Contacts, Space Missions and Cockpit need explicit naming; opening Context or selecting/viewing an aircraft never enters them. In Cockpit, next/previous uses control_cockpit; track_entity/fly_to_location are refused: never retry, exit Cockpit first.',
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
'For "what is this aircraft?" answers, read the callsign, operator, registration, type, and route only from get_entity_context selected.properties. Treat route, routeOrigin, and routeDestination as the only authoritative route fields. Every aircraft identity answer MUST explicitly cover operator, type, and route. When a route is present, repeat its endpoint codes exactly; do not expand airport codes into city names. For a missing field say exactly "Operator details are unavailable", "Aircraft type is unavailable", or "Route details are unavailable" as applicable. Never silently omit missing enrichment or infer it from the callsign.',
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking automatically stops the tracking (the result says so): mention it briefly.',
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.',
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data: report the returned feed source briefly, never call it an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
'For ANALYTICAL questions about layer data — how many / which / fastest / highest / biggest / nearest flights, ships, fires, earthquakes, satellites, datacenters, or dams ("how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Narrate the count plus two or three notable examples by name, and reflect the result\'s coverage note honestly: the answer covers data loaded by enabled layers, not the whole world. If the needed layer is disabled, say so and offer to enable it. For follow-ups about the same set ("which of THOSE is closest?"), call analyst_query with followUp=true and only the new filter/sort.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request.',
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
'FEED STATE CONTRACT — analyst_query and get_current_view_state carry feedProvenance from the same layerFeedState the Data Layers chips use (nominal/loading/degraded/stale/fallback/unavailable, or off). NEVER narrate a count as live or current when feedProvenance.overall is not nominal. Say the feedState in the same breath as the count ("12 flights in view, STALE"). If unavailable, do not invent a count; say the feed is unavailable. Fallback is valid data — name the fallback source, never call it an enable failure. Confirm only what the tool result returned.',
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, "near / nearby / how many aircraft" means the Contacts window: answer from contactsWindow in the tool result — those are the exact numbers on the user\'s panel. set_context_mode, analyst_query, and get_current_view_state carry it after Contacts settles. For "Open Contacts and tell me how many aircraft are within 250 km", call set_context_mode{mode:"contacts"} first and answer from contactsWindow.aircraft; do not answer from a pre-Contacts analyst query. analyst_query\'s own count measures currently-loaded records and is usually lower; never give it as the window count. CENTER PRECEDENCE for a nearby/how-many ask, in order: an explicit place in the question ("over Texas", "near Austin") always wins and ignores Contacts state; else the CONTACTS SUBJECT when Contacts is active and has one — a selected datacenter, dam, fire, or cable does NOT silently become the center; else an entity the user explicitly names ("around this datacenter"); else the current view, said aloud ("nothing is selected, so this is the current view"). With Contacts active but NO subject yet, use the view and say so; never read an empty panel. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two different numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge ("a few", "less than a dozen"): if a tool returns 46, say 46. (5) When it matters, add once: counts cover loaded data, and the flights layer loads where you look.',
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
'If get_entity_context has no selected object or overlay entities, use its basemap context: Google Photorealistic 3D Tiles/Cesium source, center target coordinates, reverse-geocoded place, camera altitude, active style, and enabled layers. Do not say there is nothing unless the basemap target is also unavailable.',
'If basemap context includes knownLandmarks, prefer the nearest known landmark by name for "what am I looking at" answers. For example, if knownLandmarks includes Eiffel Tower, say Eiffel Tower.',
'At local zoom, use basemap nearbyPlaces, place.labels, viewportPlaces.visibleLabels, and viewportPlaces.streetLabels to identify the building, premises, roads, and named places visible around the screen target.',
'If basemap context includes viewportPlaces, prefer dominantCountry, dominantRegion, and dominantLocality over raw coordinates.',
'When basemap context includes viewportSamples or an inferred country, trust that over a single reverse-geocoded address. If most samples indicate Iran, say Iran, not the United States.',
'When a viewport screenshot is attached after get_entity_context, read clearly legible street, building, and place labels from it and combine them with structured label context. Respect scene viewScale: at global/continental/regional scale, avoid naming a precise street/city from one center pixel.',
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
'Do not mention disabled layers or stale selections.',
'When a request requires a tool call, do not speak in the same response as the tool call. Call the tool first.',
'When a single user request contains MULTIPLE changes (e.g. "switch to operator layout, use balanced detection at density 50, and switch to Bing aerial"), call ALL the corresponding tools — multiple tool calls in sequence — before speaking. Never confirm a partial subset. If a later tool fails, say which parts succeeded and which failed.',
'After receiving tool output, speak exactly one short confirmation. Do not repeat the confirmation.',
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
//
@@ -50,27 +47,23 @@ function realtimeInstructions(
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any mention of "satellite" or "satellites" ALWAYS means the satellites DATA LAYER via set_layer_visibility, never a basemap. "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
"After frame_overhead, speak ONLY from the tool result's count field — e.g. 'Framed fourteen aircraft, labels on'; never reassess or second-guess the count aloud.",
'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
'Keep spoken confirmations short, e.g. "Opening datacenters" or "Flying to London".',
'WHITEBOARD THE WORLD: whenever you describe or explain a specific place, building, campus, district, boundary, or a spatial relationship between places, call annotate_map to mark it visually as you talk — like sketching on the map. To call out a specific building, campus, compound, park, or district, use type=area (it traces and encloses the real footprint — a building gets a glowing volume, a district gets a draped outline). Use type=highlight only for a transient pulse on a precise spot that has no meaningful footprint, and type=pin to drop a labeled marker. Examples: "what is the Palace of Fine Arts?" → an AREA on it; "the old military base next to it" → an AREA on the Presidio; "ILM is right here" → a pin; "it sits next to the Marina" → an arrow from one to the other. Prefer place NAMES so the app resolves real positions and outlines; never invent coordinates or pixel locations.',
'On every annotation, also set entityKind to what the thing IS when you know it: building (one structure), compound (campus/grounds/mall/park), district (neighborhood/area of a city), street (a named road), or point_feature (a monument, statue, memorial, plaque, fountain, or other small point landmark). entityKind is a FACT about the target, independent of the mark type you chose — monuments and statues are point_feature even when you use type=area; the app then anchors them as precise points instead of guessing at a footprint.',
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
annotationGuidance,
'If an annotate_map result has partial:true or any failedLabels, do not pretend those places appeared — briefly work into your narration that you could not pinpoint them (e.g. "I couldn\'t place X"). If a route comes back as a direct line (no street route was found), describe it as a straight-line distance, not a walking/driving time. If an annotate_map result has capped:true, the map is full — ASK the user whether to clear before drawing more; do not clear unprompted. outlinePending:true is NOT a failure, but it is also NOT an outline: the anchor mark is placed and the boundary is still being traced in the background. Narrate it in progress — e.g. "tracing the boundary now" — and NEVER state the outline is already drawn or visible; it may yet come back as just a point. A later system item of type map_annotation_outline reports the final outcome per mark (status resolved or failed, with its label): use it to quietly confirm, or to correct yourself if you implied a boundary that stayed a point — an honest miss beats a misleading guess.',
'PREFER NAMES. Only when you cannot name or geocode a place but you can clearly SEE the exact spot in the most recent viewport screenshot, fall back to screenX/screenY (normalized 0..1 from that image) to point at it; the app converts the pixel to a real world point. Never use screenX/screenY for something you could name.',
'PATHS vs DISTANCES: for "walking/driving route from A to B" (or through several stops), use type=route with the ordered points and the matching mode (walking/driving/cycling) — the app draws the real street-following path on the map and reports distance and travel time, which you can read aloud. For "how far is X from Y", "is it nearby", or "X is next to Y", use type=arrow between the two — it draws a floating connector and shows the straight-line distance. Do NOT use route for a simple distance/proximity question.',
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
].join('\n');
}
server/providers/openai/realtime.js
+215 -17
View File
@@ -10,15 +10,130 @@ import {
OPENAI_REALTIME_REASONING_DEFAULT,
OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
} from './constants.js';
import { realtimeInstructions } from './instructions.js';
import { GEV_REALTIME_TOOLS } from './tools.js';
import {
preferCodexOAuth,
readCodexOAuthAccessToken,
createCodexOAuthLogin,
} from './codex-auth.js';
const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']);
function isLoopbackRequest(req) {
return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || ''));
}
function isSameOriginRequest(req) {
const origin = String(req.headers?.origin || '').trim();
if (!origin) return true;
const host = String(req.headers?.host || '').trim();
if (!host) return false;
const protocol = req.socket?.encrypted ? 'https:' : 'http:';
try {
return new URL(origin).origin === new URL(`${protocol}//${host}`).origin;
} catch {
return false;
}
}
function requestQuery(req) {
try {
return new URL(req.url || '', 'http://localhost').searchParams;
} catch {
return new URLSearchParams();
}
}
function resolveRealtimeAuthMode(query) {
const requested = String(query.get('auth') || 'api-key').toLowerCase();
if (requested === 'api-key' || requested === 'oauth') return requested;
return null;
}
function createRealtimeOAuthStatusHandler({
oauthLogin = createCodexOAuthLogin(),
} = {}) {
return (req, res) => {
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Content-Type', 'application/json');
if (req.method !== 'GET') {
res.statusCode = 405;
res.end(JSON.stringify({ error: 'Method not allowed' }));
return;
}
if (!isLoopbackRequest(req)) {
res.statusCode = 403;
res.end(
JSON.stringify({
available: false,
error: 'ChatGPT OAuth is available only from this machine',
}),
);
return;
}
res.statusCode = 200;
res.end(JSON.stringify(oauthLogin.status()));
};
}
function createRealtimeOAuthLoginHandler({
oauthLogin = createCodexOAuthLogin(),
} = {}) {
return async (req, res) => {
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Content-Type', 'application/json');
if (req.method !== 'POST') {
res.statusCode = 405;
res.end(JSON.stringify({ error: 'Method not allowed' }));
return;
}
if (!isLoopbackRequest(req)) {
res.statusCode = 403;
res.end(
JSON.stringify({
available: false,
error: 'ChatGPT OAuth sign-in is available only from this machine',
}),
);
return;
}
if (!isSameOriginRequest(req)) {
res.statusCode = 403;
res.end(
JSON.stringify({
available: false,
error: 'Cross-origin ChatGPT OAuth sign-in is not allowed',
}),
);
return;
}
try {
const result = await oauthLogin.start();
res.statusCode = result.available ? 200 : result.loginFailed ? 503 : 202;
res.end(JSON.stringify(result));
} catch {
res.statusCode = 503;
res.end(
JSON.stringify({
available: false,
error: 'Could not start ChatGPT sign-in',
}),
);
}
};
}
function createRealtimeTokenHandler({
annotationGuidance,
endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
fetchImpl = (...args) => fetch(...args),
resolveApiKey = () => process.env.OPENAI_API_KEY,
resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
environment = process.env,
models = {},
tools = GEV_REALTIME_TOOLS,
} = {}) {
@@ -31,15 +146,84 @@ function createRealtimeTokenHandler({
return;
}
const query = requestQuery(req);
const authMode = resolveRealtimeAuthMode(query);
if (!authMode) {
res.statusCode = 400;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' }));
return;
}
let oauthPinned;
try {
oauthPinned = preferCodexOAuth(environment);
} catch (error) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: error.message }));
return;
}
if (oauthPinned && authMode !== 'oauth') {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({
error:
'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.',
code: 'CODEX_OAUTH_REQUIRED',
}),
);
return;
}
// Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
const apiKey = resolveApiKey();
if (!apiKey) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
return;
let bearer;
if (authMode === 'oauth') {
if (!isLoopbackRequest(req)) {
res.statusCode = 403;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({
error: 'ChatGPT OAuth voice is available only from this machine',
}),
);
return;
}
try {
bearer = resolveOAuthAccessToken();
} catch (error) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({
error: error?.message || 'ChatGPT sign-in is unavailable',
code: error?.code || 'CODEX_OAUTH_UNAVAILABLE',
}),
);
return;
}
if (!bearer) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({
error: 'ChatGPT sign-in is unavailable',
code: 'CODEX_OAUTH_UNAVAILABLE',
}),
);
return;
}
} else {
bearer = resolveApiKey();
if (!bearer) {
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
return;
}
}
// Voice model tier, requested by the client as ?tier=standard|mini.
@@ -48,15 +232,7 @@ function createRealtimeTokenHandler({
// bad querystring degrades to a normal session rather than a dead mic.
// The env overrides stay authoritative per tier (see .env.example) —
// a wrong upstream model id is then a config fix, not a code change.
const requestedTier = (() => {
try {
return new URL(req.url || '', 'http://localhost').searchParams.get(
'tier',
);
} catch {
return null;
}
})();
const requestedTier = query.get('tier');
const tier = resolveVoiceModel(requestedTier).tier;
const model =
tier === 'mini'
@@ -89,6 +265,14 @@ function createRealtimeTokenHandler({
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
),
);
// User captions for the voice card. Billed separately from the session.
const transcribeModel =
process.env.OPENAI_REALTIME_TRANSCRIBE_MODEL ||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT;
const transcription =
transcribeModel.toLowerCase() === 'off'
? {}
: { transcription: { model: transcribeModel } };
const sessionConfig = {
session: {
type: 'realtime',
@@ -104,6 +288,7 @@ function createRealtimeTokenHandler({
audio: {
input: {
noise_reduction: { type: 'near_field' },
...transcription,
turn_detection: {
type: 'semantic_vad',
eagerness: 'low',
@@ -125,7 +310,7 @@ function createRealtimeTokenHandler({
redirect: 'error',
signal: AbortSignal.timeout(30_000),
headers: {
Authorization: `Bearer ${apiKey}`,
Authorization: `Bearer ${bearer}`,
'Content-Type': 'application/json',
'OpenAI-Safety-Identifier': 'gev-local-dev',
},
@@ -139,6 +324,15 @@ function createRealtimeTokenHandler({
// case where a bogus ?tier= was silently downgraded to standard.
res.setHeader('X-GEV-Voice-Tier', tier);
res.setHeader('X-GEV-Voice-Model', model);
res.setHeader(
'X-GEV-Voice-Auth',
authMode === 'oauth' ? 'codex-oauth' : 'env',
);
// Captions are billed separately; the client meters them with this id.
res.setHeader(
'X-GEV-Voice-Transcribe-Model',
transcription.transcription ? transcribeModel : 'off',
);
if (requestedTier && !isKnownVoiceTier(requestedTier)) {
res.setHeader('X-GEV-Voice-Tier-Fallback', '1');
}
@@ -168,4 +362,8 @@ function createRealtimeTokenHandler({
};
}
export { createRealtimeTokenHandler };
export {
createRealtimeOAuthLoginHandler,
createRealtimeOAuthStatusHandler,
createRealtimeTokenHandler,
};
server/providers/openai/toolDescriptions.js
+64 -37
View File
@@ -1,3 +1,8 @@
import {
voiceLayerAliasHint,
voiceQueryFieldHint,
} from '../../../src/voice/layerManifest.js';
export const ACTION_DESCRIPTIONS = {
fly_to_location: {
description:
@@ -12,7 +17,11 @@ export const ACTION_DESCRIPTIONS = {
},
query: {
description:
'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina".',
'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina". "pointer" = the spot under the cursor.',
$position: 1,
},
referent: {
description: 'Numbered item n from the last result list.',
$position: 1,
},
viewMode: {
@@ -35,7 +44,7 @@ export const ACTION_DESCRIPTIONS = {
},
select_nearest_aircraft: {
description:
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result. This does not open Contacts or Cockpit.',
'Atomically fly to a place, wait for arrival, enable and load Flights or Military Flights in that viewport, exclude on-ground records, and select/follow the nearest airborne aircraft. Healthy fallback feeds remain usable and are reported in the result.',
$position: 1,
parameters: {
properties: {
@@ -76,13 +85,13 @@ export const ACTION_DESCRIPTIONS = {
$position: 1,
},
set_layer_visibility: {
description: "Enable or disable one registered God's Eye View data layer.",
description:
"Enable or disable one God's Eye View data layer. Every shipped layer is in the enum; never tell the user a listed layer does not exist.",
$position: 1,
parameters: {
properties: {
layerId: {
description:
'Common-name mapping for the non-obvious ids: space mission(s) → rocket-launches; fires/wildfires/active fires → local-firms (NASA FIRMS); ships/vessels/boats → ais-live-vessels; undersea/submarine cables → telegeography-submarine-cables; datacenters → local-datacenters; dams → local-dams; bikes/bike share → bikeshare; street traffic/congestion → traffic; traffic cameras → cctv; internet radio/stations → radio; ALPR/license plate readers/Flock cameras → alpr-cameras; local ADS-B/my receiver/my antenna (aircraft heard by a local RTL-SDR receiver) → local-adsb.',
description: `Spoken names: ${voiceLayerAliasHint()}.`,
$position: 1,
},
},
@@ -102,12 +111,18 @@ export const ACTION_DESCRIPTIONS = {
},
},
set_panel_open: {
description: 'Open or close a GEV UI panel/dropdown.',
description:
'Visual Presets/styles/filters=control-panel; Scenes/saved scenes=scene-panel; Voice Settings=voice-settings; voice control/Preview=voice-preview; Provider Settings/API keys=provider-settings. Voice panels exit Cockpit directly.',
$position: 1,
},
set_voice_inactivity_timeout: {
description:
'Set the requested exact inactivity duration directly: integer minutes 1–60, 0=None. Report applied/persisted truthfully; an unsaved live change is not saved.',
$position: 1,
},
set_context_mode: {
description:
'Enter or exit the Global Context sub-mode used by Contacts and Space Missions. Use Contacts only when the user explicitly requests Contacts, and Space Missions only when explicitly requested. A request to open the parent Context panel alone uses set_panel_open and must not activate either sub-mode. Selecting an aircraft does not imply Context.',
'Enter/exit Contacts or Space Missions only when explicitly requested; opening Context alone uses set_panel_open.',
$position: 1,
parameters: {
properties: {
@@ -120,23 +135,23 @@ export const ACTION_DESCRIPTIONS = {
},
control_cockpit: {
description:
'Read or control Cockpit when the user explicitly requests Cockpit: establish Contacts and enter from a selected or tracked aircraft; exit; or navigate nearby Contacts with optional filters. Selecting or viewing an aircraft alone must not enter Cockpit.',
'Explicit Cockpit requests only: enter from a selected/tracked aircraft (establishes Contacts itself; no set_context_mode), exit, status, or next/previous. Explicit exit-and-show voice: voicePanel preview/settings (Voice Settings), no confirmation.',
$position: 1,
parameters: {
properties: {
action: {
description:
'previous/next (or prev) navigates through nearby contacts in Cockpit context.',
'previous/next (or prev) cycles nearby Cockpit contacts.',
$position: 2,
},
targetLayer: {
description:
'Optional contact layer filter for next/previous (for example military for a military-only cycle).',
'Optional next/previous layer filter (e.g. military-only).',
$position: 2,
},
aircraftClass: {
description:
'Optional aircraft class filter (for example helicopter) when using next/previous navigation.',
'Optional next/previous aircraft class filter (e.g. helicopter).',
$position: 1,
},
},
@@ -154,9 +169,13 @@ export const ACTION_DESCRIPTIONS = {
properties: {
scope: {
description:
'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center.',
'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center; pointer returns what the cursor is on.',
$position: 2,
},
referent: {
description: 'Numbered item n from the last result list.',
$position: 1,
},
layerId: {
description: 'Optional layer filter for visible entity context.',
$position: 2,
@@ -166,7 +185,7 @@ export const ACTION_DESCRIPTIONS = {
},
get_current_view_state: {
description:
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
$position: 1,
},
set_hud: {
@@ -349,12 +368,17 @@ export const ACTION_DESCRIPTIONS = {
properties: {
query: {
description:
'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id.',
'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id. "pointer" = the contact under the cursor.',
$position: 1,
},
referent: {
description:
'Numbered item n from the last result list (-1 = last); query may then be its label.',
$position: 1,
},
layerId: {
description:
'Optional layer hint: flights | military | ais-live-vessels | satellites.',
'Optional: flights, military, ais-live-vessels or satellites. Other layers are not trackable; for fires use query "biggest fire".',
$position: 1,
},
},
@@ -381,7 +405,7 @@ export const ACTION_DESCRIPTIONS = {
},
annotate_map: {
description:
'Draw annotations on the 3D map to visually point out what you are talking about — like sketching on a whiteboard over the world. Use this whenever you mention a specific place, building, campus, boundary, district, or a relationship between two places, so the user can SEE what you mean. Give place NAMES (preferred) or explicit lat/lng; the app resolves them to real-world positions and real building/area outlines — never guess pixel positions. Call this as you begin describing something, and you may mark several places in one call.',
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
$position: 1,
parameters: {
properties: {
@@ -398,7 +422,7 @@ export const ACTION_DESCRIPTIONS = {
},
target: {
description:
'Place name to resolve, e.g. "Palace of Fine Arts, San Francisco", "the Pentagon", "Presidio of San Francisco". Preferred over coordinates. For a specific monument/statue/feature that sits within a larger landmark, use its OWN name + city ("Tejano Monument, Austin", "Texas African American History Memorial, Austin") — do NOT phrase it as "X at the Texas State Capitol", which makes the geocoder collapse several of them onto the same centroid so they stack on one spot.',
'Place name to resolve ("Palace of Fine Arts, San Francisco"), or "pointer" for the spot under the cursor. A monument inside a larger landmark gets its own name + city ("Tejano Monument, Austin"), never "X at the Texas State Capitol" (that stacks them on one centroid).',
$position: 2,
},
points: {
@@ -450,27 +474,24 @@ export const ACTION_DESCRIPTIONS = {
},
entityKind: {
description:
'What KIND of thing the target IS — a fact, not a style choice: building = one structure; compound = campus/grounds/mall/park; district = neighborhood or area of a city; street = a named road/corridor; point_feature = monument/statue/memorial/plaque/fountain or other small point landmark. Set it whenever you know it — it routes the resolver to the right footprint source (point_feature anchors monuments as precise points instead of adopting a nearby building outline).',
'What the target IS (a fact, not a style): building = one structure; compound = campus/grounds/mall/park; district = neighborhood; street = a named road; point_feature = monument/statue/memorial/fountain, anchored as a precise point. Set it whenever you know it.',
$position: 2,
},
screenX: {
description:
'Fallback only: when you cannot name/geocode the place but can SEE it in the latest viewport screenshot, the normalized horizontal position (0=left, 1=right) of the spot. The app converts it back to a real world point under that pixel.',
'Fallback only, for a spot visible in the latest screenshot that you cannot name: normalized x (0=left, 1=right).',
$position: 3,
},
screenY: {
description:
'Fallback only: normalized vertical position (0=top, 1=bottom) of the spot in the latest viewport screenshot.',
description: 'Fallback only: normalized y (0=top, 1=bottom).',
$position: 3,
},
toScreenX: {
description:
'For type=arrow: normalized x of the arrow destination from the screenshot (pixel fallback).',
description: 'For type=arrow: destination x (pixel fallback).',
$position: 3,
},
toScreenY: {
description:
'For type=arrow: normalized y of the arrow destination from the screenshot (pixel fallback).',
description: 'For type=arrow: destination y (pixel fallback).',
$position: 3,
},
},
@@ -496,18 +517,18 @@ export const ACTION_DESCRIPTIONS = {
},
move_camera: {
description:
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").',
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.',
$position: 1,
parameters: {
properties: {
direction: {
description:
'Required except for orbit (defaults right/clockwise) and stop.',
'Required for pan/tilt/rotate; orbit/rotate_globe default right.',
$position: 2,
},
mode: {
description:
'once = bounded eased nudge (default); continuous = until stop/manual input.',
'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.',
$position: 2,
},
},
@@ -528,18 +549,18 @@ export const ACTION_DESCRIPTIONS = {
},
analyst_query: {
description:
'Answer questions ABOUT the data currently loaded on the map — counts, lists, superlatives, and attribute filters over live layers (flights, military, ships, fires, earthquakes, satellites, datacenters, dams). Examples: "how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet", "fastest thing in view". For satellites and infrastructure, counts and ranks cover only bounded examined loaded records; omitted records can change nearest/count. Queries ONLY client-side data from ENABLED layers — if the needed layer is off, say so and offer to enable it. For a follow-up about the previous answer\'s set ("which of those is closest?"), set followUp=true and send only the new filters/sort.',
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
$position: 1,
parameters: {
properties: {
layers: {
description:
'Layers to query. fires/wildfires → local-firms; ships/vessels → ais-live-vessels.',
'Layers to query. fires → local-firms; ships → ais-live-vessels; buses/trains → transit; hurricanes → weather-cyclones; bases → military-installations; my receiver → local-adsb. Units differ per layer, so rank speed within one layer.',
$position: 2,
},
scope: {
description:
'Spatial scope. Default: view (near the camera). Use kind=region for "over Texas"-style asks; kind=anywhere for global questions.',
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
$position: 2,
properties: {
name: {
@@ -548,23 +569,29 @@ export const ACTION_DESCRIPTIONS = {
$position: 1,
},
km: {
description: 'For kind=radius.',
description:
'For kind=radius or pointer: the actual requested kilometers. An active-subject 250 km request uses 250; another requested radius keeps that number.',
$position: 1,
},
center: {
description:
'Omit for a radius around the active Contacts subject. Supply only an explicitly requested coordinate or a resolved named-place coordinate; never invent 0,0 or replace an explicit center with the subject.',
$position: 1,
},
},
},
filters: {
description:
'Attribute predicates, ANDed. ALTITUDE IS METERS (40,000 ft = 12192). Fields: altitudeM, speedMps, military, onGround, aircraftClass, callsign, operator, routeOrigin, routeDestination, originCountry (flights); speedKts, shipType, destination (ships); frp, confidence (fires); magnitude, depthKm, place (earthquakes).',
description: `ANDed; values keep their type (numbers, true/false). Altitude is meters (40,000 ft = 12192). Time fields take ISO; ageHours = hours ago. Every layer has id, lat, lon. Fields — ${voiceQueryFieldHint()}.`,
$position: 1,
},
sortBy: {
description: 'Field to rank by, or "distance" for nearest-first.',
description:
'Field to rank by, or "distance" (nearest first unless sortDir=desc).',
$position: 1,
},
followUp: {
description:
'true = re-query the PREVIOUS result set instead of fresh data.',
'true = re-filter the last answer (its layers) instead of fresh data.',
$position: 1,
},
},
@@ -572,7 +599,7 @@ export const ACTION_DESCRIPTIONS = {
},
next_iss_pass: {
description:
"When the user asks when the ISS / the space station will next fly over: returns the next geometric ISS pass with estimated visibility for the current camera location (or an explicit lat/lon) — rise time (ISO + minutes from now), rise compass direction, peak elevation, and duration. Requires the satellites layer to have loaded its catalog at least once this session; if it hasn't, tell the user to enable the satellites layer and try again.",
'Next ISS pass over the camera location (or lat/lon): rise time, direction, peak elevation, duration and estimated visibility. Needs the satellites layer to have loaded once; if not, ask the user to enable it.',
$position: 1,
parameters: {
properties: {
server/providers/overpass.js
+52 -59
View File
@@ -1,5 +1,6 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { readRequestBodyCapped } from './common/request.js';
import { admitSameSite } from './common/same-site.js';
import {
OVERPASS_MAX_BODY_BYTES,
OVERPASS_MAX_CONCURRENT,
@@ -27,6 +28,14 @@ const _overpassInFlight = new Map();
let _overpassConcurrent = 0;
// Bound body-reading and validation work, including malformed requests and
// cache hits. Keep the tighter upstream budget below independent of this gate.
const _overpassRequestLimiter = makeRateLimiter({
windowMs: 60_000,
max: 300,
globalMax: 900,
});
const _overpassRateLimiter = makeRateLimiter({
windowMs: 60_000,
max: 90,
@@ -94,6 +103,17 @@ function overpassProxy({ routing = {} } = {}) {
return;
}
if (admitSameSite(req, res)) return;
if (!_overpassRequestLimiter(clientKey(req))) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
'Retry-After': '60',
});
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
// Collect POST body with a hard byte cap (Overpass QL queries are small)
let body;
try {
@@ -144,6 +164,30 @@ function overpassProxy({ routing = {} } = {}) {
readDisk: () =>
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
loadUpstream: async () => {
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
return {
status: 503,
contentType: 'application/json',
retryAfterMs: 2000,
body: JSON.stringify({
error: 'Overpass proxy busy — try again shortly',
}),
};
_overpassConcurrent++;
try {
const payload = await fetchOverpassPayload(safeBody);
if (overpassPayloadIsData(payload)) {
const entry = { ...payload, cachedAt: Date.now() };
_overpassCache.set(cacheKey, entry);
trimOverpassCache();
writeOverpassDisk(cacheKey, entry);
}
return payload;
} finally {
_overpassConcurrent--;
}
},
});
if (preflight.source === 'RATE_LIMITED') {
res.writeHead(429, {
@@ -153,71 +197,20 @@ function overpassProxy({ routing = {} } = {}) {
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
if (preflight.source !== 'UPSTREAM') {
// A coalesced caller sees the same failure as the original request
// and must get the same last-good fallback, not the raw refusal.
if (!overpassPayloadIsData(preflight.payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
if (preflight.source === 'DISK') {
_overpassCache.set(cacheKey, preflight.payload);
trimOverpassCache();
}
sendOverpassResponse(res, preflight.payload, preflight.source);
return;
}
// From here onward the request is genuinely upstream-bound and has
// consumed one local limiter slot. Cache and dedupe hits above do not.
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
res.writeHead(503, {
'Content-Type': 'application/json',
'Retry-After': '2',
});
res.end(
JSON.stringify({
error: 'Overpass proxy busy — try again shortly',
}),
);
return;
}
_overpassConcurrent += 1;
const requestPromise = fetchOverpassPayload(safeBody)
.then((payload) => {
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
// refusal was written to memory AND disk — and boundary-class
// queries hold a month-long TTL, so a single 406 outlived the
// outage that caused it.
if (overpassPayloadIsData(payload)) {
const entry = { ...payload, cachedAt: Date.now() };
_overpassCache.set(cacheKey, entry);
trimOverpassCache();
writeOverpassDisk(cacheKey, entry);
}
return payload;
})
.finally(() => {
_overpassConcurrent -= 1;
_overpassInFlight.delete(cacheKey);
});
_overpassInFlight.set(cacheKey, requestPromise);
const payload = await requestPromise;
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
// error): last-good roads beat an empty layer — serve stale from
// memory or disk at ANY age before surfacing the failure.
if (!overpassPayloadIsData(payload)) {
if (!overpassPayloadIsData(preflight.payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
sendOverpassResponse(res, payload, 'MISS');
// Preflight already published the disk hit to memory; only trim.
if (preflight.source === 'DISK') trimOverpassCache();
sendOverpassResponse(
res,
preflight.payload,
preflight.source === 'UPSTREAM' ? 'MISS' : preflight.source,
);
} catch (e) {
// Every mirror threw (network-level). Same serve-stale rule.
const stale = cacheKey ? await readStaleOverpass(cacheKey) : null;
server/providers/overpass/cache.js
+35 -13
View File
@@ -1,3 +1,4 @@
import { coalesceProxyRequest } from '../common/http.js';
import { isOverpassBoundaryQuery } from './query.js';
import {
OVERPASS_BOUNDARY_DISK_TTL_MS,
@@ -5,6 +6,7 @@ import {
OVERPASS_DISK_DIR,
OVERPASS_CACHE_MS,
OVERPASS_CACHE_MAX_ENTRIES,
cachedByConfiguredUpstream,
} from './constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
@@ -47,6 +49,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
// them on both fresh and stale reads so an upgrade can recover immediately.
if (!overpassPayloadIsData(payload)) return null;
// The key is the query alone, so a file written by a different upstream —
// including the public instances that were the default before #742 — is a
// hit for the same query. Refuse it: the operator's configured instance
// must be what answers, on the stale path too.
if (!cachedByConfiguredUpstream(payload)) return null;
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
return payload;
} catch {
@@ -72,7 +79,9 @@ function writeOverpassDisk(cacheKey, payload) {
/**
* Resolve every cache/coalescing layer before admitting a request to the local
* upstream rate limiter. The injected limiter callback is invoked exactly once
* for a complete cache miss and never for memory, in-flight, or disk hits.
* for a complete cache miss and never for memory, in-flight, or disk hits. A
* caller that joined a reservation rejected by another client's limiter
* retries with its own limiter instead of inheriting that 429.
* Exported so the admission ordering can be tested without a Vite server.
*
* @param {object} options
@@ -81,6 +90,7 @@ function writeOverpassDisk(cacheKey, payload) {
* @param {Map<string, Promise<object>>} options.inFlight
* @param {()=>Promise<object|null>} options.readDisk
* @param {()=>boolean} options.allowUpstream
* @param {()=>Promise<object>} options.loadUpstream Fetch and publish the admitted result.
* @param {number} [options.now]
* @param {number} [options.cacheMs]
* @returns {Promise<{source:'HIT'|'INFLIGHT'|'DISK'|'UPSTREAM'|'RATE_LIMITED', payload:object|null}>}
@@ -91,22 +101,34 @@ async function resolveOverpassPreflight({
inFlight,
readDisk,
allowUpstream,
loadUpstream = async () => null,
now = Date.now(),
cacheMs = OVERPASS_CACHE_MS,
}) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
return { source: 'HIT', payload: cached };
for (;;) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
return { source: 'HIT', payload: cached };
const pending = inFlight.get(cacheKey);
if (pending) return { source: 'INFLIGHT', payload: await pending };
const disk = await readDisk();
if (overpassPayloadIsData(disk)) return { source: 'DISK', payload: disk };
return allowUpstream()
? { source: 'UPSTREAM', payload: null }
: { source: 'RATE_LIMITED', payload: null };
// Reserve synchronously before disk I/O. The owner retains the reservation
// through its own upstream admission, fetch and cache publication.
const request = coalesceProxyRequest(inFlight, cacheKey, async () => {
const disk = await readDisk();
if (overpassPayloadIsData(disk)) {
memoryCache.set(cacheKey, disk);
return { source: 'DISK', payload: disk };
}
if (!allowUpstream()) return { source: 'RATE_LIMITED', payload: null };
return { source: 'UPSTREAM', payload: await loadUpstream() };
});
const result = await request.promise;
if (!request.shared) return result;
// The owner's quota rejection belongs to the owner's client. The rejected
// reservation is released before joiners resume, so retry and let this
// caller own the next reservation (or join an admitted one) on its quota.
if (result.source === 'RATE_LIMITED') continue;
return { ...result, source: 'INFLIGHT' };
}
}
/** Return only last-good Overpass data, regardless of its age. */
server/providers/overpass/constants.js
+59 -4
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { createHash } from 'node:crypto';
// ---------------------------------------------------------------------------
// Overpass API proxy constants and cache state
@@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) {
return endpoints.slice(0, 8);
}
let upstreamMemo = { raw: null, endpoints: [] };
let upstreamMemo = { raw: null, endpoints: [], tag: 'none' };
/**
* Short, stable identifier for the configured upstream set. Cached responses
* carry it so an answer produced by a different instance is not served after
* the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the
* configuration change by 7 days, a month for boundary queries, and forever on
* the serve-stale path. It is a hash, never the URLs, and it is only ever
* compared on read: it is not sent to clients and does not appear in errors.
*/
function upstreamTag(endpoints) {
if (!endpoints.length) return 'none';
// Sorted, so reordering the same instances for failover keeps the cache.
return createHash('sha256')
.update([...endpoints].sort().join('\n'))
.digest('hex');
}
/** Resolve after environment loading. Public Overpass instances are not used by default. */
function resolveOverpassUpstreams() {
const raw = process.env.OVERPASS_UPSTREAMS || '';
if (upstreamMemo.raw !== raw)
upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) };
readUpstreamMemo();
return [...upstreamMemo.endpoints];
}
/** The configured-upstream tag stored alongside, and checked against, cache entries. */
function resolveOverpassUpstreamTag() {
return readUpstreamMemo().tag;
}
let reportedForeignCache = false;
/**
* Was this cached entry produced by the upstream set configured now?
*
* With nothing configured there is no instance to contradict, and serving
* last-good data at any age is the point of the cache (the proxy's
* not-configured path depends on it), so any entry is accepted. Once an
* operator does configure an instance, only its own answers are served: an
* entry with no tag, or one from a different instance, is refused rather than
* matched on the query alone. Shared by both caches fed by these upstreams.
*/
function cachedByConfiguredUpstream(entry) {
const tag = resolveOverpassUpstreamTag();
if (tag === 'none' || entry?.upstreams === tag) return true;
if (!reportedForeignCache) {
reportedForeignCache = true;
console.warn(
'[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.',
);
}
return false;
}
function readUpstreamMemo() {
const raw = process.env.OVERPASS_UPSTREAMS || '';
if (upstreamMemo.raw !== raw) {
const endpoints = parseOverpassUpstreams(raw);
upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) };
}
return upstreamMemo;
}
/**
* TTL for FRESH cached Overpass responses (ms). Road geometry is static for
* months — the original 45 s TTL forced a public-mirror round-trip on nearly
@@ -153,6 +206,8 @@ export {
OVERPASS_MAX_RESPONSE_BYTES,
parseOverpassUpstreams,
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
cachedByConfiguredUpstream,
OVERPASS_USER_AGENT,
OVERPASS_TIMEOUT_MS,
};
server/providers/overpass/query.js
+31 -5
View File
@@ -18,6 +18,29 @@ function isOverpassBoundaryQuery(cacheKey) {
return /is_in\s*\(|\bpivot\b/i.test(String(cacheKey || ''));
}
// Input has already had quoted literals and comments removed. Consume each
// character once: a regex that searches for a closing bracket from every `[` is
// quadratic on an unterminated filter. Nested or unmatched brackets are invalid.
function stripTagFilters(text) {
const parts = [];
let start = 0;
let inFilter = false;
for (let i = 0; i < text.length; i += 1) {
if (text[i] === '[') {
if (inFilter) return null;
parts.push(text.slice(start, i), ' ');
inFilter = true;
} else if (text[i] === ']') {
if (!inFilter) return null;
inFilter = false;
start = i + 1;
}
}
if (inFilter) return null;
parts.push(text.slice(start));
return parts.join('');
}
/**
* Validate + clamp an Overpass form body. Defends the generic proxy against
* planet-scale abuse: requires exactly one `data` query in which EVERY element
@@ -50,6 +73,10 @@ function sanitizeOverpassBody(rawBody) {
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
const stripped = stripOverpassNoise(data);
const withoutTags = stripTagFilters(stripped);
if (withoutTags === null) {
return { ok: false, error: 'Malformed Overpass tag filter' };
}
// Reject oversized radii in EVERY around form — point `around:r,lat,lon` AND the
// input-set form `around.set:r` — and parse the full numeric token so scientific
@@ -93,20 +120,19 @@ function sanitizeOverpassBody(rawBody) {
// statement. So `way[...]->.a` (global assigned to a set) is rejected, while the
// app's `is_in(...)->.a; area.a[...]` and `area(id)->.x; rel(pivot.x)` validate.
const boundedSets = new Set();
for (let stmt of stripped.split(';')) {
for (let stmt of withoutTags.split(';')) {
stmt = stmt.trim();
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
// Strip output-set assignments (NOT input bounds), then strip bracket tag
// filters so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) can never be
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
// misread as a spatial bound. Bounds live in (...) / function calls / set
// refs, never inside [...], so the probe loses nothing real.
const outSets = [];
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
outSets.push(name);
return ' ';
});
const probe = body.replace(/\[[^\]]*\]/g, ' ');
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
// between the selector and the area filter (way["highway"](area.a)) can't hide
server/providers/overpass/requestAdmission.test.mjs
+178
View File
@@ -0,0 +1,178 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { Readable } from 'node:stream';
import { spawnSync } from 'node:child_process';
import { overpassProxy } from '../overpass.js';
import { sanitizeOverpassBody } from './query.js';
const form = (query) => `data=${encodeURIComponent(query)}`;
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
for (const query of [
'node[',
'node]',
'node[[name]](around:20,1,1);out;',
'node[name=around:20];out;',
'node[is_in];out;',
'node[name="(around:20,1,1)"];out;',
'node(around:20,1,1);way[name];out;',
'way[highway](area.a);out;',
'node[->.a](around:20,1,1);way.a;out;',
]) {
assert.equal(sanitizeOverpassBody(form(query)).ok, false, query);
}
});
test('bounded queries preserve quoted brackets, comments, multiple tags and timeout clamping', () => {
for (const query of [
'[out:json][timeout:999];node[name="[Town]; ]"](around:20,1,1);out;',
'/* [ */way[highway][name](30.1,-97.9,30.5,-97.5);out geom;',
'is_in(30.2,-97.7)->.a;area.a[boundary="administrative"];out tags;',
'area(3600039152)->.x;rel(pivot.x);out geom;',
]) {
const result = sanitizeOverpassBody(form(query));
assert.equal(result.ok, true, query);
assert.equal(
new URLSearchParams(result.body).get('data'),
query.replace('[timeout:999]', '[timeout:30]'),
);
}
});
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
// A child deadline can stop synchronous regex backtracking; a test timeout in
// the same event loop cannot. The old parser takes several seconds for this.
const result = spawnSync(
process.execPath,
[
'--input-type=module',
'-e',
`import { sanitizeOverpassBody } from ${JSON.stringify(new URL('./query.js', import.meta.url).href)};
for (let i = 0; i < 20; i++) {
if (sanitizeOverpassBody('data=node' + '['.repeat(24000)).ok) process.exit(1);
}`,
],
{ timeout: 2000, encoding: 'utf8' },
);
assert.ifError(result.error);
assert.equal(result.status, 0, result.stderr);
});
function handler(hook = 'configureServer') {
let route;
overpassProxy()[hook]({
middlewares: {
use(path, fn) {
if (path === '/api/overpass') route = fn;
},
},
});
return route;
}
async function call(
route,
{
headers = {},
peer = '127.0.0.1',
body = 'data=node[',
forbidRead = false,
method = 'POST',
url = '/',
} = {},
) {
let reads = 0;
const req = new Readable({
read() {
reads++;
assert.equal(forbidRead, false, 'refused request must not read its body');
this.push(Buffer.from(body));
this.push(null);
},
});
Object.assign(req, {
method,
url,
headers: { host: 'localhost:4173', ...headers },
socket: { remoteAddress: peer },
});
const res = {
statusCode: 200,
headers: {},
setHeader(key, value) {
this.headers[key.toLowerCase()] = value;
},
writeHead(status, values) {
this.statusCode = status;
for (const [key, value] of Object.entries(values))
this.setHeader(key, value);
},
end(value) {
this.body = value;
},
};
await route(req, res);
return { ...res, reads };
}
for (const hook of ['configureServer', 'configurePreviewServer']) {
test(`${hook}: cross-site requests are rejected before consuming a body or fetching`, async (t) => {
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler(hook);
for (const headers of [
{ origin: 'https://foreign.example', 'content-type': 'text/plain' },
{ 'sec-fetch-site': 'cross-site' },
{ 'sec-fetch-site': 'same-site' },
{ origin: 'null' },
{ 'x-forwarded-for': '127.0.0.1' },
]) {
const response = await call(route, { headers, forbidRead: true });
assert.equal(response.statusCode, 403);
assert.equal(response.reads, 0);
assert.equal(response.headers['cache-control'], 'no-store');
}
});
}
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
const route = handler();
for (const headers of [
{},
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
]) {
const response = await call(route, { headers });
assert.equal(response.statusCode, 400);
assert.equal(response.reads, 1);
assert.match(response.body, /Malformed Overpass tag filter/);
}
const response = await call(route, {
method: 'GET',
url: '/status',
forbidRead: true,
});
assert.equal(response.statusCode, 200);
assert.equal(typeof JSON.parse(response.body).configured, 'boolean');
});
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
let now = Date.now() + 120000;
t.mock.method(Date, 'now', () => now);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler();
const peer = 'admission-limit-test';
for (let i = 0; i < 300; i++) {
assert.equal((await call(route, { peer })).statusCode, 400);
}
const denied = await call(route, { peer, forbidRead: true });
assert.equal(denied.statusCode, 429);
assert.equal(denied.reads, 0);
assert.equal(denied.headers['retry-after'], '60');
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
now += 60001;
assert.equal((await call(route, { peer })).statusCode, 400);
});
server/providers/overpass/transport.js
+22 -4
View File
@@ -1,6 +1,7 @@
import {
OVERPASS_MAX_RESPONSE_BYTES,
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
OVERPASS_USER_AGENT,
OVERPASS_TIMEOUT_MS,
} from './constants.js';
@@ -13,7 +14,7 @@ import { simplifyOverpassPayloadBody } from './geometry.js';
* Checks for known rate-limit phrases in the body text regardless of
* HTTP status code, since some mirrors return 200 with an error payload.
*
* @param {string} bodyText - Upstream response body.
* @param {string} bodyText - The remark of a data reply, or a non-data body.
* @returns {boolean} True if the body looks rate-limited.
*/
function overpassLooksRateLimited(bodyText) {
@@ -40,6 +41,14 @@ function overpassLooksRuntimeError(bodyText) {
);
}
function parseJson(text) {
try {
return JSON.parse(text);
} catch {
return undefined;
}
}
/**
* True only for an upstream response that is actually Overpass data.
*
@@ -150,15 +159,22 @@ async function fetchOverpassPayload(
signal: controller.signal,
});
const responseBody = await readBody(upstream, maxResponseBytes);
const parsed = parseJson(responseBody);
// An Overpass data reply reports failure in its remark; its element
// tags are data and can contain any phrase. Any other body (an error
// page, or JSON without elements) is matched whole.
const failureText = Array.isArray(parsed?.elements)
? String(parsed.remark || '')
: responseBody;
const rateLimited =
upstream.status === 429 ||
upstream.status === 406 ||
overpassLooksRateLimited(responseBody);
overpassLooksRateLimited(failureText);
if (
rateLimited ||
upstream.status < 200 ||
upstream.status >= 300 ||
overpassLooksRuntimeError(responseBody)
overpassLooksRuntimeError(failureText)
) {
const failures = (previous?.failures || 0) + 1;
const delay = retryDelay(
@@ -177,7 +193,6 @@ async function fetchOverpassPayload(
failure = refusal(status, delay);
continue;
}
const parsed = JSON.parse(responseBody);
if (!Array.isArray(parsed?.elements) || parsed.remark)
throw new Error('Malformed Overpass response');
cooldowns.delete(endpoint);
@@ -187,6 +202,9 @@ async function fetchOverpassPayload(
contentType: 'application/json',
// Never retain a secret-bearing endpoint in cache or response metadata.
endpoint: 'configured',
// Which configured instance produced this, as a hash rather than a URL,
// so a cached answer is not reused after the operator repoints.
upstreams: resolveOverpassUpstreamTag(),
rateLimited: false,
};
} catch {
server/providers/places.js
+6 -1
View File
@@ -1,8 +1,13 @@
/** Place-search and route middleware for Node development servers. */
export { googlePlacesContextProxy } from './places/google.js';
export {
googlePlacesContextProxy,
validateGeocodeAddress,
geocodeBounds,
} from './places/google.js';
export {
googleServerApiKey,
keylessGooglePlacesResponse,
keylessGoogleGeocodeResponse,
} from './places/google-key.js';
export { installRouteMiddleware } from './places/routes.js';
export {
server/providers/places/google-key.js
+28 -6
View File
@@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) {
}
/**
* Google API key for the SERVER-SIDE calls (Places nearby/text search, the
* CCTV Street View fallback). These never reach the browser, so this key can
* be restricted by server IP and scoped to Places API + Street View Static
* API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles +
* Geocoding for the browser (#33). Splitting them is opt-in: unset, this
* falls back to the shared browser key and nothing changes.
* Geocoding speaks Google's own response shape rather than the `places: []`
* contract its two siblings use, because the browser adapters read `status`
* and `results` directly. The keyless capability reads the same way: a status
* that is not `OK` and no results, which the adapters already treat as "this
* provider did not answer" and pass to the next geocoder in the chain.
*/
export function keylessGoogleGeocodeResponse(apiKey) {
if (String(apiKey ?? '').trim()) return null;
return {
statusCode: 200,
payload: {
configured: false,
status: 'REQUEST_DENIED',
results: [],
error: null,
},
};
}
/**
* Google API key for the SERVER-SIDE calls (Places nearby/text search,
* geocoding, the CCTV Street View fallback). These never reach the browser, so
* this key can be restricted by server IP and scoped to Places API + Geocoding
* API + Street View Static API — while GOOGLE_MAPS_API_KEY stays
* referrer-restricted to Map Tiles + Maps JavaScript API for the browser
* (#33, #363). Splitting them is opt-in: unset, this falls back to the shared
* browser key, which then has to carry every API both sides use and cannot
* take a referrer restriction.
*/
export function googleServerApiKey() {
return resolveGoogleServerKey(process.env);
server/providers/places/google.js
+191 -1
View File
@@ -1,9 +1,11 @@
import {
googleServerApiKey,
keylessGooglePlacesResponse,
keylessGoogleGeocodeResponse,
} from './google-key.js';
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
import { admitSameSite } from '../common/same-site.js';
import { readResponseJsonCapped } from '../common/http.js';
import {
projectNearbyPlaces,
projectTextSearchPlaces,
@@ -25,7 +27,7 @@ let _googleRateLimiter;
*/
export const GOOGLE_DEFAULT_PER_MIN = 120;
/** Google cost endpoints (nearby-places + text-search). Null only when set to 0. */
/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */
function googleRateLimiter() {
if (_googleRateLimiter === undefined)
_googleRateLimiter = makeCostRateLimiter(
@@ -56,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) {
return { ok: true, latitude, longitude };
}
/** Longest accepted geocoder query. Real place names are far shorter. */
const GEOCODE_MAX_QUERY = 200;
/**
* Upstream deadline and body ceiling for the geocode routes. The deadline stays
* armed through body consumption, so a stalled upstream cannot hold a handler
* open, and the cap bounds a chunked or length-omitted reply. A geocode answer
* is a handful of results; 256 KiB is far above any honest one.
*/
const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000;
const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024;
/** Validate a free-text geocoder query before consuming request quota. */
export function validateGeocodeAddress(searchParams) {
const address = String(searchParams.get('address') || '').trim();
if (!address) return { ok: false, error: 'address is required' };
if (address.length > GEOCODE_MAX_QUERY)
return { ok: false, error: 'address is too long' };
return { ok: true, address };
}
/**
* Validate the optional viewport bias, which Google reads as
* `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than
* refused: the bias only ranks results, so a malformed one costs relevance,
* not an answer.
*/
export function geocodeBounds(searchParams) {
const raw = String(searchParams.get('bounds') || '').trim();
if (!raw) return null;
const corners = raw.split('|');
if (corners.length !== 2) return null;
const parts = corners.flatMap((corner) => corner.split(','));
// A blank component must be refused before Number() turns it into 0 and a
// half-written corner like `30.4,` reads as the equator.
if (parts.length !== 4 || parts.some((part) => !part.trim())) return null;
const numbers = parts.map(Number);
if (!numbers.every(Number.isFinite)) return null;
const [swLat, swLon, neLat, neLon] = numbers;
if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null;
if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null;
return raw;
}
/** Nearby place labels and view-biased text search, with request-time key resolution. */
export function googlePlacesContextProxy({
resolveApiKey = googleServerApiKey,
@@ -293,6 +340,149 @@ export function googlePlacesContextProxy({
);
}
});
/**
* Geocoding belongs on the server, not in the page. Google's Geocoding web
* service refuses referrer-restricted keys, so calling it from the browser
* forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be
* left unrestricted (#363). Here the request carries the server key, and
* the browser key no longer needs the Geocoding API at all.
*
* Unlike the two Places routes above, these answer in Google's own shape
* (`status` + `results`) rather than the `places: []` contract: the browser
* normalizers read that shape directly, and keeping it means the proxy adds
* a hop without adding a translation nobody asked for.
*/
function installGeocodeRoute(path, readQuery) {
middlewares.use(path, async (req, res) => {
const refuse = (statusCode, error) => {
res.statusCode = statusCode;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }),
);
};
// Gate first, like the Places routes above: a cross-site caller learns
// nothing about this endpoint's method surface.
if (admitSameSite(req, res)) return;
if (req.method !== 'GET') {
refuse(405, 'Method not allowed');
return;
}
// A keyless geocode has no provider cost, so it resolves before the
// paid-endpoint limiter can consume or exhaust quota (mirrors the
// Places routes above).
const apiKey = resolveApiKey();
const keyless = keylessGoogleGeocodeResponse(apiKey);
if (keyless) {
res.statusCode = keyless.statusCode;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify(keyless.payload));
return;
}
const requestUrl = new URL(req.url || '', 'http://localhost');
const query = readQuery(requestUrl.searchParams);
if (!query.ok) {
refuse(400, query.error);
return;
}
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the
// Places routes because both spend the same Google budget.
const _grl = googleRateLimiter();
if (_grl && !_grl(clientKey(req))) {
res.statusCode = 429;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Retry-After', '5');
res.end(
JSON.stringify({
status: 'OVER_QUERY_LIMIT',
results: [],
error: 'Rate limit exceeded',
}),
);
return;
}
try {
const upstream = new URL(
endpoints.geocode ||
'https://maps.googleapis.com/maps/api/geocode/json',
);
for (const [name, value] of Object.entries(query.params))
upstream.searchParams.set(name, value);
upstream.searchParams.set('key', apiKey);
const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS);
const response = await fetchImpl(upstream.toString(), {
redirect: 'error',
signal: deadline,
});
// An unreadable, oversized or timed-out body is not an answer: it
// falls through to the same shaped failure as a refusal below.
const data = await readResponseJsonCapped(
response,
GEOCODE_MAX_RESPONSE_BYTES,
deadline,
).catch(() => null);
// A body we could not read is an upstream failure, not an answer with
// a status of its own, so it does not borrow the upstream's code.
if (!data) {
res.statusCode = 502;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.end(
JSON.stringify({
status: 'UNKNOWN_ERROR',
results: [],
error: 'Google Geocoding request failed',
}),
);
return;
}
res.statusCode = response.ok ? 200 : response.status;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'private, max-age=300');
res.end(JSON.stringify(data));
} catch {
// Fixed public text: a transport, URL or runtime message would
// describe the server to the browser (SECURITY.md, sanitized errors).
res.statusCode = 502;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.end(
JSON.stringify({
status: 'UNKNOWN_ERROR',
results: [],
error: 'Google Geocoding request failed',
}),
);
}
});
}
installGeocodeRoute('/api/google/geocode', (searchParams) => {
const address = validateGeocodeAddress(searchParams);
if (!address.ok) return address;
const bounds = geocodeBounds(searchParams);
return {
ok: true,
params: {
address: address.address,
...(bounds ? { bounds } : {}),
},
};
});
installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => {
const coordinates = validatePlacesCoordinates(searchParams);
if (!coordinates.ok) return coordinates;
const { latitude, longitude } = coordinates;
return { ok: true, params: { latlng: `${latitude},${longitude}` } };
});
}
return {
server/providers/radio/catalog.js
+30 -24
View File
@@ -5,7 +5,7 @@ import { readResponseTextCapped } from '../common/http.js';
import {
normalizeRadioBrowserStation,
publicRadioStation,
cleanRadioText,
normalizeRadioTag,
} from './stations.js';
import {
radioMirrorOrigin,
@@ -25,8 +25,9 @@ import {
RADIO_USER_AGENT,
RADIO_UUID_RE,
RADIO_FALLBACK_MIRRORS,
RADIO_REJECTED_STATION_COUNT_MAX,
} from './constants.js';
export async function mapRadioConcurrent(values, concurrency, mapper) {
async function mapRadioConcurrent(values, concurrency, mapper) {
const results = new Array(values.length);
let cursor = 0;
const workers = Array.from(
@@ -177,29 +178,25 @@ export function createRadioProxyMiddleware({
const rows = await fetchPath(`/json/stations/search?${params}`);
if (!Array.isArray(rows))
throw new Error('Radio Browser catalog payload was not an array');
if (
!rows.every(
(row) =>
row &&
typeof row === 'object' &&
!Array.isArray(row) &&
typeof row.stationuuid === 'string' &&
typeof row.name === 'string' &&
(typeof row.url_resolved === 'string' ||
typeof row.url === 'string'),
)
)
throw new Error(
'Radio Browser catalog contained a malformed station row',
);
const stations = rows
const structuralRows = rows.filter(
(row) =>
row &&
typeof row === 'object' &&
!Array.isArray(row) &&
typeof row.stationuuid === 'string' &&
typeof row.name === 'string' &&
(typeof row.url_resolved === 'string' ||
typeof row.url === 'string'),
);
// Malformed rows are skipped individually, but counted so the
// catalog keeps a diagnostic for upstream schema drift. Policy
// rejections in normalization (offline, non-HTTPS, codec) are normal
// filtering and are not counted.
const rejectedRows = rows.length - structuralRows.length;
const stations = structuralRows
.map(normalizeRadioBrowserStation)
.filter(Boolean);
const requestedTag = cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim();
const requestedTag = normalizeRadioTag(tag);
const requestedTagCovered =
!requestedTag ||
stations.some((station) =>
@@ -215,9 +212,10 @@ export function createRadioProxyMiddleware({
// also contain an accepted station tagged for the requested category.
succeeded: stations.length > 0 && requestedTagCovered,
stations,
rejectedRows,
};
} catch {
return { succeeded: false, stations: [] };
return { succeeded: false, stations: [], rejectedRows: 0 };
}
},
);
@@ -257,6 +255,10 @@ export function createRadioProxyMiddleware({
if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS)
healthReasons.push('station-coverage-below-policy');
const degraded = healthReasons.length > 0;
const rejectedStationCount = Math.min(
RADIO_REJECTED_STATION_COUNT_MAX,
outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0),
);
const coverage = {
successfulQueries,
totalQueries: queries.length,
@@ -271,6 +273,9 @@ export function createRadioProxyMiddleware({
degraded,
degradedReason: degraded ? healthReasons.join(',') : null,
coverage,
// Diagnostic only: it never fails health policy, so a usable generation
// with a few skipped rows is still admitted. The client reports it.
rejectedStationCount,
};
if (degraded && catalogCache) {
const error = new Error(
@@ -360,6 +365,7 @@ export function createRadioProxyMiddleware({
coverage: catalog.coverage || null,
acceptedGeneration: catalog.acceptedGeneration ?? null,
catalogInstance,
rejectedStationCount: catalog.rejectedStationCount ?? 0,
});
} catch (error) {
sendJson(res, 503, {
server/providers/radio/constants.js
+3
View File
@@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
export const RADIO_USER_AGENT =
'GodsEyeView/1.0 (Radio Browser directory client)';
export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js';
// Upper bound for the structurally malformed upstream rows reported in the
// /stations envelope. The client layer validates against the same bound.
export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000;
export const RADIO_FALLBACK_MIRRORS = Object.freeze([
'https://de1.api.radio-browser.info',
'https://de2.api.radio-browser.info',
server/providers/radio/stations.js
+1
View File
@@ -1,5 +1,6 @@
export {
cleanRadioText,
normalizeRadioTag,
isNonGlobalIpv4,
publicRadioHttpsUrl,
normalizeRadioBrowserStation,
server/providers/radio/transport.js
+29 -12
View File
@@ -1,6 +1,9 @@
import https from 'node:https';
import { Readable } from 'node:stream';
import { isNonGlobalIpv4 } from './stations.js';
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
export function radioMirrorOrigin(value) {
const hostname = String(value ?? '')
.toLowerCase()
@@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) {
},
},
(response) => {
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value))
value.forEach((item) => headers.append(name, item));
else if (value !== undefined) headers.set(name, String(value));
// Nothing thrown here may escape: this callback runs outside the
// promise, so an exception would be uncaught and crash the server.
try {
const status = response.statusCode || 500;
// A `Response` only represents statuses up to 599; Node never hands
// this callback a 1xx.
if (status > 599)
throw new Error(
`Radio Browser returned unsupported status ${status}`,
);
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value))
value.forEach((item) => headers.append(name, item));
else if (value !== undefined) headers.set(name, String(value));
}
const nullBody = NULL_BODY_STATUSES.has(status);
const upstream = new Response(
nullBody ? null : Readable.toWeb(response),
{ status, statusText: response.statusMessage || '', headers },
);
if (nullBody) response.resume();
resolve(upstream);
} catch (error) {
response.destroy();
reject(error);
}
resolve(
new Response(Readable.toWeb(response), {
status: response.statusCode || 500,
statusText: response.statusMessage || '',
headers,
}),
);
},
);
request.on('error', reject);
server/providers/regional/nominatimCache.js
+23 -12
View File
@@ -15,7 +15,7 @@ export const NOMINATIM_CACHE_TTL_MS = Object.freeze({
});
/**
* Comparable form of a place name: Unicode-normalized (NFKC), accents
* Comparable form of a place name: Unicode-normalized (NFKC), Latin accents
* removed, case folded, punctuation and symbols collapsed to single spaces,
* and a leading article dropped. Non-Latin scripts are kept as they are:
* "東京" stays "東京", while "São Paulo", "sao paulo" and "SÃO PAULO!" meet.
@@ -26,7 +26,7 @@ export const NOMINATIM_CACHE_TTL_MS = Object.freeze({
export function normalizePlaceName(value) {
return String(value ?? '')
.normalize('NFKD')
.replace(/\p{M}+/gu, '')
.replace(/(\p{Script=Latin})\p{M}+/gu, '$1')
.normalize('NFKC')
.toLocaleLowerCase('en-US')
.replace(/[\p{P}\p{S}\p{Z}\s]+/gu, ' ')
@@ -55,11 +55,11 @@ export function createNominatimCache({
let writesSinceSweep = 0;
let sweeping = null;
const digestFor = (key) =>
crypto.createHash('sha256').update(key).digest('hex');
const fileFor = (key) =>
path.join(
dir,
`${crypto.createHash('sha256').update(key).digest('hex').slice(0, 40)}.json`,
);
path.join(dir, `${digestFor(key).slice(0, 40)}.json`);
function remember(key, entry) {
memory.delete(key);
@@ -80,7 +80,8 @@ export function createNominatimCache({
if (!dir) return null;
try {
const entry = JSON.parse(await fs.readFile(fileFor(key), 'utf8'));
if (entry?.key !== key || !(entry.expiresAt > now())) return null;
if (entry?.keyDigest !== digestFor(key) || !(entry.expiresAt > now()))
return null;
remember(key, entry);
return entry;
} catch {
@@ -89,19 +90,29 @@ export function createNominatimCache({
}
async function set(key, payload, ttlMs) {
const entry = { key, payload, storedAt: now(), expiresAt: now() + ttlMs };
const entry = {
keyDigest: digestFor(key),
payload,
storedAt: now(),
expiresAt: now() + ttlMs,
};
remember(key, entry);
if (!dir) return;
const body = JSON.stringify(entry);
if (Buffer.byteLength(body) > maxEntryBytes) return;
let temp = null;
try {
await fs.mkdir(dir, { recursive: true });
await fs.mkdir(dir, { recursive: true, mode: 0o700 });
await fs.chmod(dir, 0o700);
const file = fileFor(key);
const temp = `${file}.${process.pid}.tmp`;
await fs.writeFile(temp, body);
temp = `${file}.${process.pid}.${crypto.randomBytes(6).toString('hex')}.tmp`;
await fs.writeFile(temp, body, { mode: 0o600, flag: 'wx' });
await fs.rename(temp, file);
temp = null;
} catch {
return;
} finally {
if (temp) await fs.rm(temp, { force: true }).catch(() => {});
}
writesSinceSweep += 1;
if (writesSinceSweep >= sweepEvery) {
@@ -167,5 +178,5 @@ export function createNominatimCache({
return sweeping;
}
return { get, set, sweep, memorySize: () => memory.size };
return { get, set, sweep };
}
server/providers/regional/nominatimGate.js
+10 -10
View File
@@ -23,23 +23,23 @@ const PUBLIC_NOMINATIM_HOST = 'nominatim.openstreetmap.org';
* The public usage policy asks for a User-Agent or Referer that identifies the
* application. The identity is stable: it is never rotated or varied.
*/
export const NOMINATIM_HEADERS = Object.freeze({
const NOMINATIM_HEADERS = Object.freeze({
'User-Agent':
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)',
Referer: 'https://github.com/bilawalsidhu/gods-eye-view',
});
/** Absolute policy maximum is one request per second; 1.1 s absorbs jitter. */
export const NOMINATIM_MIN_SPACING_MS = 1100;
const NOMINATIM_MIN_SPACING_MS = 1100;
/** Requests allowed to wait for their turn; past this, refuse at once. */
export const NOMINATIM_MAX_PENDING = 4;
/** A queued request older than this has no reader left; drop it unsent. */
export const NOMINATIM_MAX_WAIT_MS = 10_000;
const NOMINATIM_MAX_WAIT_MS = 10_000;
/** Default public requests per install per UTC day, outlines and search together. */
export const NOMINATIM_DEFAULT_DAILY_CAP = 50;
const NOMINATIM_DEFAULT_DAILY_CAP = 50;
/**
* Pause after a refusal without Retry-After. A Retry-After from the server is
@@ -88,12 +88,15 @@ export function resolveNominatimSettings(env = process.env) {
url.hash
)
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
const publicHost =
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST;
if (publicHost && url.protocol !== 'https:')
return { endpoint: null, isPublic: false, dailyCap, invalid: true };
if (url.pathname === '/' || url.pathname === '') url.pathname = '/search';
return {
endpoint: url.href.replace(/\/$/, ''),
// A trailing DNS dot names the same host.
isPublic:
url.hostname.toLowerCase().replace(/\.$/, '') === PUBLIC_NOMINATIM_HOST,
isPublic: publicHost,
dailyCap,
};
}
@@ -546,9 +549,6 @@ export function createGateStateStore({
};
}
/** Compatibility name for the state store. */
export const createUsageStore = createGateStateStore;
/**
* Construct a gate. `settings` may be a function so the environment is read
* after the dev server loads `.env`.
@@ -558,7 +558,7 @@ export const createUsageStore = createGateStateStore;
* @param {typeof fetch} [options.fetchImpl]
* @param {() => number} [options.now]
* @param {(ms: number) => Promise<void>} [options.sleep]
* @param {ReturnType<typeof createUsageStore>} [options.usage]
* @param {ReturnType<typeof createGateStateStore>} [options.usage]
*/
export function createNominatimGate({
settings = () => resolveNominatimSettings(),
server/providers/regional/outlineSelect.js
+129 -126
View File
@@ -1,4 +1,5 @@
import { normalizePlaceName } from './nominatimCache.js';
import { pointInRing } from '../../../src/data/adminBoundaries.js';
/**
* Which Nominatim results may answer which voice ask, and how their polygons
@@ -30,7 +31,7 @@ export const OUTLINE_POLYGON_THRESHOLD = Object.freeze({
});
/** Total vertices returned for one outline, and parts/holes kept. */
export const OUTLINE_MAX_VERTICES = 6000;
const OUTLINE_MAX_VERTICES = 6000;
const OUTLINE_MAX_PARTS = 24;
const OUTLINE_MAX_HOLES = 16;
@@ -106,7 +107,7 @@ const MAN_MADE_STRUCTURES = [
* class. Anything absent is rejected: roads, railways, stations, shops,
* guideposts and single facilities never answer an area ask.
*/
export const OUTLINE_ACCEPT = Object.freeze({
const OUTLINE_ACCEPT = Object.freeze({
city: {
boundary: ['administrative'],
place: SETTLEMENT_PLACES,
@@ -165,9 +166,9 @@ export const OUTLINE_ACCEPT = Object.freeze({
});
/**
* Administrative evidence per ask kind: an address type, or a Nominatim
* place_rank band (country 4, state 8, county 10-12, city 12-16, suburb 18-20,
* neighbourhood 20-22). An administrative boundary with neither is refused.
* Administrative evidence per ask kind: a recognized address type is required.
* A supplied numeric place_rank must also fit the ask's broad band (country 4,
* state 8, county 10-12, city 12-16, suburb 18-20, neighbourhood 20-22).
*/
const ADMIN_EVIDENCE = Object.freeze({
city: {
@@ -192,99 +193,31 @@ const ADMIN_EVIDENCE = Object.freeze({
},
});
const KNOWN_ADMIN_TYPES = new Set(
Object.values(ADMIN_EVIDENCE).flatMap((evidence) => evidence.types),
);
/**
* Nominatim address types whose meaning is stable enough for the area consumer's
* country/admin1/admin2 contract. `region`, `district` and numeric admin levels
* vary by country, so they remain unverified instead of being guessed.
*/
const VERIFIED_ADMIN_LEVEL = Object.freeze({
country: 'country',
state: 'admin1',
province: 'admin1',
county: 'admin2',
state_district: 'admin2',
});
const EXPLICIT_NAME_LEVEL = Object.freeze({
country: 'country',
nation: 'country',
province: 'admin1',
state: 'admin1',
region: 'admin1',
oblast: 'admin1',
prefecture: 'admin1',
canton: 'admin1',
governorate: 'admin1',
county: 'admin2',
district: 'admin2',
department: 'admin2',
zone: 'retired',
});
function explicitNameEvidence(value) {
const normalized = String(value || '')
.normalize('NFKC')
.toLocaleLowerCase()
.replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ')
.trim();
const prefix = normalized.match(
/^(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)\s+of\b/,
)?.[1];
const suffix = normalized.match(
/\b(country|nation|province|state|region|zone|oblast|prefecture|canton|governorate|county|district|department)$/,
)?.[1];
const levels = new Set(
[prefix, suffix]
.filter(Boolean)
.map((type) => EXPLICIT_NAME_LEVEL[type])
.filter(Boolean),
);
if (levels.size > 1) return { contradictory: true, level: null };
return { contradictory: false, level: levels.values().next().value || null };
}
/**
* Verified administrative identity carried by the selected result itself.
* Never infer it from the request, desired kind, rank, or a separate geocoder
* anchor. A retired name is preserved without a level so downstream policy can
* reject it explicitly; every other contradiction stays wholly unverified.
*/
function selectedAdminIdentity(kind, row) {
if (kind !== 'admin') return null;
const adminLevel =
VERIFIED_ADMIN_LEVEL[String(row?.addresstype || '').toLowerCase()];
const authoritativeAdminArea = String(row?.name || '').trim();
if (!adminLevel || !authoritativeAdminArea) return null;
// Validate the complete authoritative name. `adminArea` is truncated only
// for display after every prefix/suffix signal has been considered.
const { contradictory, level: nameLevel } = explicitNameEvidence(
authoritativeAdminArea,
);
if (contradictory) return null;
const adminArea = authoritativeAdminArea.slice(0, 160);
if (nameLevel === 'retired') return { adminArea };
if (nameLevel && nameLevel !== adminLevel) return null;
return { adminArea, adminLevel };
}
/** Whether an administrative boundary is at the level the ask names. */
export function adminLevelFits(kind, row) {
/** Whether supplied administrative evidence contradicts the accepted identity. */
function adminLevelFits(kind, row) {
const evidence = ADMIN_EVIDENCE[kind];
if (!evidence) return true;
const type = String(row?.addresstype || '').toLowerCase();
if (type && evidence.types.includes(type)) return true;
// A recognized type is direct evidence. Rank is only a fallback when the
// upstream answer omits the type or supplies a value we do not understand;
// the overlapping county/city rank bands must not override `county`.
if (KNOWN_ADMIN_TYPES.has(type)) return false;
const rank = Number(row?.place_rank);
if (Number.isFinite(rank))
return rank >= evidence.ranks[0] && rank <= evidence.ranks[1];
return false;
const place = String(row?.category || row?.class).toLowerCase() === 'place';
// A place row already has an accepted type. Missing address evidence must
// not erase it, but explicit contradictory evidence must not bypass checks.
if (!place || Object.hasOwn(row, 'addresstype')) {
if (typeof row?.addresstype !== 'string') return false;
const type = row.addresstype.toLowerCase();
if (
!evidence.types.includes(type) &&
!(place && type === String(row.type).toLowerCase())
)
return false;
}
// Rank can corroborate or contradict an explicit identity, never invent it.
// JSONv2 supplies a number; coercing arrays or strings hides malformed data.
if (!Object.hasOwn(row, 'place_rank')) return true;
const rank = row.place_rank;
return (
Number.isInteger(rank) &&
// Boundary rank bands do not describe every accepted place type.
(place || (rank >= evidence.ranks[0] && rank <= evidence.ranks[1]))
);
}
/** Largest footprint (m²) a non-`building` result may have for a building ask. */
@@ -332,7 +265,7 @@ function queryWords(query) {
* appear among one name's words, or one name must contain the whole query
* (for scripts written without spaces).
*/
export function outlineNameMatches(row, query) {
function outlineNameMatches(row, query) {
const words = queryWords(query);
if (!words.length) return false;
const whole = words.join(' ');
@@ -371,8 +304,7 @@ function cleanRing(ring) {
const out = [];
for (const point of ring) {
if (!Array.isArray(point)) return null;
const lon = Number(point[0]);
const lat = Number(point[1]);
const [lon, lat] = point;
if (!finiteLon(lon) || !finiteLat(lat)) return null;
out.push([lon, lat]);
}
@@ -391,30 +323,31 @@ function ringArea(ring) {
}
/** Polygon/MultiPolygon GeoJSON → `[[outer, ...holes], …]`, largest first. */
export function polygonsFromGeoJson(geometry) {
function polygonsFromGeoJson(geometry) {
const raw =
geometry?.type === 'Polygon'
? [geometry.coordinates]
: geometry?.type === 'MultiPolygon'
? geometry.coordinates
: null;
if (!Array.isArray(raw)) return null;
if (!Array.isArray(raw) || !raw.length || raw.length > OUTLINE_MAX_PARTS)
return null;
const polygons = [];
for (const polygon of raw) {
if (!Array.isArray(polygon)) continue;
if (
!Array.isArray(polygon) ||
!polygon.length ||
polygon.length - 1 > OUTLINE_MAX_HOLES
)
return null;
const outer = cleanRing(polygon[0]);
if (!outer) continue;
const holes = polygon
.slice(1)
.map(cleanRing)
.filter(Boolean)
.sort((a, b) => ringArea(b) - ringArea(a))
.slice(0, OUTLINE_MAX_HOLES);
if (!outer) return null;
const holes = polygon.slice(1).map(cleanRing);
if (holes.some((hole) => !hole)) return null;
polygons.push({ rings: [outer, ...holes], area: ringArea(outer) });
}
if (!polygons.length) return null;
polygons.sort((a, b) => b.area - a.area);
return polygons.slice(0, OUTLINE_MAX_PARTS).map((entry) => entry.rings);
return polygons.map((entry) => entry.rings);
}
/** Iterative Douglas–Peucker on a closed ring (keeps it closed, ≥ 4 points). */
@@ -460,6 +393,79 @@ const countVertices = (polygons) =>
0,
);
const edgeSide = (p, q, r) =>
(q[0] - p[0]) * (r[1] - p[1]) - (q[1] - p[1]) * (r[0] - p[0]);
const onEdge = (p, q, r) =>
r[0] >= Math.min(p[0], q[0]) &&
r[0] <= Math.max(p[0], q[0]) &&
r[1] >= Math.min(p[1], q[1]) &&
r[1] <= Math.max(p[1], q[1]);
/** Intersection, including touching, of two planar closed-ring edges. */
function edgesIntersect(a, b, c, d) {
if (
Math.max(a[0], b[0]) < Math.min(c[0], d[0]) ||
Math.max(c[0], d[0]) < Math.min(a[0], b[0]) ||
Math.max(a[1], b[1]) < Math.min(c[1], d[1]) ||
Math.max(c[1], d[1]) < Math.min(a[1], b[1])
)
return false;
const ac = edgeSide(a, b, c);
const ad = edgeSide(a, b, d);
const ca = edgeSide(c, d, a);
const cb = edgeSide(c, d, b);
return (
(ac === 0 && onEdge(a, b, c)) ||
(ad === 0 && onEdge(a, b, d)) ||
(ca === 0 && onEdge(c, d, a)) ||
(cb === 0 && onEdge(c, d, b)) ||
(ac > 0 !== ad > 0 && ca > 0 !== cb > 0)
);
}
function ringsIntersect(a, b) {
for (let i = 1; i < a.length; i++)
for (let j = 1; j < b.length; j++)
if (edgesIntersect(a[i - 1], a[i], b[j - 1], b[j])) return true;
return false;
}
/**
* Check only generated bounded output, not a new input-geometry policy.
* Touching generated boundaries are conservatively unsupported here.
*/
function simplifiedTopologyFits(polygons) {
for (const [outer, ...holes] of polygons) {
for (const ring of [outer, ...holes]) {
if (ringArea(ring) === 0) return false;
for (let i = 1; i < ring.length; i++)
for (let j = i + 2; j < ring.length; j++) {
if (i === 1 && j === ring.length - 1) continue;
if (edgesIntersect(ring[i - 1], ring[i], ring[j - 1], ring[j]))
return false;
}
}
for (let i = 0; i < holes.length; i++) {
const hole = holes[i];
if (
!hole.every(([lon, lat]) => pointInRing(outer, lat, lon)) ||
ringsIntersect(outer, hole)
)
return false;
for (let j = 0; j < i; j++) {
const other = holes[j];
if (
ringsIntersect(hole, other) ||
pointInRing(other, hole[0][1], hole[0][0]) ||
pointInRing(hole, other[0][1], other[0][0])
)
return false;
}
}
}
return true;
}
/** Simplify until the whole outline fits the vertex cap. */
export function capOutlineVertices(
polygons,
@@ -468,22 +474,20 @@ export function capOutlineVertices(
let current = polygons;
let tolerance = 0.00002;
for (let round = 0; round < 12; round++) {
if (countVertices(current) <= maxVertices) return current;
if (countVertices(current) <= maxVertices)
return current === polygons || simplifiedTopologyFits(current)
? current
: null;
current = polygons.map((rings) =>
rings.map((ring) => simplifyClosedRing(ring, tolerance)),
);
tolerance *= 2;
}
// Still over: keep the largest parts that fit, outer rings only.
const kept = [];
let total = 0;
for (const rings of current) {
const outer = rings[0];
if (total + outer.length > maxVertices) break;
kept.push([outer]);
total += outer.length;
}
return kept.length ? kept : null;
// Never turn a resource refusal into an apparently complete partial outline.
return countVertices(current) <= maxVertices &&
simplifiedTopologyFits(current)
? current
: null;
}
/**
@@ -504,8 +508,9 @@ export function selectOutlineResult(rows, { kind, query }) {
continue;
}
if (
String(osmClass).toLowerCase() === 'boundary' &&
String(osmType).toLowerCase() === 'administrative' &&
(String(osmClass).toLowerCase() === 'place' ||
(String(osmClass).toLowerCase() === 'boundary' &&
String(osmType).toLowerCase() === 'administrative')) &&
!adminLevelFits(kind, row)
) {
skipped.push(`${label}: level`);
@@ -535,7 +540,6 @@ export function selectOutlineResult(rows, { kind, query }) {
skipped.push(`${label}: too large`);
continue;
}
const adminIdentity = selectedAdminIdentity(kind, row);
const lat = Number(row.lat);
const lon = Number(row.lon);
return {
@@ -548,7 +552,6 @@ export function selectOutlineResult(rows, { kind, query }) {
row.osm_type && row.osm_id != null
? `${row.osm_type}/${row.osm_id}`
: null,
...(adminIdentity || {}),
center: finiteLat(lat) && finiteLon(lon) ? { lat, lon } : null,
polygons,
},
server/providers/regional/place.js
+130 -60
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
import { coalesceProxyRequest } from '../common/http.js';
import { admitSameSite } from '../common/same-site.js';
import { defaultSourceRoot } from '../common/source-root.js';
import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
import {
nominatimToGeocodeResult,
@@ -30,8 +31,8 @@ const OUTLINE_RESULT_LIMIT = 3;
/** Largest upstream answer read for an outline (polygons are simplified upstream). */
const OUTLINE_MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
/** Bump when the cached payload shape changes, so old entries are never read. */
const CACHE_SCHEMA = 'v2';
/** Bump when cached answers change shape or selection authority. */
const CACHE_SCHEMA = 'v3';
/**
* The query text actually sent upstream: Unicode-composed, whitespace
@@ -57,9 +58,73 @@ function malformed() {
});
}
function abandoned() {
return Object.assign(new Error('Place search was abandoned'), {
code: 'NOMINATIM_ABANDONED',
});
}
/**
* Share one upstream call while keeping each caller's cancellation independent.
* The shared controller is aborted only after the last subscriber leaves.
*/
function subscribeSharedRequest(inFlight, key, signal, create) {
if (signal?.aborted) return Promise.reject(abandoned());
let flight = inFlight.get(key);
if (!flight) {
flight = {
controller: new AbortController(),
users: 0,
settled: false,
promise: null,
};
const owned = flight;
owned.promise = Promise.resolve()
.then(() => create(owned.controller.signal))
.finally(() => {
owned.settled = true;
if (inFlight.get(key) === owned) inFlight.delete(key);
});
inFlight.set(key, owned);
}
flight.users += 1;
return new Promise((resolve, reject) => {
let left = false;
const leave = () => {
if (left) return;
left = true;
signal?.removeEventListener('abort', onAbort);
flight.users -= 1;
if (
flight.users === 0 &&
!flight.settled &&
inFlight.get(key) === flight
) {
inFlight.delete(key);
flight.controller.abort();
}
};
const onAbort = () => {
leave();
reject(abandoned());
};
signal?.addEventListener('abort', onAbort, { once: true });
flight.promise.then(
(value) => {
leave();
resolve(value);
},
(error) => {
leave();
reject(error);
},
);
});
}
/** Where the dev server keeps Nominatim answers and the daily count. */
export const NOMINATIM_CACHE_DIR = path.join(
process.cwd(),
defaultSourceRoot,
'.gev-cache',
'nominatim',
);
@@ -76,6 +141,9 @@ export const fetchRegionalPlace = createRegionalPlaceProvider();
* moves reuse one cache entry. The viewbox is a soft preference upstream.
*/
function coarseBounds(bounds) {
// Validate the original hint before snapping or clamping can disguise a
// wrapped, inverted or out-of-range box as an unrelated valid viewbox.
if (!nominatimViewboxFromBounds(bounds)) return null;
const match = String(bounds || '')
.trim()
.split('|')
@@ -122,27 +190,33 @@ export function createNominatimSearchProvider({
const cacheKey = requestKey(gateNow, 'search', params);
const cached = await cache.get(cacheKey);
if (cached) return { ...cached.payload, cached: true };
const { promise } = coalesceProxyRequest(inFlight, cacheKey, async () => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal },
);
// Only a well-formed empty list is "no such place"; anything else is a
// failure and is never cached.
if (!Array.isArray(rows)) throw malformed();
const result = nominatimToGeocodeResult(rows[0] ?? null);
if (rows.length && !result) throw malformed();
const payload = result
? { status: 'OK', results: [result] }
: { status: 'ZERO_RESULTS', results: [] };
await cache.set(
cacheKey,
payload,
result ? NOMINATIM_CACHE_TTL_MS.place : NOMINATIM_CACHE_TTL_MS.notFound,
);
return payload;
});
return await promise;
return await subscribeSharedRequest(
inFlight,
cacheKey,
signal,
async (sharedSignal) => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal: sharedSignal },
);
// Only a well-formed empty list is "no such place"; anything else is a
// failure and is never cached.
if (!Array.isArray(rows)) throw malformed();
const result = nominatimToGeocodeResult(rows[0] ?? null);
if (rows.length && !result) throw malformed();
const payload = result
? { status: 'OK', results: [result] }
: { status: 'ZERO_RESULTS', results: [] };
await cache.set(
cacheKey,
payload,
result
? NOMINATIM_CACHE_TTL_MS.place
: NOMINATIM_CACHE_TTL_MS.notFound,
);
return payload;
},
);
};
}
@@ -178,41 +252,38 @@ export function createNominatimOutlineProvider({
const cacheKey = requestKey(gateNow, `outline-${kind}`, params);
const cached = await cache.get(cacheKey);
if (cached) return { ...cached.payload, cached: true };
const { promise } = coalesceProxyRequest(inFlight, cacheKey, async () => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal, maxBytes: OUTLINE_MAX_RESPONSE_BYTES },
);
if (!Array.isArray(rows)) throw malformed();
const { outline, skipped } = selectOutlineResult(rows, {
kind,
query: sent,
});
const payload = outline
? { status: 'OK', outline }
: { status: 'ZERO_RESULTS', outline: null, skipped };
await cache.set(
cacheKey,
payload,
outline
? NOMINATIM_CACHE_TTL_MS.outline
: NOMINATIM_CACHE_TTL_MS.notFound,
);
return payload;
});
return await promise;
return await subscribeSharedRequest(
inFlight,
cacheKey,
signal,
async (sharedSignal) => {
const rows = await gateNow.requestJson(
(endpoint) => `${endpoint}?${params}`,
{ signal: sharedSignal, maxBytes: OUTLINE_MAX_RESPONSE_BYTES },
);
if (!Array.isArray(rows)) throw malformed();
const { outline, skipped } = selectOutlineResult(rows, {
kind,
query: sent,
});
const payload = outline
? { status: 'OK', outline }
: { status: 'ZERO_RESULTS', outline: null, skipped };
await cache.set(
cacheKey,
payload,
outline
? NOMINATIM_CACHE_TTL_MS.outline
: NOMINATIM_CACHE_TTL_MS.notFound,
);
return payload;
},
);
};
}
export const fetchNominatimSearch = createNominatimSearchProvider();
/** A browser on another site may not spend this install's allowance. */
function crossSite(req) {
return (
String(req.headers?.['sec-fetch-site'] || '').toLowerCase() === 'cross-site'
);
}
function sendJson(res, status, body, headers = {}) {
if (res.writableEnded) return;
res.writeHead(status, {
@@ -312,14 +383,11 @@ export function geocodeProxy({
/** Common front door: method, same-site, rate limit, abandonment signal. */
function admit(req, res, limiter) {
if (admitSameSite(req, res)) return null;
if (req.method !== 'GET') {
sendJson(res, 405, { error: 'Method Not Allowed' });
return null;
}
if (crossSite(req)) {
sendJson(res, 403, { error: 'Cross-site requests are not accepted' });
return null;
}
if (!limiter(clientKey(req))) {
sendJson(
res,
@@ -394,7 +462,9 @@ export function geocodeProxy({
res,
200,
{ status: payload.status, results: payload.results },
{ 'Cache-Control': payload.cached ? 'public, max-age=60' : 'no-store' },
{
'Cache-Control': payload.cached ? 'private, max-age=60' : 'no-store',
},
);
} catch (error) {
sendFailure(res, error, 'Place search');
server/providers/space/celestrak.js
+70 -8
View File
@@ -1,6 +1,9 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import { providerCooldownMs } from './cooldown.js';
import { admitSameSiteKeyless } from '../common/same-site.js';
import { readResponseTextCapped } from '../common/http.js';
/**
* Vite plugin: CelesTrak TLE proxy.
@@ -23,6 +26,18 @@ import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
export function celestrakProxy() {
const TLE_TTL_MS = 6 * 3600_000;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const groups = new Set([
'stations',
'visual',
'gps-ops',
'glo-ops',
'galileo',
'geo',
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]);
let cooldownUntil = 0;
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
@@ -48,10 +63,14 @@ export function celestrakProxy() {
}
}
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
async function fetchUpstream(group) {
const url = celestrakTleUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal: AbortSignal.timeout(20000),
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
@@ -59,8 +78,21 @@ export function celestrakProxy() {
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const body = await res.text();
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
cooldownUntil = Math.max(
cooldownUntil,
Date.now() + providerCooldownMs(res),
);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
return { at: Date.now(), body };
@@ -71,12 +103,20 @@ export function celestrakProxy() {
const group = String(req.url || '')
.replace(/^\//, '')
.split('?')[0];
if (!/^[a-z0-9-]+$/i.test(group)) {
// Keyless and needed to render the satellites layer: refuse cross-site
// browsers but keep the documented reverse-proxy deployment working.
if (admitSameSiteKeyless(req, res)) return;
if (req.method !== 'GET') {
res.writeHead(405);
res.end();
return;
}
if (!groups.has(group)) {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.end('invalid group');
return;
}
const send = (status, body, cacheStatus) => {
const send = (status, body, cacheStatus, entry) => {
// Guard against a double-send (e.g. a throw AFTER a response already
// went out routing into the catch's send): writeHead after headersSent
// throws "Cannot set headers after they are sent".
@@ -84,6 +124,17 @@ export function celestrakProxy() {
res.writeHead(status, {
'Content-Type': 'text/plain',
'x-tle-cache': cacheStatus,
'Cache-Control': 'no-store',
...(entry
? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
: {}),
...(cooldownUntil > 0 && Date.now() < cooldownUntil
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
),
}
: {}),
});
res.end(body);
};
@@ -95,7 +146,16 @@ export function celestrakProxy() {
if (entry) mem.set(group, entry);
}
if (entry && now - entry.at < TLE_TTL_MS) {
send(200, entry.body, 'HIT');
send(200, entry.body, 'HIT', entry);
return;
}
if (Date.now() < cooldownUntil) {
send(
entry ? 200 : 502,
entry?.body || 'celestrak upstream cooling down',
entry ? 'STALE-ERROR' : 'NONE',
entry,
);
return;
}
// Stale or missing → refresh, single-flight per group.
@@ -109,6 +169,8 @@ export function celestrakProxy() {
return fresh;
})
.catch((err) => {
if (Date.now() >= cooldownUntil)
cooldownUntil = Date.now() + 60000;
console.warn(
'[celestrak-proxy] refresh failed — serving cache if any',
);
@@ -119,9 +181,9 @@ export function celestrakProxy() {
}
const fresh = await inflight.get(group);
if (fresh) {
send(200, fresh.body, 'MISS');
send(200, fresh.body, 'MISS', fresh);
} else if (entry) {
send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty
send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty
} else {
send(502, 'celestrak fetch failed and no cache available', 'NONE');
}
server/providers/space/cooldown.js
+12
View File
@@ -0,0 +1,12 @@
/** Bound an upstream Retry-After delay (seconds or HTTP date). */
export function providerCooldownMs(response, now = Date.now()) {
const raw = response?.headers?.get('retry-after')?.trim();
const seconds = raw && /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) : NaN;
const deadline = raw && !Number.isFinite(seconds) ? Date.parse(raw) : NaN;
const delay = Number.isFinite(seconds)
? seconds * 1000
: Number.isFinite(deadline)
? deadline - now
: 60000;
return Math.min(30 * 60000, Math.max(5000, delay));
}
server/providers/space/launch-library.js
+40 -2
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { providerCooldownMs } from './cooldown.js';
import { promises as fsp } from 'node:fs';
import {
readResponseTextCapped,
@@ -13,6 +14,8 @@ export function launchLibraryRequestHeaders(token = process.env.LL2_API_TOKEN) {
const normalized = String(token || '').trim();
return {
Accept: 'application/json',
'User-Agent':
'gods-eye-view-launch-library-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
...(normalized ? { Authorization: `Token ${normalized}` } : {}),
};
}
@@ -28,6 +31,8 @@ export function rocketLaunchesProxy() {
'launch-library-2-v2.3.json',
);
let cache = null;
let cooldownUntil = 0;
let cooldownStatus = 502;
let diskLoaded = false;
const inFlight = new Map();
@@ -60,7 +65,18 @@ export function rocketLaunchesProxy() {
function send(res, status, body, cacheState) {
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': status === 200 ? 'public, max-age=900' : 'no-store',
'Cache-Control':
status === 200 && cacheState !== 'STALE-ERROR'
? 'public, max-age=900'
: 'no-store',
...(cache ? { 'X-GEV-Cached-At': new Date(cache.at).toISOString() } : {}),
...(Date.now() < cooldownUntil
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
),
}
: {}),
'X-GEV-Cache': cacheState,
});
res.end(body);
@@ -73,17 +89,25 @@ export function rocketLaunchesProxy() {
signal: AbortSignal.timeout(20000),
headers: launchLibraryRequestHeaders(),
});
const body = await readResponseTextCapped(upstream, maxResponseBytes);
if (!upstream.ok) {
const deadline = Date.now() + providerCooldownMs(upstream);
// Never shorten a longer cooldown that is already in force.
if (deadline >= cooldownUntil) {
cooldownUntil = deadline;
cooldownStatus = upstream.status;
}
void upstream.body?.cancel().catch(() => {});
const error = new Error(`upstream HTTP ${upstream.status}`);
error.upstreamStatus = upstream.status;
throw error;
}
const body = await readResponseTextCapped(upstream, maxResponseBytes);
const parsed = JSON.parse(body);
if (!Array.isArray(parsed?.results))
throw new Error('malformed upstream response');
const fresh = { at: Date.now(), body };
cache = fresh;
cooldownUntil = 0;
void saveDiskCache(fresh);
return fresh;
}
@@ -101,6 +125,16 @@ export function rocketLaunchesProxy() {
return;
}
const stale = cache;
if (now < cooldownUntil) {
send(
res,
stale ? 200 : cooldownStatus,
stale?.body ||
JSON.stringify({ error: 'Launch Library 2 unavailable' }),
stale ? 'STALE-ERROR' : 'NONE',
);
return;
}
const request = coalesceProxyRequest(
inFlight,
'recent-launches',
@@ -110,6 +144,10 @@ export function rocketLaunchesProxy() {
const fresh = await request.promise;
send(res, 200, fresh.body, request.shared ? 'INFLIGHT' : 'MISS');
} catch (error) {
if (Date.now() >= cooldownUntil) {
cooldownUntil = Date.now() + 60000;
cooldownStatus = 502;
}
// Log only a bounded status, never upstream bodies, URLs, or credentials.
const status = Number.isInteger(error?.upstreamStatus)
? error.upstreamStatus
server/providers/terrain.js
+113 -9
View File
@@ -8,6 +8,8 @@ import {
terrainPointKey,
validTerrainResult,
} from '../../src/data/terrainHeightsProxy.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { admitSameSiteKeyless } from './common/same-site.js';
/**
* Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height
@@ -18,13 +20,33 @@ import {
* cache and serve-stale shape. Cache entries and stale fallback are keyed per
* 5dp point, so reordered and partially overlapping batches reuse prior work.
* Only missing/stale points go upstream; the response is rebuilt in exact
* request order. Larger requests are chunked sequentially, and one failing
* chunk does not discard the chunks that resolved.
* request order. Each request is bounded to one 64-point upstream chunk;
* larger client workloads are split by the browser terrain service.
*
* Bounds, because every distinct 5dp point is its own key: coordinates must be
* in WGS-84 range (rejected at parse), the point cache has an entry ceiling
* with oldest-first eviction, an over-large cache file is not read back, and
* the route carries an always-on per-client limiter. The route also refuses
* cross-site browser requests, so another web page cannot use the visitor's
* browser to fill the cache or spend the visitor's own rate-limit quota.
* Reverse-proxied same-origin requests are admitted: terrain is keyless and
* needed for rendering, and a 403 would send every chunk to the geoid fallback.
*/
export function terrainHeightsProxy() {
export function terrainHeightsProxy(options = {}) {
// Destructured in the body, not the signature: `proxyErrorResponses.test.mjs`
// extracts these proxies by slicing to the first closing brace in column
// zero, so a multi-line parameter list would cut the function in half.
// Overridable so the ceilings can be exercised as behaviour rather than
// asserted as constants; production constructs this with no arguments.
const {
cacheMaxEntries = 50_000,
cacheMaxDiskBytes = 24 * 1024 * 1024,
cachePath,
} = options;
const TTL_MS = 30 * 24 * 3600_000;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json');
const CACHE_PATH =
cachePath || path.join(process.cwd(), '.gev-cache', 'terrain-heights.json');
const CACHE_DIR = path.dirname(CACHE_PATH);
// Sized against measured upstream latency, not the documented page cap.
// Re:Earth serves 87-186 ms/point depending on load (observed 2026-09-12,
// a 2x swing within one hour). 256 points therefore costs 22-48 s and blows
@@ -33,10 +55,59 @@ export function terrainHeightsProxy() {
// 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
// that range.
const UPSTREAM_CHUNK = 64;
const MAX_POINTS = 2000;
const MAX_POINTS = UPSTREAM_CHUNK;
/**
* Entry ceiling for the point cache, and with it the file written from it.
*
* Every distinct 5dp point is a separate key, so without a ceiling the map
* — and the JSON dumped from it verbatim — grow for as long as a caller
* supplies new coordinates. Measured against real use rather than guessed:
* this checkout's `.gev-cache/terrain-heights.json` held 6,189 entries in
* 1.0 MB after ordinary sessions, i.e. ~168 bytes/entry. 50,000 entries is
* eight times that working set and ~8 MB on disk, comfortably inside the
* 24 MB ceiling `rocketLaunchesProxy` already applies to its own cache file.
*/
const CACHE_MAX_ENTRIES = cacheMaxEntries;
/** Refuse to load a cache file larger than this (mirrors rocketLaunchesProxy). */
const CACHE_MAX_DISK_BYTES = cacheMaxDiskBytes;
/**
* Requests/min/client. Terrain is keyless and free, so this guards the point
* cache and the upstream community service rather than a bill. Sized from
* measurement: across a full `npm run test:track` run — the heaviest
* exercise this app has — the route served 10 requests, peaking at 8/min.
* 90/min is an order of magnitude above that, and the band `/api/overpass`
* and `/api/military-installations` already use.
*/
const RATE_PER_MIN = 90;
const allow = makeRateLimiter({
windowMs: 60_000,
max: RATE_PER_MIN,
globalMax: RATE_PER_MIN * 3,
});
/** @type {Map<string, {at:number, result:object}>} keyed by canonical 5dp lon/lat. */
const mem = new Map();
/**
* Drop the oldest entries until the map is back inside its ceiling.
* The map is kept in fetch-time order (sorted once on load, and the resolver
* re-inserts every point it refetches), so the oldest are at the front and
* no write sorts the map. Terrain does not move, so evicting an old point
* costs one refetch, never a wrong answer.
*/
function evictOverflow() {
for (const key of mem.keys()) {
if (mem.size <= CACHE_MAX_ENTRIES) return;
mem.delete(key);
}
}
/** Put the map in fetch-time order; a file is not guaranteed to be. */
function orderByAge() {
const byAge = [...mem.entries()].sort((a, b) => a[1].at - b[1].at);
mem.clear();
for (const [key, entry] of byAge) mem.set(key, entry);
}
/** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
const inflight = new Map();
let diskLoaded = false;
@@ -47,6 +118,21 @@ export function terrainHeightsProxy() {
if (diskLoaded) return;
diskLoaded = true;
try {
// A cache file past the ceiling is not read back into memory: loading it
// would reinstate exactly the unbounded map the ceiling exists to stop.
// The next flush rewrites a bounded one.
//
// "Cannot measure it" is not "too large": if stat is unavailable, fall
// through and let readFile decide, so an absent file behaves exactly as
// it did before this guard existed.
let sizeBytes = null;
try {
sizeBytes = (await fsp.stat(CACHE_PATH)).size;
} catch {
sizeBytes = null;
}
if (sizeBytes !== null && sizeBytes > CACHE_MAX_DISK_BYTES)
throw new Error('cache file too large');
const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
const pointEntries =
parsed?.version === 2 &&
@@ -87,6 +173,8 @@ export function terrainHeightsProxy() {
}
diskDirty = mem.size > 0;
}
orderByAge();
evictOverflow();
} catch {
/* no disk cache yet */
}
@@ -157,13 +245,25 @@ export function terrainHeightsProxy() {
const installMiddleware = (server) => {
server.middlewares.use('/api/terrain/heights', async (req, res) => {
// Refuse cross-site browser requests (an <img> or fetch from another
// site) before the limiter, cache or upstream see them. Same-origin app
// requests, non-browser loopback callers (MCP panel_request) and the app
// served behind a reverse proxy pass.
if (admitSameSiteKeyless(req, res)) return;
const send = (status, bodyObj) => {
if (res.headersSent) return;
res.writeHead(status, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(bodyObj));
};
try {
await loadDiskOnce();
if (!allow(clientKey(req))) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Retry-After': '10',
});
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
const parsedUrl = new URL(req.url || '', 'http://internal');
const rawPoints = parsedUrl.searchParams.get('points');
const points = parseTerrainPoints(rawPoints);
@@ -175,19 +275,23 @@ export function terrainHeightsProxy() {
return;
}
if (points.length > MAX_POINTS) {
send(500, {
send(413, {
error: `too many points (${points.length}); max ${MAX_POINTS} per request`,
});
return;
}
await loadDiskOnce();
const outcome = await resolveTerrainHeightRequest({
points,
cache: mem,
fetchMissing: fetchMissingSingleFlight,
ttlMs: TTL_MS,
});
if (outcome.cacheChanged) diskDirty = true;
if (outcome.cacheChanged) {
evictOverflow();
diskDirty = true;
}
if (outcome.upstreamError) {
console.warn(
'[terrain-heights-proxy] refresh incomplete' +
server/providers/traffic.js
+30 -10
View File
@@ -1,12 +1,14 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseBytesCapped } from './common/http.js';
import {
isValidTileCoord as isValidTomTomTile,
utcDayKey as tomtomUtcDayKey,
normalizeBudget as normalizeTomTomBudget,
isOverBudget as isTomTomOverBudget,
} from '../../src/data/tomtomTiles.js';
import { admitSameSite } from './common/same-site.js';
/**
* TomTom traffic-flow vector-tile proxy with a daily budget governor.
@@ -34,6 +36,13 @@ import {
* for the rest of the billing period. 6,000/day keeps a full 31-day month
* inside the allowance (186,000) with headroom to spare.
*
* Cross-site gate: both routes refuse cross-site browser requests with 403
* (the gate Mapillary, OpenAI and Places already use). Every uncached tile is
* a billed upstream attempt counted against the persistent daily budget, so
* without the gate any web page the user visits could fire `<img>` requests
* at distinct tiles, spend the TomTom quota and leave the traffic layer on
* 429 budget for the rest of the UTC day.
*
* GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode:
* status reports hasKey:false and the tile endpoint 503s {error:'no_key'}
* without touching upstream — the traffic layer then stays in simulation mode.
@@ -48,6 +57,8 @@ export function tomtomProxy() {
const DEFAULT_DAILY_BUDGET = 6000;
const MEM_MAX_ENTRIES = 256;
const UPSTREAM_TIMEOUT_MS = 15000;
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
const mem = new Map();
@@ -143,17 +154,23 @@ export function tomtomProxy() {
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
recordUpstreamFetch(); // attempts count — upstream bills the request either way
const res = await fetch(url, {
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
const bytes = await readResponseBytesCapped(
res,
TOMTOM_TILE_MAX_RESPONSE_BYTES,
signal,
);
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (buf.length === 0) throw new Error('empty tile body');
return buf;
}
const installMiddleware = (server) => {
server.middlewares.use('/api/tomtom', async (req, res) => {
// Gate first: a cross-site <img>/fetch must not reach the budget counter.
if (admitSameSite(req, res)) return;
// Sanitized responses only (proxy/security baseline): no upstream
// error details, and never echo the key or the upstream URL.
const sendJson = (status, obj, extraHeaders = {}) => {
@@ -165,14 +182,17 @@ export function tomtomProxy() {
});
res.end(JSON.stringify(obj));
};
const sendTile = (buf, cacheStatus) => {
// x-tomtom-fetched-at is when the proxy fetched the tile (ms since the
// epoch), so a client handed a STALE-* copy can say how old it is.
const sendTile = (entry, cacheStatus) => {
if (res.headersSent) return;
res.writeHead(200, {
'Content-Type': 'application/x-protobuf',
'Cache-Control': 'no-store',
'x-tomtom-cache': cacheStatus,
'x-tomtom-fetched-at': String(Math.round(entry.at)),
});
res.end(buf);
res.end(entry.buf);
};
try {
@@ -218,14 +238,14 @@ export function tomtomProxy() {
}
// Fresh cache hit — never counts against the budget.
if (entry && now - entry.at < TILE_TTL_MS) {
sendTile(entry.buf, 'HIT');
sendTile(entry, 'HIT');
return;
}
// Budget governor: over the soft cap, last-good data beats a dead layer.
if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) {
if (entry) {
sendTile(entry.buf, 'STALE-BUDGET');
sendTile(entry, 'STALE-BUDGET');
} else {
sendJson(429, { error: 'budget' });
}
@@ -254,9 +274,9 @@ export function tomtomProxy() {
}
const fresh = await inflight.get(key);
if (fresh) {
sendTile(fresh.buf, 'MISS');
sendTile(fresh, 'MISS');
} else if (entry) {
sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty
sendTile(entry, 'STALE-ERROR'); // upstream down — stale beats empty
} else {
sendJson(502, { error: 'upstream' });
}
server/providers/transitHistory.js
-1
View File
@@ -1,5 +1,4 @@
export {
createTransitHistory,
TRANSIT_HISTORY_LIMITS,
validTransitIdentifier,
} from '../../src/sources/transitHistoryStore.js';
server/providers/transitHistory.test.mjs
+32 -7
View File
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
const rows = Array.from({ length: 15000 }, (_, i) =>
row(now, `vehicle-${i}`),
);
const start = performance.now();
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
const start = process.threadCpuUsage();
h.ingest(feed, rows, now);
times.push(performance.now() - start);
const usage = process.threadCpuUsage(start);
times.push((usage.user + usage.system) / 1000);
}
assert.equal(h.diagnostics().keys, 15000);
for (let i = 0; i < 15000; i++) {
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
}
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
t.diagnostic(
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
);
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
// well under two seconds on a loaded laptop or a slow CI runner. The
// per-run figure above is the diagnostic to read.
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
// CPU time measures ingestion work without counting scheduler delays on
// loaded runners. Keep the two-second ceiling as a guard against quadratic
// work, not a benchmark; the per-poll figures above show any slowdown.
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
});
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {
@@ -177,3 +179,26 @@ test('snapshot capacity pressure preserves members not yet visited and discloses
assert.equal(recreated.fixes.length, 1);
assert.equal(recreated.truncated, true);
});
test('a feed fleet-number mode outranks its route mode in the epoch context', (t) => {
const h = history(t);
const ttc = {
id: 'ttc',
historyRetention: true,
defaultMode: 'bus',
routeMode: (route) => (/^5\d\d$/.test(route ?? '') ? 'tram' : 'bus'),
vehicleMode: (id) =>
/^4[4-6]\d\d$/.test(id) ? 'tram' : /^\d{4}$/.test(id) ? 'bus' : 'unknown',
};
h.ingest(
ttc,
[
row(100000, '8515', { routeId: '506' }),
row(100000, '4507', { routeId: null }),
row(100000, 'x1', { routeId: '501' }),
],
100000,
);
assert.equal(h.get('ttc', '8515', 100000).epochs[0].mode, 'bus');
assert.equal(h.get('ttc', '4507', 100000).epochs[0].mode, 'tram');
assert.equal(h.get('ttc', 'x1', 100000).epochs[0].mode, 'tram');
});
server/providers/vessels/ais-live.js
+5 -4
View File
@@ -71,14 +71,15 @@ let _aisWebSocketImpl;
*/
export function aisLiveProxy() {
function install(middlewares) {
middlewares.use('/api/ais-live', async (req, res) => {
middlewares.use('/api/vessels', async (req, res) => {
try {
res.setHeader('X-Feed-Source', 'AISStream');
ensureAisStreamConnection();
const incoming = new URL(req.url || '', 'http://localhost');
// Track sub-route MUST be handled before the rows snapshot — this
// mount prefix-matches every subpath, so without this branch
// /api/ais-live/track would be silently answered with vessel rows.
// /api/vessels/track would be silently answered with vessel rows.
if (
incoming.pathname === '/track' ||
incoming.pathname.startsWith('/track/')
@@ -288,7 +289,7 @@ function aisKeyFingerprint() {
}
/**
* Drive the watchdog once. Called on every /api/ais-live request and on the
* Drive the watchdog once. Called on every /api/vessels request and on the
* background interval, so recovery does not depend on browser traffic.
*/
function ensureAisStreamConnection() {
@@ -307,7 +308,7 @@ function ensureAisStreamConnection() {
keyFingerprint: aisKeyFingerprint(),
});
}
/** Status metadata for /api/ais-live, safe to call before the first connect. */
/** Status metadata for /api/vessels, safe to call before the first connect. */
function aisStreamStatusSnapshot() {
const snapshot = _aisAdapter ? _aisAdapter.snapshot() : null;
if (snapshot) return snapshot;
server/providers/vessels/ais-store.js
+55 -10
View File
@@ -43,14 +43,55 @@ export function ingestAisStreamEnvelope(envelope) {
if (!mmsi) return false;
if (messageType === 'ShipStaticData' || messageType === 'StaticDataReport') {
const previous = _aisStreamStatic.get(mmsi) || {};
const live = _aisStreamVessels.get(mmsi);
const liveName = live?.name === `MMSI ${mmsi}` ? '' : live?.name;
const fields =
messageType === 'StaticDataReport'
? {
Name:
message.ReportA?.Valid === false
? undefined
: message.ReportA?.Name,
Type:
message.ReportB?.Valid === false
? undefined
: message.ReportB?.ShipType,
}
: message;
// Static reports may be partial (including Class B's separate A/B parts).
// Keep known values in the cache as well as in the current live row.
const staticData = {
name: vesselNameFromAis(metadata, message, _aisStreamStatic.get(mmsi)),
type: vesselTypeFromAis(message, _aisStreamStatic.get(mmsi)),
destination: stringValue(message.Destination),
imo: stringValue(message.ImoNumber ?? message.IMO),
// The envelope repeats a cached name even on Class B's type-only part.
// Only an actual static name report replaces a known live name.
name:
stringValue(fields.Name ?? fields.ShipName) ||
liveName ||
previous.name ||
stringValue(metadata.ShipName),
type:
staticCode(fields.Type ?? fields.ShipType) ||
staticCode(live?.type) ||
staticCode(previous.type) ||
'',
destination:
stringValue(fields.Destination) ||
live?.destination ||
previous.destination ||
'',
imo:
staticCode(fields.ImoNumber ?? fields.IMO) ||
staticCode(live?.imo) ||
staticCode(previous.imo) ||
'',
};
_aisStreamStatic.set(mmsi, staticData);
mergeAisStaticIntoLiveVessel(mmsi, staticData);
// AISStream repeats the last known position in a static message's
// MetaData, but the message has no speed, course or heading and is not a
// new fix. Rebuilding a tracked vessel from it would blank its motion and
// restamp its position time, so only merge the static fields.
if (_aisStreamVessels.has(mmsi)) return true;
}
const lat = numberValue(
@@ -191,12 +232,10 @@ function approxMetersBetween(lat1, lon1, lat2, lon2) {
function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
const existing = _aisStreamVessels.get(mmsi);
if (!existing) return;
if (staticData.name && (!existing.name || existing.name === `MMSI ${mmsi}`))
existing.name = staticData.name;
if (staticData.type && !existing.type) existing.type = staticData.type;
if (staticData.destination && !existing.destination)
existing.destination = staticData.destination;
if (staticData.imo && !existing.imo) existing.imo = staticData.imo;
if (staticData.name) existing.name = staticData.name;
if (staticData.type) existing.type = staticData.type;
if (staticData.destination) existing.destination = staticData.destination;
if (staticData.imo) existing.imo = staticData.imo;
}
function vesselNameFromAis(metadata, message, staticData = {}) {
@@ -266,6 +305,12 @@ function stringValue(value) {
return String(value).trim();
}
// AIS uses zero for unavailable ship type and IMO, not as a replacement value.
function staticCode(value) {
const number = numberValue(value);
return Number.isInteger(number) && number > 0 ? String(number) : '';
}
function numberValue(value) {
if (value === null || value === undefined || value === '') return null;
const number = Number(value);
server/providers/wind.js
+6 -2
View File
@@ -3,6 +3,10 @@ import { fetchIfsWind } from './wind/ifs.js';
import { decodeWindGribMessage } from './wind/decode.js';
import { weatherScalarMetadata, weatherScalarError } from './wind/grid.js';
// Data, not UI copy: the client (src/layers/wind/index.js) marks this exact
// English with msg() and translates it when shown, so keep the two in sync.
const UPSTREAM_UNAVAILABLE = 'Wind upstream unavailable';
/** Serve bounded, single-flight, per-model/overlay forecast snapshots from fixed providers. */
export function windProxy({
fetchImpl = fetch,
@@ -25,7 +29,7 @@ export function windProxy({
...(overlay === 'none' ? {} : { overlay }),
unavailable: true,
stale: true,
reason: 'Wind upstream unavailable',
reason: UPSTREAM_UNAVAILABLE,
},
});
const sendJson = (res, value, status = 200) => {
@@ -139,7 +143,7 @@ export function windProxy({
old.manifest = {
...old.manifest,
stale: true,
reason: 'Wind upstream unavailable',
reason: UPSTREAM_UNAVAILABLE,
};
return old;
}
server/providers/wind/ifs.js
+1 -1
View File
@@ -3,7 +3,7 @@ import { decodeWindGribMessage } from './decode.js';
import { resampleWeatherSnapshot, weatherScalarMetadata } from './grid.js';
/** Keyless ECMWF Open Data root for real-time IFS forecasts. */
export const IFS_BASE = 'https://data.ecmwf.int/forecasts';
const IFS_BASE = 'https://data.ecmwf.int/forecasts';
/**
* Select the latest IFS cycle that should be available.
server/standalone/google-token-unavailable.js
+27
View File
@@ -0,0 +1,27 @@
/**
* Treat an unavailable Google Maps tile-token endpoint as a supported, silent
* outcome. Install after real token providers and before the unknown-API 404.
*/
export function googleTokenUnavailablePlugin() {
const install = (server) => {
server.middlewares.use((req, res, next) => {
const pathname = String(req.url || '').split('?')[0];
if (
(req.method !== 'GET' && req.method !== 'HEAD') ||
pathname !== '/api/google/tiles-token'
) {
next();
return;
}
res.writeHead(204, { 'Cache-Control': 'no-store' });
res.end();
});
};
return {
name: 'google-token-unavailable',
configureServer: install,
configurePreviewServer: install,
};
}
server/standalone/key-setup.js
+29 -7
View File
@@ -134,6 +134,8 @@ function keySetupEndpoint({ sourceRoot = defaultSourceRoot } = {}) {
};
// The gate itself is pure and unit-tested (admitKeySetupRequest in
// src/keySetupCore.mjs) — this just feeds it the request.
// `code`/`params` pick the panel's sentence (src/keySetup.js); `error` is for logs.
const refusal = ({ error, code, params }) => ({ error, code, params });
const admit = (req) =>
admitKeySetupRequest({
method: req.method,
@@ -247,18 +249,24 @@ function keySetupEndpoint({ sourceRoot = defaultSourceRoot } = {}) {
configureServer(server) {
server.middlewares.use('/api/setup/status', (req, res) => {
if (req.method !== 'GET')
return respond(res, 405, { error: 'Method not allowed' });
return respond(res, 405, {
error: 'Method not allowed',
code: 'method',
});
const admission = admit(req);
if (!admission.ok)
return respond(res, admission.status, { error: admission.error });
return respond(res, admission.status, refusal(admission));
respond(res, 200, providerStatus());
});
server.middlewares.use('/api/setup/keys', (req, res) => {
if (req.method !== 'POST')
return respond(res, 405, { error: 'Method not allowed' });
return respond(res, 405, {
error: 'Method not allowed',
code: 'method',
});
const admission = admit(req);
if (!admission.ok)
return respond(res, admission.status, { error: admission.error });
return respond(res, admission.status, refusal(admission));
let body = '';
let overflowed = false;
req.on('data', (chunk) => {
@@ -270,15 +278,21 @@ function keySetupEndpoint({ sourceRoot = defaultSourceRoot } = {}) {
});
req.on('end', () => {
if (overflowed)
return respond(res, 413, { error: 'Request too large' });
return respond(res, 413, {
error: 'Request too large',
code: 'too-large',
});
let parsed;
try {
parsed = JSON.parse(body || '{}');
} catch {
return respond(res, 400, { error: 'Invalid JSON' });
return respond(res, 400, {
error: 'Invalid JSON',
code: 'invalid-json',
});
}
const verdict = validateKeySetupUpdates(parsed);
if (!verdict.ok) return respond(res, 400, { error: verdict.error });
if (!verdict.ok) return respond(res, 400, refusal(verdict));
// Neither a replace NOR a removal may touch an externally-supplied
// credential (shell env, Keychain, another workflow). This backs the
// UI's read-only "configured externally" state with a real contract —
@@ -289,6 +303,8 @@ function keySetupEndpoint({ sourceRoot = defaultSourceRoot } = {}) {
if (isExternallyManaged(name, inStore)) {
return respond(res, 409, {
error: `${name} is configured outside Provider Settings and can only be changed where it was set`,
code: 'external',
params: { name },
});
}
}
@@ -305,10 +321,16 @@ function keySetupEndpoint({ sourceRoot = defaultSourceRoot } = {}) {
) {
return respond(res, 500, {
error: `The key was not saved: ${error.message}`,
code:
error.code === 'GEV_HARDEN_FAILED'
? 'harden-failed'
: 'store-unreadable',
});
}
return respond(res, 500, {
error: `Could not write the ${storeName()} store`,
code: 'write-failed',
params: { store: storeName() },
});
}
// Live for the server-side proxies immediately; the restart below is
server/standalone/unwatched-cache.js
+28
View File
@@ -0,0 +1,28 @@
/**
* Directories the app writes to while it runs. None is source, and none
* should reload anything.
*/
const RUNTIME_WRITE_GLOBS = Object.freeze([
'**/.gev-cache/**',
'**/.gev-logs/**',
]);
/**
* Keep the directories the app writes to out of the dev server's file watcher.
*
* Disk-cached providers write a file per response into `.gev-cache`, and the
* voice debug log appends to `.gev-logs`. Left watched, each write costs the
* watcher stat calls on libuv's thread pool, which `getaddrinfo` shares; heavy
* cache writes have been seen to delay upstream DNS lookups.
*
* @returns {import('vite').Plugin} Config-only plugin; no server hooks.
*/
export function unwatchedRuntimeDirsPlugin() {
return {
name: 'gev-unwatched-runtime-dirs',
apply: 'serve',
config: () => ({
server: { watch: { ignored: [...RUNTIME_WRITE_GLOBS] } },
}),
};
}
server/standalone/vite.config.js
+18 -3
View File
@@ -5,28 +5,43 @@ import { createBrowserViteConfig } from '../../build/vite.js';
import { localProviderPlugins } from '../providers/local.js';
import { localMcpPlugin } from '../mcp/plugin.js';
import { apiNotFoundPlugin } from './api-not-found.js';
import { googleTokenUnavailablePlugin } from './google-token-unavailable.js';
import { standaloneVoiceTools } from './voiceTools.js';
import { unwatchedRuntimeDirsPlugin } from './unwatched-cache.js';
const root = fileURLToPath(new URL('../../', import.meta.url));
/** Load this checkout's configuration and attach its local provider middleware. */
export default defineConfig(({ command, mode }) => {
export function standaloneConfig(
{ command, mode },
{ nominatimStorageDir } = {},
) {
const loaded = loadEnv(mode, root, '');
for (const [key, value] of Object.entries(loaded)) {
if (process.env[key] === undefined) process.env[key] = value;
}
const tools = standaloneVoiceTools();
return createBrowserViteConfig({
plugins: [
...localProviderPlugins({ realtime: { tools: standaloneVoiceTools() } }),
...localProviderPlugins({
realtime: { tools },
gemini: { tools },
nominatimStorageDir,
}),
localMcpPlugin(),
googleTokenUnavailablePlugin(),
apiNotFoundPlugin(),
unwatchedRuntimeDirsPlugin(),
],
googleApiKey: process.env.GOOGLE_MAPS_API_KEY,
cesiumToken: process.env.CESIUM_ION_TOKEN,
photonUrl: process.env.PHOTON_URL,
mapillaryToken: process.env.MAPILLARY_CLIENT_TOKEN,
host: process.env.HOST,
port: process.env.PORT,
allowedHosts: resolveAllowedHosts(process.env.GEV_ALLOWED_HOSTS),
command,
});
});
}
export default defineConfig(standaloneConfig);