mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Merge branch 'main' into feat/lithuania-cctv
This commit is contained in:
Sameh Khamis committed
commit
b4d852587a
1150 files changed
+212712
-7996
No files matched your search
@@ -1,7 +1,9 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
|
||||
*
|
||||
* Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. When upstream
|
||||
* Serves GET /api/military from https://api.adsb.lol/v2/mil. When upstream
|
||||
* fails — a thrown fetch OR a non-OK status such as 429 — the proxy serves its
|
||||
* cached body as STALE instead of relaying the failure, and it backs off from
|
||||
* upstream for the Retry-After period (bounded) so a rate limit is never
|
||||
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
|
||||
let _cooldownStatus = 0;
|
||||
/** Response cache TTL (ms). */
|
||||
const CACHE_MS = 12000;
|
||||
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
|
||||
const UPSTREAM_TIMEOUT_MS = 12000;
|
||||
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
|
||||
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
|
||||
const RATE_LIMIT_COOLDOWN_MS = 30000;
|
||||
/** Cooldown after a 5xx (ms). */
|
||||
@@ -55,10 +61,11 @@ export function adsbLolProxy() {
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-ADS-B-Cache': cacheStatus,
|
||||
'X-Feed-Source': 'adsb.lol',
|
||||
'X-Feed-Cache': cacheStatus,
|
||||
...(['HIT', 'STALE'].includes(cacheStatus)
|
||||
? {
|
||||
'X-ADS-B-Cache-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)),
|
||||
'X-Feed-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)),
|
||||
}
|
||||
: {}),
|
||||
...extra,
|
||||
@@ -67,8 +74,11 @@ export function adsbLolProxy() {
|
||||
}
|
||||
|
||||
const installMiddleware = (server) => {
|
||||
server.middlewares.use('/api/adsblol/mil', async (req, res) => {
|
||||
server.middlewares.use('/api/military', async (req, res, next) => {
|
||||
try {
|
||||
// This mount also matches its /track route, served by its own handler.
|
||||
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
|
||||
return next();
|
||||
const now = Date.now();
|
||||
if (_cache && now - _cacheAt < CACHE_MS) {
|
||||
serve(res, 200, _cache, 'HIT');
|
||||
@@ -94,11 +104,17 @@ export function adsbLolProxy() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
@@ -125,7 +141,11 @@ export function adsbLolProxy() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
serve(
|
||||
res,
|
||||
upstream.status,
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
|
||||
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
/**
|
||||
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
|
||||
* airports) and hex → aircraft type/registration. Free community API — cached
|
||||
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
|
||||
kind === 'route'
|
||||
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
|
||||
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
|
||||
const signal = AbortSignal.timeout(8000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (res.ok) {
|
||||
const payload = await readResponseJsonCapped(
|
||||
res,
|
||||
ADSBDB_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const data =
|
||||
kind === 'route'
|
||||
? parseRoute(await res.json())
|
||||
: parseAircraft(await res.json());
|
||||
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
|
||||
store[key] = { at: Date.now(), data }; // data may be null — negative cache
|
||||
dirty = true;
|
||||
return data;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
import { requiredFiniteQueryNumber } from '../common/query.js';
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -25,6 +26,73 @@ let _openskyCacheMeta = null;
|
||||
let _openskyCacheSourceEpochMs = null;
|
||||
/** TTL for the OpenSky response cache (ms). */
|
||||
const OPENSKY_CACHE_MS = 9000;
|
||||
/** Per-attempt limit for the global snapshot; it normally arrives in ~1.5 s. */
|
||||
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
|
||||
const OPENSKY_STATES_URL =
|
||||
'https://opensky-network.org/api/states/all?extended=1';
|
||||
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
|
||||
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
|
||||
* 30 s … 30 min, or 2 min when the header is absent.
|
||||
* @param {Response} upstream Rate-limited response.
|
||||
* @returns {number} Cooldown (ms).
|
||||
*/
|
||||
function openSkyRateLimitCooldownMs(upstream) {
|
||||
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
|
||||
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
|
||||
const retryAfterSec =
|
||||
header == null || header.trim() === '' ? NaN : Number(header);
|
||||
return Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and read the global snapshot, giving each attempt
|
||||
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
|
||||
* the connection fails. An oversized body is not retried: it would fail again
|
||||
* and spend more credits. A second failure throws, which the proxy answers
|
||||
* from its stale cache or the regional fallback.
|
||||
* @param {Record<string, string>} headers Request headers.
|
||||
* @returns {Promise<{upstream: Response, body: string}>}
|
||||
*/
|
||||
async function fetchOpenSkyStates(headers) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
|
||||
if (upstream.status === 429) {
|
||||
// Start the cooldown from the headers and drop the rejected body
|
||||
// unread, so a stalled or oversized 429 body cannot skip it.
|
||||
_openskyCooldownUntil =
|
||||
Date.now() + openSkyRateLimitCooldownMs(upstream);
|
||||
void upstream.body?.cancel().catch(() => {});
|
||||
return {
|
||||
upstream,
|
||||
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
|
||||
};
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
OPENSKY_STATES_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
return { upstream, body };
|
||||
} catch (error) {
|
||||
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
|
||||
console.warn(
|
||||
'[OpenSky] global snapshot attempt failed, retrying:',
|
||||
error?.message || error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// --- OpenSky credit governor (field-test fix 2026-07-06) -------------------
|
||||
// The global /states/all this proxy fetches costs 4 CREDITS per call against
|
||||
// OpenSky's ~4000/day authenticated budget — a day with the app open burned
|
||||
@@ -107,6 +175,7 @@ export async function getOpenSkyToken() {
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
|
||||
body: `grant_type=client_credentials&client_id=${encodeURIComponent(clientId)}&client_secret=${encodeURIComponent(clientSecret)}`,
|
||||
},
|
||||
);
|
||||
@@ -349,8 +418,12 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
|
||||
*/
|
||||
export function openSkyProxy() {
|
||||
const installMiddleware = (server) => {
|
||||
server.middlewares.use('/api/opensky', async (req, res) => {
|
||||
server.middlewares.use('/api/flights', async (req, res, next) => {
|
||||
try {
|
||||
// This mount also matches its /track route, served by its own handler.
|
||||
if (new URL(req.url || '/', 'http://localhost').pathname !== '/')
|
||||
return next();
|
||||
res.setHeader('X-Feed-Source', 'OpenSky Network');
|
||||
const requestedMode = normalizeOpenSkyAuthMode(
|
||||
process.env.OPENSKY_AUTH_MODE,
|
||||
);
|
||||
@@ -470,10 +543,7 @@ export function openSkyProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
let upstream = await fetch(
|
||||
'https://opensky-network.org/api/states/all?extended=1',
|
||||
{ headers },
|
||||
);
|
||||
let { upstream, body } = await fetchOpenSkyStates(headers);
|
||||
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
|
||||
if (
|
||||
(upstream.status === 401 || upstream.status === 403) &&
|
||||
@@ -485,15 +555,11 @@ export function openSkyProxy() {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
|
||||
};
|
||||
upstream = await fetch(
|
||||
'https://opensky-network.org/api/states/all?extended=1',
|
||||
{ headers: retryHeaders },
|
||||
);
|
||||
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_rejected_fallback_basic';
|
||||
}
|
||||
|
||||
let body = await upstream.text();
|
||||
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
|
||||
if (
|
||||
upstream.ok &&
|
||||
@@ -516,19 +582,9 @@ export function openSkyProxy() {
|
||||
}
|
||||
if (upstream.status === 429) {
|
||||
reason = 'rate_limited';
|
||||
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
|
||||
// 2 min when the header is absent) — no upstream attempts until then.
|
||||
const retryAfterSec = Number(
|
||||
upstream.headers.get('x-rate-limit-retry-after-seconds'),
|
||||
);
|
||||
const cooldownMs = Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
_openskyCooldownUntil = now + cooldownMs;
|
||||
// Credit governor: fetchOpenSkyStates already started the cooldown
|
||||
// from OpenSky's retry-after — no upstream attempts until then.
|
||||
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
|
||||
// Serve the last-good body instead of the 429 when we have one —
|
||||
// the layer keeps rendering (STALE-cued) instead of dying.
|
||||
if (_openskyCacheBody && _openskyCacheStatus === 200) {
|
||||
|
||||
@@ -3,11 +3,11 @@ import { readCappedResponseText } from '../common/http.js';
|
||||
/**
|
||||
* Vite plugin: aircraft track-history backfill proxies (PRD WS-F F1/F2).
|
||||
*
|
||||
* /api/opensky-track?icao24=<hex6> — OpenSky GET /tracks/all (experimental;
|
||||
* /api/flights/track?icao24=<hex6> — OpenSky GET /tracks/all (experimental;
|
||||
* own credit bucket, 4 credits per call on the free tier). OAuth via the
|
||||
* shared coalesced token. 60s per-icao cache; 404/429 forwarded so the
|
||||
* client can fall back to its accumulated trail silently.
|
||||
* /api/adsblol/trace?hex=<hex> — adsb.lol tar1090 readsb trace
|
||||
* /api/military/track?hex=<hex> — adsb.lol tar1090 readsb trace
|
||||
* (undocumented but live; no browser CORS, hence this proxy). Up to ~24h
|
||||
* of real history per aircraft. Treat as best-effort; data is ODbL —
|
||||
* credit "adsb.lol (ODbL)" in the UI.
|
||||
@@ -45,7 +45,9 @@ export function trackBackfillProxies() {
|
||||
RESPONSE_CAP_BYTES,
|
||||
);
|
||||
let body;
|
||||
let status = upstream.status;
|
||||
if (tooLarge) {
|
||||
status = 502;
|
||||
body = JSON.stringify({ error: 'Upstream track response too large' });
|
||||
} else if (!upstream.ok) {
|
||||
// Sanitize upstream error surface; status code is signal enough
|
||||
@@ -53,15 +55,17 @@ export function trackBackfillProxies() {
|
||||
} else {
|
||||
body = text;
|
||||
}
|
||||
cachePut(key, { at: Date.now(), status: upstream.status, body });
|
||||
res.statusCode = upstream.status;
|
||||
// Cache the 502 like any upstream error: /tracks/all costs OpenSky
|
||||
// credits, so a retry inside the window must not re-download the body.
|
||||
cachePut(key, { at: Date.now(), status, body });
|
||||
res.statusCode = status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/opensky-track', async (req, res) => {
|
||||
middlewares.use('/api/flights/track', async (req, res) => {
|
||||
try {
|
||||
const incoming = new URL(req.url || '', 'http://localhost');
|
||||
const icao24 = String(incoming.searchParams.get('icao24') || '')
|
||||
@@ -89,7 +93,7 @@ export function trackBackfillProxies() {
|
||||
}
|
||||
});
|
||||
|
||||
middlewares.use('/api/adsblol/trace', async (req, res) => {
|
||||
middlewares.use('/api/military/track', async (req, res) => {
|
||||
try {
|
||||
const incoming = new URL(req.url || '', 'http://localhost');
|
||||
const hex = String(incoming.searchParams.get('hex') || '')
|
||||
|
||||
+134
-4
@@ -17,6 +17,7 @@ import {
|
||||
CCTV_MAX_SOURCES_CEILING,
|
||||
} from './cctv/constants.js';
|
||||
import { sanitizeCctvRangeHeader } from './cctv/range.js';
|
||||
import { createHlsPuller } from './cctv/stream.js';
|
||||
import { googleServerApiKey } from './places/google-key.js';
|
||||
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
|
||||
/**
|
||||
@@ -40,6 +41,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
* CCTV_MAX_SOURCES ceiling so health/status observability is never evicted
|
||||
* for any catalog the proxy can actually serve. */
|
||||
const HEALTH_MAX_ENTRIES = CCTV_MAX_SOURCES_CEILING;
|
||||
/** Live HLS strategies (see ./cctv/stream.js). Shared across dev and preview. */
|
||||
const puller = createHlsPuller();
|
||||
|
||||
/** Update the health entry for a camera, evicting the oldest entry if at capacity. */
|
||||
const setHealth = (cameraId, patch) => {
|
||||
@@ -127,6 +130,9 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
};
|
||||
|
||||
const installMiddleware = (server) => {
|
||||
server.httpServer?.on('close', () => {
|
||||
puller.shutdown();
|
||||
});
|
||||
server.middlewares.use('/api/cctv', async (req, res) => {
|
||||
try {
|
||||
const sources = await getCctvSources();
|
||||
@@ -137,8 +143,10 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
|
||||
if (url.pathname === '/sources') {
|
||||
const body = {
|
||||
trimmedPacks: getCctvSources.trimmedPacks?.() ?? [],
|
||||
sources: sources.map((source) => ({
|
||||
id: source.id,
|
||||
pack: source.pack,
|
||||
name: source.name,
|
||||
city: source.city,
|
||||
cityId: source.cityId,
|
||||
@@ -194,12 +202,124 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
|
||||
if (url.pathname.startsWith('/media/')) {
|
||||
const cameraId =
|
||||
decodeURIComponent(url.pathname.replace('/media/', '').trim()) ||
|
||||
'camera';
|
||||
const match = /^\/media\/([^/]+)(?:\/(seg_(\d+)\.ts))?$/.exec(
|
||||
url.pathname,
|
||||
);
|
||||
if (!match) {
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
const cameraId = decodeURIComponent(match[1]);
|
||||
const source = sourceById.get(cameraId);
|
||||
const mediaUrl = source?.url || '';
|
||||
const feedType = normalizeFeedType(source?.feedType || 'image');
|
||||
const leaseId = url.searchParams.get('lease');
|
||||
if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) {
|
||||
res.writeHead(400);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (req.method === 'DELETE') {
|
||||
if (leaseId) puller.release(cameraId, leaseId);
|
||||
res.writeHead(204);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (req.method !== 'GET') {
|
||||
res.writeHead(405);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (feedType === 'hls') {
|
||||
if (
|
||||
!/^https?:\/\//i.test(mediaUrl) ||
|
||||
!/\.m3u8(?:\?|$)/i.test(mediaUrl)
|
||||
) {
|
||||
res.writeHead(503, { 'Content-Type': 'application/json' });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error:
|
||||
'This stream requires an unsupported transport; use the frame fallback',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (match[2]) {
|
||||
const body = puller.getSegment(
|
||||
cameraId,
|
||||
url.searchParams.get('session'),
|
||||
Number(match[3]),
|
||||
leaseId,
|
||||
);
|
||||
res.writeHead(body ? 200 : 404, {
|
||||
'Content-Type': 'video/mp2t',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(body || undefined);
|
||||
return;
|
||||
}
|
||||
const downstream = watchDownstreamClose(res);
|
||||
let entry;
|
||||
const cancelPending = () => {
|
||||
if (entry) puller.release(cameraId, leaseId);
|
||||
};
|
||||
try {
|
||||
entry = await puller.ensure(cameraId, mediaUrl, leaseId);
|
||||
if (downstream.closed) {
|
||||
cancelPending();
|
||||
return;
|
||||
}
|
||||
downstream.signal.addEventListener('abort', cancelPending, {
|
||||
once: true,
|
||||
});
|
||||
if (!(await puller.waitReady(entry, downstream.signal)))
|
||||
throw new Error('Stream unavailable');
|
||||
const playlist = await puller.buildPlaylist(
|
||||
entry,
|
||||
cameraId,
|
||||
leaseId,
|
||||
);
|
||||
if (downstream.closed) return;
|
||||
if (!playlist) throw new Error('Stream unavailable');
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
sourceKind: 'live',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: 'Live HLS connected',
|
||||
});
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/vnd.apple.mpegurl',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-CCTV-Source': 'hls-pull',
|
||||
'X-CCTV-Session': entry.token,
|
||||
});
|
||||
res.end(playlist);
|
||||
} catch {
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
sourceKind: 'fallback',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: 'Live HLS unavailable',
|
||||
});
|
||||
if (!downstream.closed) {
|
||||
res.writeHead(503, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'Retry-After': '2',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Live stream unavailable' }));
|
||||
}
|
||||
} finally {
|
||||
downstream.signal.removeEventListener('abort', cancelPending);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (match[2] || req.method !== 'GET') {
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!mediaUrl || !/^https?:\/\//i.test(mediaUrl)) {
|
||||
setHealth(cameraId, {
|
||||
@@ -308,11 +428,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
const timedOut =
|
||||
error?.name === 'AbortError' || error?.name === 'TimeoutError';
|
||||
// GET /api/cctv/health serializes `message`, and the CCTV panel
|
||||
// renders it as a status label, so the raw error would leave the
|
||||
// server by a different door than the sanitized body below and
|
||||
// land on screen.
|
||||
//
|
||||
// The log line drops the text too, unlike the catch at the bottom
|
||||
// of this file: a media-fetch failure names the camera's upstream
|
||||
// host, and these lines get pasted into issues. The status codes
|
||||
// below carry the diagnosis — 504 for a timeout, 502 otherwise.
|
||||
console.warn('[CCTV Proxy] media fetch failed');
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
sourceKind: 'upstream',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: error?.message || 'Media fetch failed',
|
||||
message: 'Media fetch failed',
|
||||
});
|
||||
res.writeHead(timedOut ? 504 : 502, {
|
||||
'Content-Type': 'application/json',
|
||||
|
||||
@@ -3,7 +3,7 @@ import path from 'node:path';
|
||||
import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
|
||||
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
|
||||
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
|
||||
import { normalizeSourceItem } from './normalize.js';
|
||||
import { cameraRegion, normalizeSourceItem } from './normalize.js';
|
||||
import { loadLithuaniaSources } from './lithuania.js';
|
||||
import {
|
||||
loadAustinSourcesFromOpenData,
|
||||
@@ -18,24 +18,25 @@ import {
|
||||
loadWarendorfSourcesFromCatalog,
|
||||
loadNswSourcesFromOpenData,
|
||||
loadCalgarySourcesFromOpenData,
|
||||
loadDelDOTSourcesFromOpenData,
|
||||
loadVegvesenSourcesFromOpenData,
|
||||
loadNycSourcesFromOpenData,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
|
||||
/** Env opt-in: only "1" enables. For packs whose feed access needs an
|
||||
* operator agreement, so an unset flag never reaches the upstream. */
|
||||
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
|
||||
|
||||
/**
|
||||
* Live open-data packs, in merge order. Adding a region is one entry here
|
||||
* plus its loader in sources.js; the catalog cap is shared across entries
|
||||
* round-robin (cap.js), so a new pack never silently evicts an older one.
|
||||
* Each pack fails independently (allSettled) and is gated by its own env
|
||||
* kill switch.
|
||||
* kill switch, or by an opt-in flag where access needs an agreement.
|
||||
*/
|
||||
const LIVE_PACKS = [
|
||||
{
|
||||
name: 'lithuania',
|
||||
enabled: () => envEnabled('CCTV_LITHUANIA_ENABLED'),
|
||||
load: loadLithuaniaSources,
|
||||
},
|
||||
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
|
||||
{
|
||||
name: 'caltrans',
|
||||
@@ -92,6 +93,28 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
|
||||
load: loadCalgarySourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'deldot',
|
||||
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
|
||||
load: loadDelDOTSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'vegvesen',
|
||||
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
|
||||
load: loadVegvesenSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
|
||||
// off unless the operator opts in (see DATA_SOURCES.md).
|
||||
name: 'nyc',
|
||||
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
|
||||
load: loadNycSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'lithuania',
|
||||
enabled: () => envEnabled('CCTV_LITHUANIA_ENABLED'),
|
||||
load: loadLithuaniaSources,
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
|
||||
@@ -143,6 +166,9 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
/** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
|
||||
* callers so a post-TTL burst launches ONE refetch, not one per request. */
|
||||
let _cctvSourceInflight = null;
|
||||
/** @type {Array<{pack: string, available: number, served: number}>} Packs
|
||||
* that serve fewer cameras than they offer. */
|
||||
let _cctvTrimmedPacks = [];
|
||||
|
||||
/**
|
||||
* Assemble and cache the merged CCTV source list.
|
||||
@@ -209,8 +235,12 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
name,
|
||||
sources: items
|
||||
.filter((item) => item && typeof item === 'object')
|
||||
.map((item) => normalizeSourceItem(item))
|
||||
.map((item) => ({ ...normalizeSourceItem(item), pack: name }))
|
||||
.filter((item) => item.id),
|
||||
available: Number.isFinite(items.available)
|
||||
? items.available
|
||||
: items.length,
|
||||
region: items.region ?? null,
|
||||
});
|
||||
const packs = [
|
||||
...LIVE_PACKS.map((pack, index) =>
|
||||
@@ -234,6 +264,16 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
loadGroundHeights(sourceRoot),
|
||||
);
|
||||
const trimmed = allocation.packs.filter((pack) => pack.kept < pack.offered);
|
||||
// Each pack's own nearest-first cap and the shared catalog cap both trim.
|
||||
const trimmedPacks = packs
|
||||
.map((pack, index) => ({
|
||||
pack: pack.name,
|
||||
available: Math.max(pack.available, allocation.packs[index].offered),
|
||||
served: allocation.packs[index].kept,
|
||||
// Where the pack's cameras are, including those not served.
|
||||
region: pack.region ?? cameraRegion(pack.sources),
|
||||
}))
|
||||
.filter((pack) => pack.served < pack.available);
|
||||
if (trimmed.length) {
|
||||
const detail = trimmed
|
||||
.map((pack) => `${pack.name} ${pack.kept}/${pack.offered}`)
|
||||
@@ -244,6 +284,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
if (capped.length > 0 || _cctvSourceCache.length === 0) {
|
||||
_cctvSourceCache = capped;
|
||||
_cctvTrimmedPacks = trimmedPacks;
|
||||
} else {
|
||||
// Every source came back empty (all live packs timed out / upstream outage)
|
||||
// but a good catalog is already cached — serve it stale rather than blanking
|
||||
@@ -257,5 +298,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
|
||||
return _cctvSourceCache;
|
||||
}
|
||||
|
||||
/** Packs serving fewer cameras than they offer, as of the last refresh. */
|
||||
getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
|
||||
return getCctvSources;
|
||||
}
|
||||
@@ -236,6 +236,75 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 };
|
||||
* body cannot be buffered without limit. */
|
||||
export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
|
||||
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
|
||||
* country (~900 cameras). The raw DATEX node needs an account; this view does
|
||||
* not. Frames are stills on the agency's camera host. */
|
||||
export const DEFAULT_VEGVESEN_CCTV_URL =
|
||||
'https://ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items?f=application/geo%2Bjson&limit=5000';
|
||||
/** The only origin Vegvesen camera frames may come from. */
|
||||
export const VEGVESEN_IMAGE_ORIGIN =
|
||||
'https://kamera.atlas.vegvesen.no/api/images/';
|
||||
/** Live HLS for the ~140 cameras that publish video; only this exact
|
||||
* per-camera manifest path is registered. */
|
||||
export const VEGVESEN_VIDEO_URL = (cameraId) =>
|
||||
`https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`;
|
||||
/** Above the ~850 working cameras so a default install keeps them all, while
|
||||
* staying close to them: the per-pack defaults are meant to sum to no more
|
||||
* than DEFAULT_CCTV_MAX_SOURCES. */
|
||||
export const DEFAULT_VEGVESEN_MAX_SOURCES = 900;
|
||||
/** Hard ceiling on the catalog body. The whole country is ~800 KB. */
|
||||
export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024;
|
||||
/** Prioritization anchors: the largest cities from south to north, so a cap
|
||||
* keeps national coverage rather than just the capital region. */
|
||||
export const NORWAY_ANCHORS = [
|
||||
{ lat: 59.9139, lon: 10.7522 }, // Oslo
|
||||
{ lat: 60.3913, lon: 5.3221 }, // Bergen
|
||||
{ lat: 58.97, lon: 5.7331 }, // Stavanger
|
||||
{ lat: 63.4305, lon: 10.3951 }, // Trondheim
|
||||
{ lat: 58.1467, lon: 7.9956 }, // Kristiansand
|
||||
{ lat: 67.2804, lon: 14.4049 }, // Bodø
|
||||
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
|
||||
];
|
||||
|
||||
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
|
||||
* five boroughs (~980 cameras). Frames are stills synthesized from each
|
||||
* camera UUID on the city's webcam host — never taken from the payload. */
|
||||
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
|
||||
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
|
||||
* is ignored; the registered path is always this origin plus a validated
|
||||
* UUID plus `/image`. */
|
||||
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
|
||||
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
|
||||
export const NYC_CAMERA_ID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
export const DEFAULT_NYC_MAX_SOURCES = 250;
|
||||
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
|
||||
* this only exists so an upstream that streams an unbounded body cannot be
|
||||
* buffered without limit. */
|
||||
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
|
||||
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
|
||||
export const NYC_GROUND_ELEVATION_M = 10;
|
||||
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
|
||||
* rather than Midtown alone. */
|
||||
export const NYC_ANCHORS = [
|
||||
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
|
||||
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
|
||||
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
|
||||
{ lat: 40.7282, lon: -73.7949 }, // Queens
|
||||
{ lat: 40.8448, lon: -73.8648 }, // Bronx
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
|
||||
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
|
||||
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
|
||||
export const DELDOT_ANCHORS = [
|
||||
{ lat: 39.7459, lon: -75.5466 }, // Wilmington (New Castle)
|
||||
{ lat: 39.1582, lon: -75.5244 }, // Dover (Kent)
|
||||
{ lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex)
|
||||
];
|
||||
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */
|
||||
export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
|
||||
/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one
|
||||
@@ -253,5 +322,10 @@ export const CCTV_FRAME_MAX_BODY_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
/** Deadline for upstream response headers; live bodies keep streaming afterward. */
|
||||
export const CCTV_MEDIA_FETCH_TIMEOUT_MS = 15 * 1000;
|
||||
/** Silence a live body may carry before the relay gives up on it. Twice the
|
||||
* header deadline, because a camera that is merely slow between frames is far
|
||||
* more common than one that has died mid-stream, and a viewer would rather
|
||||
* wait than be dropped. */
|
||||
export const CCTV_MEDIA_IDLE_TIMEOUT_MS = 30 * 1000;
|
||||
/** Declared size ceiling for fixed media responses. */
|
||||
export const CCTV_MEDIA_MAX_BODY_BYTES = 64 * 1024 * 1024;
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
CCTV_FRAME_FETCH_TIMEOUT_MS,
|
||||
CCTV_FRAME_MAX_BODY_BYTES,
|
||||
CCTV_MEDIA_FETCH_TIMEOUT_MS,
|
||||
CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
CCTV_MEDIA_MAX_BODY_BYTES,
|
||||
NSW_IMAGE_ORIGIN,
|
||||
NSW_IMAGE_USER_AGENT,
|
||||
@@ -103,11 +104,17 @@ export function toReadable(body) {
|
||||
* @param {Response} upstream - fetch() Response object.
|
||||
* @param {object} [opts]
|
||||
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
|
||||
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
|
||||
* allowed between upstream chunks before the stream is released. Injectable
|
||||
* only to keep the deadline unit-testable.
|
||||
*/
|
||||
export async function proxyMediaResponse(
|
||||
res,
|
||||
upstream,
|
||||
{ sourceHeader = 'upstream' } = {},
|
||||
{
|
||||
sourceHeader = 'upstream',
|
||||
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
} = {},
|
||||
) {
|
||||
const contentType =
|
||||
upstream.headers.get('content-type') || 'application/octet-stream';
|
||||
@@ -153,7 +160,20 @@ export async function proxyMediaResponse(
|
||||
return;
|
||||
}
|
||||
|
||||
// The header deadline only covers the wait for a response line. Past that an
|
||||
// upstream can hold the connection open and send nothing at all, and the
|
||||
// relay would wait on it for as long as the camera host cared to. The
|
||||
// deadline below measures the gap between upstream chunks rather than the
|
||||
// life of the stream, so a feed that keeps delivering keeps its connection.
|
||||
let idleTimer = null;
|
||||
const clearIdleDeadline = () => {
|
||||
if (!idleTimer) return;
|
||||
clearTimeout(idleTimer);
|
||||
idleTimer = null;
|
||||
};
|
||||
|
||||
stream.on('error', () => {
|
||||
clearIdleDeadline();
|
||||
if (!res.writableEnded) res.end();
|
||||
});
|
||||
|
||||
@@ -164,6 +184,7 @@ export async function proxyMediaResponse(
|
||||
const releaseUpstream = () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
clearIdleDeadline();
|
||||
stream.unpipe(res);
|
||||
// Destroying the Node stream cancels the web body it wraps; the direct
|
||||
// cancel covers a body that was never wrapped, and rejects harmlessly when
|
||||
@@ -176,14 +197,37 @@ export async function proxyMediaResponse(
|
||||
/* already closed */
|
||||
}
|
||||
};
|
||||
const armIdleDeadline = () => {
|
||||
clearIdleDeadline();
|
||||
idleTimer = setTimeout(onIdleDeadline, idleTimeoutMs);
|
||||
idleTimer.unref?.();
|
||||
};
|
||||
const onIdleDeadline = () => {
|
||||
// A viewer who cannot keep up pauses the pipe, and no upstream bytes arrive
|
||||
// while it is paused. That is a slow client rather than a dead camera, so
|
||||
// it gets the deadline again instead of a teardown.
|
||||
if (res.writableNeedDrain) {
|
||||
armIdleDeadline();
|
||||
return;
|
||||
}
|
||||
releaseUpstream();
|
||||
if (!res.writableEnded) res.end();
|
||||
};
|
||||
res.once('close', () => {
|
||||
clearIdleDeadline();
|
||||
if (!res.writableEnded) releaseUpstream();
|
||||
});
|
||||
res.once('error', releaseUpstream);
|
||||
stream.once('end', () => {
|
||||
clearIdleDeadline();
|
||||
released = true;
|
||||
});
|
||||
armIdleDeadline();
|
||||
stream.pipe(res);
|
||||
// Attached after the pipe because a data listener resumes the stream, and
|
||||
// flowing before the destination is attached would spill chunks nobody
|
||||
// forwards. Only bytes from upstream renew the deadline.
|
||||
stream.on('data', armIdleDeadline);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -367,6 +367,28 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** New York City's five boroughs, with slack for the harbor and city line. */
|
||||
export function isLikelyNycCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 40.45 &&
|
||||
lat <= 40.95 &&
|
||||
lon >= -74.3 &&
|
||||
lon <= -73.65
|
||||
);
|
||||
}
|
||||
|
||||
/** Mainland Norway, with slack for the coast and the Finnmark border. */
|
||||
export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 57.9 &&
|
||||
lat <= 71.3 &&
|
||||
lon >= 4.4 &&
|
||||
lon <= 31.3
|
||||
);
|
||||
}
|
||||
|
||||
export function isLikelyFinlandCoordinate(lat, lon) {
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
|
||||
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
|
||||
@@ -470,7 +492,31 @@ export function prioritizeSources(cameras, maxCount, anchors) {
|
||||
return a.idx - b.idx;
|
||||
});
|
||||
|
||||
return scored.slice(0, cap).map((entry) => entry.camera);
|
||||
const kept = scored.slice(0, cap).map((entry) => entry.camera);
|
||||
// How many cameras the pack offered before trimming, and where, for
|
||||
// coverage reports.
|
||||
Object.defineProperty(kept, 'available', { value: list.length });
|
||||
Object.defineProperty(kept, 'region', { value: cameraRegion(list) });
|
||||
return kept;
|
||||
}
|
||||
|
||||
/** The bounding box of cameras with coordinates, or null. */
|
||||
export function cameraRegion(cameras) {
|
||||
let region = null;
|
||||
for (const camera of cameras || []) {
|
||||
const lat = Number(camera?.lat);
|
||||
const lon = Number(camera?.lon);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
region = region
|
||||
? {
|
||||
west: Math.min(region.west, lon),
|
||||
south: Math.min(region.south, lat),
|
||||
east: Math.max(region.east, lon),
|
||||
north: Math.max(region.north, lat),
|
||||
}
|
||||
: { west: lon, south: lat, east: lon, north: lat };
|
||||
}
|
||||
return region;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -56,6 +56,22 @@ import {
|
||||
CALGARY_DOWNTOWN,
|
||||
CALGARY_MAX_CATALOG_BYTES,
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
DELDOT_CCTV_URL,
|
||||
DEFAULT_DELDOT_MAX_SOURCES,
|
||||
DELDOT_ANCHORS,
|
||||
DEFAULT_VEGVESEN_CCTV_URL,
|
||||
VEGVESEN_IMAGE_ORIGIN,
|
||||
VEGVESEN_VIDEO_URL,
|
||||
DEFAULT_VEGVESEN_MAX_SOURCES,
|
||||
VEGVESEN_MAX_CATALOG_BYTES,
|
||||
NORWAY_ANCHORS,
|
||||
NYC_CAMERAS_URL,
|
||||
NYC_IMAGE_URL,
|
||||
NYC_CAMERA_ID_RE,
|
||||
DEFAULT_NYC_MAX_SOURCES,
|
||||
NYC_MAX_CATALOG_BYTES,
|
||||
NYC_GROUND_ELEVATION_M,
|
||||
NYC_ANCHORS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -73,6 +89,8 @@ import {
|
||||
isLikelyTexasCoordinate,
|
||||
isLikelyNswCoordinate,
|
||||
isLikelyCalgaryCoordinate,
|
||||
isLikelyNycCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
prioritizeSources,
|
||||
@@ -1594,3 +1612,407 @@ export async function loadCalgarySourcesFromOpenData() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadDelDOTSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(DELDOT_CCTV_URL, {
|
||||
headers: { Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
redirect: 'error',
|
||||
});
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] DelDOT source download failed:', resp.status);
|
||||
return [];
|
||||
}
|
||||
const payload = await readResponseJsonCapped(resp, 2 * 1024 * 1024);
|
||||
const rows = Array.isArray(payload?.videoCameras)
|
||||
? payload.videoCameras
|
||||
: [];
|
||||
if (!rows.length) return [];
|
||||
|
||||
const cameras = [];
|
||||
for (const row of rows) {
|
||||
if (String(row?.status).toLowerCase() !== 'active') continue;
|
||||
const lat = toFiniteNumber(row?.lat);
|
||||
const lon = toFiniteNumber(row?.lon);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
// Delaware bounding box — a bad upstream coord can't place a camera out of state.
|
||||
if (lat < 38.4 || lat > 39.9 || lon < -75.85 || lon > -75.0) continue;
|
||||
|
||||
// Prefer the catalog's HTTPS HLS transport; no subprocess is required.
|
||||
let stream;
|
||||
try {
|
||||
stream = new URL(String(row?.urls?.m3u8s || ''));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
stream.origin !== 'https://video.deldot.gov' ||
|
||||
stream.username ||
|
||||
stream.password ||
|
||||
!/^\/live\/[A-Za-z0-9_.-]+\/playlist\.m3u8$/.test(stream.pathname)
|
||||
)
|
||||
continue;
|
||||
const streamUrl = stream.href;
|
||||
|
||||
const id = String(row?.id || '').trim();
|
||||
if (!/^[A-Za-z0-9_-]{1,80}$/.test(id)) continue;
|
||||
const cameraId = `deldot-${id.toLowerCase()}`;
|
||||
const title = String(row?.title || '').trim();
|
||||
|
||||
// Title carries an unambiguous travel token for ~1 in 5 cameras
|
||||
// ("US 13 SB @ ..."). directionToHeading (allowBare=false) matches only
|
||||
// NB/SB/EB/WB, never a bare "W"/"N" street prefix — "W NORTH ST" yields
|
||||
// no heading and falls back.
|
||||
const heading = directionToHeading(title);
|
||||
const hasHeading = Number.isFinite(heading);
|
||||
|
||||
cameras.push({
|
||||
id: cameraId,
|
||||
name: title || `DelDOT ${id}`,
|
||||
city: String(row?.county ? `${row.county} County` : 'Delaware'),
|
||||
cityId: `deldot-${String(row?.county || 'de')
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, '-')}`,
|
||||
provider: 'DelDOT',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: hasHeading ? 'high' : 'low',
|
||||
// Fabricated RAW PRIOR poses (same personalities as Austin/Caltrans);
|
||||
// the client ground-snap + manual calibration own the truth.
|
||||
pitchDeg: hasHeading ? -24 : -18,
|
||||
fovDeg: hasHeading ? 56 : 44,
|
||||
rangeM: hasHeading ? 210 : 145,
|
||||
mountHeightM: hasHeading ? 10 : 8,
|
||||
groundElevationM: 10, // Estimated prior; client ground resolution owns placement.
|
||||
feedType: 'hls',
|
||||
url: streamUrl,
|
||||
snapshotUrl: '',
|
||||
sourceKind: 'deldot-open-data',
|
||||
license: 'Public DelDOT traffic camera',
|
||||
});
|
||||
}
|
||||
|
||||
const unique = Array.from(
|
||||
new Map(cameras.map((camera) => [camera.id, camera])).values(),
|
||||
);
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_DELDOT_MAX_SOURCES || DEFAULT_DELDOT_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_DELDOT_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(unique, maxCount, DELDOT_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded DelDOT camera sources: ${unique.length} Active (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] DelDOT source download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One Statens vegvesen `CctvSimple` feature -> one catalog source, or null.
|
||||
*
|
||||
* `orientationDescription` names the place the camera looks towards
|
||||
* ("Svelgen"), not a bearing, so it goes into the label and the heading uses
|
||||
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
|
||||
* Cameras the feed reports as faulty are dropped. Cameras that publish HLS
|
||||
* get live video, with the still as the snapshot fallback.
|
||||
*
|
||||
* @param {object} feature - GeoJSON feature from the OGC items response.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function vegvesenCameraToSource(feature) {
|
||||
if (!feature || typeof feature !== 'object') return null;
|
||||
const props = feature.properties;
|
||||
if (!props || typeof props !== 'object') return null;
|
||||
const availability = String(
|
||||
props['status.stillImageAvailability'] ?? '',
|
||||
).trim();
|
||||
if (availability && availability !== 'videoOrImagesAvailable') return null;
|
||||
|
||||
const coordinates = feature?.geometry?.coordinates;
|
||||
if (!Array.isArray(coordinates) || coordinates.length < 2) return null;
|
||||
const lon = toFiniteNumber(coordinates[0]);
|
||||
const lat = toFiniteNumber(coordinates[1]);
|
||||
if (!isLikelyNorwayCoordinate(lat, lon)) return null;
|
||||
|
||||
const rawId = String(props.cameraId ?? '').trim();
|
||||
if (!/^[A-Za-z0-9_-]{1,40}$/.test(rawId)) return null;
|
||||
let image;
|
||||
try {
|
||||
image = new URL(String(props.stillImageUrl ?? ''));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (image.username || image.password) return null;
|
||||
const imageUrl = image.href;
|
||||
if (imageUrl !== `${VEGVESEN_IMAGE_ORIGIN}${rawId}`) return null;
|
||||
|
||||
const cameraId = `no-vegvesen-${rawId.toLowerCase()}`;
|
||||
const place = String(props.description ?? '').trim() || `Kamera ${rawId}`;
|
||||
const towards = String(props.orientationDescription ?? '').trim();
|
||||
const road = String(props.roadNumber ?? '').trim();
|
||||
const name = towards && towards !== place ? `${place} → ${towards}` : place;
|
||||
// Live video when the camera publishes HLS on the agency's own host; the
|
||||
// still stays the snapshot fallback. CCTV_VEGVESEN_VIDEO=0 keeps stills only.
|
||||
const videoUrl =
|
||||
String(process.env.CCTV_VEGVESEN_VIDEO || '1').trim() !== '0' &&
|
||||
Number(props.videoServiceLevel) > 0 &&
|
||||
String(props.videoEncodingStandard ?? '').toLowerCase() === 'hls' &&
|
||||
String(props.videoUrl ?? '').trim() === VEGVESEN_VIDEO_URL(rawId)
|
||||
? VEGVESEN_VIDEO_URL(rawId)
|
||||
: '';
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name: road ? `${road} ${name}` : name,
|
||||
// One country-wide category in the camera picker, like Finland; the
|
||||
// place itself is already in the name.
|
||||
city: 'Norway',
|
||||
cityId: 'norway',
|
||||
provider: 'Statens vegvesen',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
// KNOWN LIMITATION: the feed's coordinates are 2D, so this is one flat
|
||||
// prior for the whole country, while cameras run from sea level to
|
||||
// mountain passes near 1,000 m (Haukelifjell, Sjonfjellet, Rugeldalen).
|
||||
// The point-height prior and the client's one-shot ground snap correct it
|
||||
// where they resolve; on a stack where neither does (no 3D tiles), a pass
|
||||
// camera stays hundreds of metres below the terrain, the same risk the
|
||||
// Caltrans pack documents. A per-camera height (e.g. Kartverket's keyless
|
||||
// point-height API) would remove it.
|
||||
groundElevationM: 150,
|
||||
feedType: videoUrl ? 'hls' : 'image',
|
||||
url: videoUrl || imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'vegvesen-datex',
|
||||
license:
|
||||
'Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen',
|
||||
code: cameraDisplayCode(place.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Statens vegvesen (Norway) road cameras from the keyless OGC API
|
||||
* Features view of the DATEX 3.1 CCTV table. Frames are stills on
|
||||
* kamera.atlas.vegvesen.no.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadVegvesenSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_VEGVESEN_URL || DEFAULT_VEGVESEN_CCTV_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/geo+json,application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Vegvesen catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Vegvesen camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
resp,
|
||||
VEGVESEN_MAX_CATALOG_BYTES,
|
||||
);
|
||||
const features = Array.isArray(payload?.features) ? payload.features : [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const feature of features) {
|
||||
const camera = vegvesenCameraToSource(feature);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_VEGVESEN_MAX_SOURCES || DEFAULT_VEGVESEN_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(1000, Math.floor(maxRaw)))
|
||||
: DEFAULT_VEGVESEN_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, NORWAY_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Vegvesen camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Vegvesen camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const NYC_BOROUGHS = new Set([
|
||||
'Manhattan',
|
||||
'Brooklyn',
|
||||
'Queens',
|
||||
'Bronx',
|
||||
'Staten Island',
|
||||
]);
|
||||
|
||||
/** NYC TMC publishes `isOnline` as the strings "true"/"false". */
|
||||
export function nycCameraIsOnline(value) {
|
||||
return (
|
||||
value === true ||
|
||||
String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase() === 'true'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one NYC camera: the intersection an operator recognises
|
||||
* ("Central Park West @ 86 St"). The catalog's `area` is the borough, not
|
||||
* a facing, and is not part of the name.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @param {string} cameraId - Derived stable id.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function nycCameraName(record, cameraId) {
|
||||
const name = String(record?.name ?? '').trim();
|
||||
if (name) return name;
|
||||
const short = String(cameraId || '')
|
||||
.replace(/^nyc-/, '')
|
||||
.slice(0, 8);
|
||||
return short ? `NYC Camera ${short}` : 'NYC Camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* One NYC DOT TMC camera row -> one catalog source, or null.
|
||||
*
|
||||
* Frame URLs are synthesized from a validated UUID on the pinned TMC image
|
||||
* origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer
|
||||
* the frame proxy off-host. There is no heading in the feed; headings use
|
||||
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function nycCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
if (!nycCameraIsOnline(record.isOnline)) return null;
|
||||
|
||||
const lat = toFiniteNumber(record.latitude);
|
||||
const lon = toFiniteNumber(record.longitude);
|
||||
if (!isLikelyNycCoordinate(lat, lon)) return null;
|
||||
|
||||
const uuid = String(record.id ?? '').trim();
|
||||
if (!NYC_CAMERA_ID_RE.test(uuid)) return null;
|
||||
const cameraId = `nyc-${uuid.toLowerCase()}`;
|
||||
const imageUrl = NYC_IMAGE_URL(uuid);
|
||||
const name = nycCameraName(record, cameraId);
|
||||
const area = String(record.area ?? '').trim();
|
||||
const city = NYC_BOROUGHS.has(area) ? area : 'New York';
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city,
|
||||
cityId: 'nyc',
|
||||
provider: 'NYC DOT Traffic Management Center',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: NYC_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'nyc-dot-tmc',
|
||||
license: 'Public NYC DOT Traffic Management Center traffic camera',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch NYC DOT Traffic Management Center cameras from the keyless
|
||||
* `webcams.nyctmc.org` catalog. Frames are stills on that same host,
|
||||
* registered from a validated UUID rather than from the payload URL.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadNycSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] NYC catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] NYC camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES);
|
||||
if (!Array.isArray(rows)) return [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = nycCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_NYC_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] NYC camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
/** Bounded, demand-owned HLS sessions. No subprocess, disk cache or idle sweep. */
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
export const HLS_LIMITS = Object.freeze({
|
||||
sessions: 2,
|
||||
leasesPerSession: 8,
|
||||
playlistBytes: 256 * 1024,
|
||||
segmentBytes: 4 * 1024 * 1024,
|
||||
sessionBytes: 24 * 1024 * 1024,
|
||||
segments: 12,
|
||||
pollMs: 2000,
|
||||
idleMs: 15000,
|
||||
timeoutMs: 10000,
|
||||
readyMs: 12000,
|
||||
});
|
||||
|
||||
/** Refuse redirects; count actual streamed bytes, including chunked responses. */
|
||||
export async function fetchHlsBytes(
|
||||
url,
|
||||
{
|
||||
fetchImpl = fetch,
|
||||
signal,
|
||||
maxBytes,
|
||||
timeoutMs = HLS_LIMITS.timeoutMs,
|
||||
} = {},
|
||||
) {
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort();
|
||||
if (signal?.aborted) controller.abort();
|
||||
else signal?.addEventListener('abort', abort, { once: true });
|
||||
const timer = setTimeout(abort, timeoutMs);
|
||||
let reader;
|
||||
let response;
|
||||
try {
|
||||
controller.signal.throwIfAborted();
|
||||
response = await fetchImpl(url, {
|
||||
redirect: 'error',
|
||||
signal: controller.signal,
|
||||
headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' },
|
||||
});
|
||||
if (
|
||||
!response.ok ||
|
||||
response.redirected ||
|
||||
(response.url && new URL(response.url).href !== new URL(url).href)
|
||||
)
|
||||
throw new Error('HLS upstream refused');
|
||||
if (Number(response.headers.get('content-length')) > maxBytes)
|
||||
throw new Error('HLS response too large');
|
||||
reader = response.body?.getReader?.();
|
||||
if (!reader) throw new Error('HLS response is not streamable');
|
||||
const chunks = [];
|
||||
let length = 0;
|
||||
for (;;) {
|
||||
controller.signal.throwIfAborted();
|
||||
const { done, value } = await reader.read();
|
||||
controller.signal.throwIfAborted();
|
||||
if (done) break;
|
||||
length += value.byteLength;
|
||||
if (length > maxBytes) throw new Error('HLS response too large');
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
return Buffer.concat(chunks, length);
|
||||
} finally {
|
||||
controller.abort();
|
||||
if (reader) await reader.cancel().catch(() => {});
|
||||
else await response?.body?.cancel().catch(() => {});
|
||||
reader?.releaseLock();
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', abort);
|
||||
}
|
||||
}
|
||||
|
||||
export function sameOriginHlsUrl(value, base) {
|
||||
const url = new URL(value, base);
|
||||
if (
|
||||
!['https:', 'http:'].includes(url.protocol) ||
|
||||
url.origin !== new URL(base).origin ||
|
||||
url.username ||
|
||||
url.password
|
||||
)
|
||||
throw new Error('HLS reference escapes registered origin');
|
||||
return url.href;
|
||||
}
|
||||
|
||||
/** Only clear MPEG-TS media playlists: encrypted, fMP4 and byte ranges fail closed. */
|
||||
export function parseHlsMedia(text, base, limit = HLS_LIMITS.segments) {
|
||||
if (
|
||||
!text.startsWith('#EXTM3U') ||
|
||||
/#EXT-X-(?:KEY|MAP|BYTERANGE|I-FRAMES-ONLY)/.test(text)
|
||||
)
|
||||
throw new Error('Unsupported HLS playlist');
|
||||
let seq = 0;
|
||||
let duration = null;
|
||||
let discontinuity = false;
|
||||
const segments = [];
|
||||
for (const raw of text.split('\n')) {
|
||||
const line = raw.trim();
|
||||
if (line.startsWith('#EXT-X-MEDIA-SEQUENCE:')) {
|
||||
seq = Number(line.slice(22));
|
||||
if (!Number.isSafeInteger(seq) || seq < 0)
|
||||
throw new Error('Invalid media sequence');
|
||||
} else if (line === '#EXT-X-DISCONTINUITY') {
|
||||
discontinuity = true;
|
||||
} else if (line.startsWith('#EXTINF:')) {
|
||||
duration = Number.parseFloat(line.slice(8));
|
||||
if (!Number.isFinite(duration) || duration <= 0 || duration > 60)
|
||||
throw new Error('Invalid segment duration');
|
||||
} else if (line && !line.startsWith('#')) {
|
||||
if (duration === null || !Number.isSafeInteger(seq))
|
||||
throw new Error('Invalid HLS segment');
|
||||
const uri = sameOriginHlsUrl(line, base);
|
||||
if (!new URL(uri).pathname.endsWith('.ts'))
|
||||
throw new Error('Only MPEG-TS segments are supported');
|
||||
segments.push({ seq: seq++, duration, uri, discontinuity });
|
||||
discontinuity = false;
|
||||
duration = null;
|
||||
}
|
||||
}
|
||||
return segments.slice(-limit);
|
||||
}
|
||||
|
||||
export function createHlsPuller({
|
||||
fetchImpl = fetch,
|
||||
limits = HLS_LIMITS,
|
||||
} = {}) {
|
||||
const active = new Map();
|
||||
let closed = false;
|
||||
const stop = (cameraId, token) => {
|
||||
const entry = active.get(cameraId);
|
||||
if (!entry || (token && token !== entry.token)) return;
|
||||
active.delete(cameraId);
|
||||
entry.stopping = true;
|
||||
entry.controller.abort();
|
||||
clearTimeout(entry.timer);
|
||||
for (const lease of entry.leases.values()) clearTimeout(lease.timer);
|
||||
entry.leases.clear();
|
||||
entry.segments.clear();
|
||||
entry.upstream.clear();
|
||||
entry.bytes = 0;
|
||||
};
|
||||
const release = (cameraId, leaseId) => {
|
||||
const entry = active.get(cameraId);
|
||||
const lease = entry?.leases.get(leaseId);
|
||||
if (!lease) return;
|
||||
clearTimeout(lease.timer);
|
||||
entry.leases.delete(leaseId);
|
||||
if (!entry.leases.size) stop(cameraId, entry.token);
|
||||
};
|
||||
const touch = (entry, leaseId, create = false) => {
|
||||
let lease = entry.leases.get(leaseId);
|
||||
if (!lease) {
|
||||
if (!create || entry.leases.size >= limits.leasesPerSession)
|
||||
throw new Error('HLS lease capacity reached');
|
||||
lease = {};
|
||||
entry.leases.set(leaseId, lease);
|
||||
}
|
||||
clearTimeout(lease.timer);
|
||||
lease.timer = setTimeout(
|
||||
() => release(entry.cameraId, leaseId),
|
||||
limits.idleMs,
|
||||
);
|
||||
lease.timer.unref?.();
|
||||
};
|
||||
const read = (entry, url, maxBytes) =>
|
||||
fetchHlsBytes(url, {
|
||||
fetchImpl,
|
||||
signal: entry.controller.signal,
|
||||
maxBytes,
|
||||
timeoutMs: limits.timeoutMs,
|
||||
});
|
||||
const poll = async (entry) => {
|
||||
try {
|
||||
let base = entry.chunklistUrl || entry.url;
|
||||
let text = (await read(entry, base, limits.playlistBytes)).toString(
|
||||
'utf8',
|
||||
);
|
||||
if (!text.includes('#EXTINF:')) {
|
||||
const variant = text
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.find((line) => line && !line.startsWith('#'));
|
||||
if (!variant || !text.includes('#EXT-X-STREAM-INF:'))
|
||||
throw new Error('Missing HLS variant');
|
||||
base = sameOriginHlsUrl(variant, entry.url);
|
||||
text = (await read(entry, base, limits.playlistBytes)).toString('utf8');
|
||||
}
|
||||
entry.chunklistUrl = base;
|
||||
const segments = parseHlsMedia(text, base, limits.segments);
|
||||
const newest = segments.at(-1)?.seq ?? -1;
|
||||
const restarted =
|
||||
newest < entry.upstreamNewest ||
|
||||
segments.some((segment) => {
|
||||
const prior = entry.upstream.get(segment.seq);
|
||||
return prior && prior !== segment.uri;
|
||||
});
|
||||
if (restarted) {
|
||||
entry.upstream.clear();
|
||||
entry.pendingDiscontinuity = true;
|
||||
}
|
||||
entry.upstreamNewest = newest;
|
||||
for (const segment of segments) {
|
||||
if (entry.stopping) return;
|
||||
if (entry.upstream.has(segment.seq)) continue;
|
||||
const body = await read(entry, segment.uri, limits.segmentBytes);
|
||||
if (entry.stopping) return;
|
||||
// Evict BEFORE retaining the incoming segment. One bounded download may
|
||||
// exist in addition to the cache; no unbounded disk output is possible.
|
||||
while (
|
||||
entry.segments.size &&
|
||||
(entry.bytes + body.length > limits.sessionBytes ||
|
||||
entry.segments.size >= limits.segments)
|
||||
) {
|
||||
const oldest = entry.segments.keys().next().value;
|
||||
const removed = entry.segments.get(oldest);
|
||||
entry.bytes -= removed.body.length;
|
||||
if (removed.discontinuity) entry.discontinuitiesRemoved++;
|
||||
entry.segments.delete(oldest);
|
||||
}
|
||||
if (body.length > limits.sessionBytes)
|
||||
throw new Error('Segment exceeds session budget');
|
||||
const seq = entry.nextSeq++;
|
||||
entry.segments.set(seq, {
|
||||
...segment,
|
||||
seq,
|
||||
body,
|
||||
discontinuity: segment.discontinuity || entry.pendingDiscontinuity,
|
||||
});
|
||||
entry.pendingDiscontinuity = false;
|
||||
entry.upstream.set(segment.seq, segment.uri);
|
||||
while (entry.upstream.size > limits.segments * 2)
|
||||
entry.upstream.delete(entry.upstream.keys().next().value);
|
||||
entry.bytes += body.length;
|
||||
}
|
||||
entry.failures = 0;
|
||||
} catch {
|
||||
entry.chunklistUrl = null;
|
||||
entry.failures++;
|
||||
if (entry.failures >= 3) {
|
||||
stop(entry.cameraId, entry.token);
|
||||
return;
|
||||
}
|
||||
} finally {
|
||||
if (!entry.stopping) {
|
||||
entry.timer = setTimeout(() => {
|
||||
entry.polling = poll(entry);
|
||||
}, limits.pollMs);
|
||||
entry.timer.unref?.();
|
||||
}
|
||||
}
|
||||
};
|
||||
const ensure = async (cameraId, url, leaseId = 'legacy') => {
|
||||
if (closed) throw new Error('HLS service closed');
|
||||
sameOriginHlsUrl(url, url);
|
||||
let entry = active.get(cameraId);
|
||||
if (entry && entry.url !== url) {
|
||||
stop(cameraId);
|
||||
entry = null;
|
||||
}
|
||||
if (entry) {
|
||||
touch(entry, leaseId, true);
|
||||
return entry;
|
||||
}
|
||||
// Reserve synchronously, before any await: concurrent creation is singleflight.
|
||||
if (active.size >= limits.sessions)
|
||||
throw new Error('HLS session capacity reached');
|
||||
entry = {
|
||||
cameraId,
|
||||
url,
|
||||
token: randomUUID(),
|
||||
controller: new AbortController(),
|
||||
segments: new Map(),
|
||||
bytes: 0,
|
||||
stopping: false,
|
||||
failures: 0,
|
||||
nextSeq: 0,
|
||||
upstream: new Map(),
|
||||
leases: new Map(),
|
||||
upstreamNewest: -1,
|
||||
pendingDiscontinuity: false,
|
||||
discontinuitiesRemoved: 0,
|
||||
};
|
||||
active.set(cameraId, entry);
|
||||
touch(entry, leaseId, true);
|
||||
entry.polling = poll(entry);
|
||||
return entry;
|
||||
};
|
||||
const waitReady = async (entry, signal) => {
|
||||
const until = Date.now() + limits.readyMs;
|
||||
while (!entry.stopping && !signal?.aborted && Date.now() < until) {
|
||||
if (entry.segments.size >= 2) return true;
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const buildPlaylist = async (entry, cameraId, leaseId = 'legacy') => {
|
||||
if (entry.stopping || entry.segments.size < 2) return null;
|
||||
touch(entry, leaseId);
|
||||
const segments = [...entry.segments.values()].sort((a, b) => a.seq - b.seq);
|
||||
const lines = [
|
||||
'#EXTM3U',
|
||||
'#EXT-X-VERSION:3',
|
||||
`#EXT-X-TARGETDURATION:${Math.ceil(Math.max(...segments.map((s) => s.duration)))}`,
|
||||
`#EXT-X-MEDIA-SEQUENCE:${segments[0].seq}`,
|
||||
`#EXT-X-DISCONTINUITY-SEQUENCE:${entry.discontinuitiesRemoved}`,
|
||||
];
|
||||
let previous;
|
||||
for (const segment of segments) {
|
||||
if (
|
||||
segment.discontinuity ||
|
||||
(previous !== undefined && segment.seq !== previous + 1)
|
||||
)
|
||||
lines.push('#EXT-X-DISCONTINUITY');
|
||||
lines.push(
|
||||
`#EXTINF:${segment.duration.toFixed(3)},`,
|
||||
`/api/cctv/media/${encodeURIComponent(cameraId)}/seg_${segment.seq}.ts?session=${entry.token}&lease=${encodeURIComponent(leaseId)}`,
|
||||
);
|
||||
previous = segment.seq;
|
||||
}
|
||||
return lines.join('\n') + '\n';
|
||||
};
|
||||
const getSegment = (cameraId, token, seq, leaseId = 'legacy') => {
|
||||
const entry = active.get(cameraId);
|
||||
if (!entry || entry.token !== token || !entry.leases.has(leaseId))
|
||||
return null;
|
||||
const body = entry.segments.get(seq)?.body;
|
||||
if (body) touch(entry, leaseId);
|
||||
return body || null;
|
||||
};
|
||||
const shutdown = async () => {
|
||||
closed = true;
|
||||
const entries = [...active.values()];
|
||||
for (const entry of entries) stop(entry.cameraId);
|
||||
await Promise.allSettled(entries.map((entry) => entry.polling));
|
||||
};
|
||||
return {
|
||||
ensure,
|
||||
waitReady,
|
||||
buildPlaylist,
|
||||
getSegment,
|
||||
stop,
|
||||
release,
|
||||
shutdown,
|
||||
stats: () => ({
|
||||
sessions: active.size,
|
||||
bytes: [...active.values()].reduce((total, e) => total + e.bytes, 0),
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -2,24 +2,52 @@ import { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
export { makeRateLimiter } from '../../../src/sources/rateLimit.js';
|
||||
|
||||
/**
|
||||
* Opt-in per-IP rate limiter for the cost-bearing API proxies (OpenAI / Google).
|
||||
* DEFAULT IS UNLIMITED: when the env var is unset, `0`, or non-numeric, this
|
||||
* returns `null` and the caller skips the check entirely — a runtime no-op that
|
||||
* preserves the original behavior. Only a positive integer N enables a fixed
|
||||
* 60s window of N requests/IP (built lazily once, then reused so its per-IP
|
||||
* window state persists across requests). The global backstop is set to a
|
||||
* generous multiple of the per-IP cap so a single host can't starve the rest.
|
||||
* Resolve the per-IP, per-minute cap for a cost-bearing proxy from its env var.
|
||||
*
|
||||
* @param {string|undefined} envValue - Raw env value (requests/min/IP).
|
||||
* The default is ON. An unset or blank value takes `defaultPerMin`; a positive
|
||||
* integer overrides it; an explicit `0` disables the limiter for deployments
|
||||
* that genuinely want no throttle.
|
||||
*
|
||||
* A value that cannot be read as a number — a typo, a stray unit, a negative —
|
||||
* also takes the default rather than disabling the guard. That asymmetry is the
|
||||
* point: `GEV_RATELIMIT_OPENAI_PER_MIN=3O` should not quietly hand an exposed
|
||||
* server's API key to whoever asks. Removing the throttle stays possible, but
|
||||
* only by saying `0`, which cannot be typed by accident.
|
||||
*
|
||||
* @param {string|number|undefined|null} envValue - Raw env value (requests/min/IP).
|
||||
* @param {number} defaultPerMin - Cap to apply when the value is absent or unreadable.
|
||||
* @returns {number} Requests/min/IP; `0` means unlimited.
|
||||
*/
|
||||
export function resolvePerMinuteCap(envValue, defaultPerMin) {
|
||||
const raw = typeof envValue === 'string' ? envValue.trim() : envValue;
|
||||
if (raw === undefined || raw === null || raw === '') return defaultPerMin;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) return defaultPerMin;
|
||||
// Only an explicit 0 disables the guard: a positive fraction such as 0.5
|
||||
// would floor to 0, so it takes the strictest real cap instead.
|
||||
if (parsed > 0 && parsed < 1) return 1;
|
||||
return Math.floor(parsed); // includes the explicit 0 = unlimited opt-out
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-IP rate limiter for the cost-bearing API proxies (OpenAI / Google).
|
||||
* Returns `null` only when the cap resolves to `0` (the explicit opt-out), in
|
||||
* which case the caller skips the check entirely. Otherwise a fixed 60s window
|
||||
* of N requests/IP, built lazily once and reused so its per-IP window state
|
||||
* persists across requests. The global backstop is a generous multiple of the
|
||||
* per-IP cap so a single host can't starve the rest.
|
||||
*
|
||||
* @param {string|number|undefined|null} envValue - Raw env value (requests/min/IP).
|
||||
* @param {number} defaultPerMin - Cap to apply when the value is absent or unreadable.
|
||||
* @returns {((key:string)=>boolean)|null} An `allow(key)` fn, or null when unlimited.
|
||||
*/
|
||||
export function makeOptInRateLimiter(envValue) {
|
||||
const max = Number(envValue);
|
||||
if (!Number.isFinite(max) || max <= 0) return null; // unset/0/garbage -> unlimited
|
||||
export function makeCostRateLimiter(envValue, defaultPerMin) {
|
||||
const max = resolvePerMinuteCap(envValue, defaultPerMin);
|
||||
if (max <= 0) return null; // explicit opt-out
|
||||
return makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: Math.floor(max),
|
||||
globalMax: Math.floor(max) * 20,
|
||||
max,
|
||||
globalMax: max * 20,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -17,17 +17,29 @@ async function readRequestBodyCapped(req, maxBytes) {
|
||||
function readRequestBody(req, maxBytes = 1024 * 1024) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let total = 0;
|
||||
let bodyTooLarge = false;
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => {
|
||||
// Keep draining after the cap without retaining more bytes. Destroying
|
||||
// the request also destroys its socket, so the handler cannot deliver
|
||||
// the fixed error response it constructs for BODY_TOO_LARGE.
|
||||
if (bodyTooLarge) return;
|
||||
total += chunk.length;
|
||||
if (total > maxBytes) {
|
||||
reject(new Error(`Request body exceeds ${maxBytes} bytes`));
|
||||
req.destroy();
|
||||
// Same marker readRequestBodyCapped sets, so a caller can tell an
|
||||
// oversized body from a genuine failure without matching on text.
|
||||
bodyTooLarge = true;
|
||||
chunks.length = 0;
|
||||
const error = new Error(`Request body exceeds ${maxBytes} bytes`);
|
||||
error.code = 'BODY_TOO_LARGE';
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
req.on('end', () => {
|
||||
if (!bodyTooLarge) resolve(Buffer.concat(chunks).toString('utf8'));
|
||||
});
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { admitSameSiteRequest } from '../../../src/localRequestGate.mjs';
|
||||
|
||||
/**
|
||||
* Cross-site request gate for cost-bearing and log endpoints. Feeds the
|
||||
* request to the pure, unit-tested `admitSameSiteRequest` and answers 403
|
||||
* itself when the request is refused. Returns true when it has already
|
||||
* responded (the caller returns); false when the handler should continue.
|
||||
* Refuses cross-site browser requests (foreign or opaque Origin, a
|
||||
* Sec-Fetch-Site other than same-origin/none, or reverse-proxy headers) while
|
||||
* keeping loopback non-browser tools and the HOST=0.0.0.0 LAN opt-in working.
|
||||
* @param {import('node:http').IncomingMessage} req
|
||||
* @param {import('node:http').ServerResponse} res
|
||||
* @returns {boolean} True when a 403 was sent.
|
||||
*/
|
||||
export function admitSameSite(req, res) {
|
||||
const verdict = admitSameSiteRequest({
|
||||
hostHeader: req.headers?.host,
|
||||
protocol: req.socket?.encrypted ? 'https:' : 'http:',
|
||||
origin: req.headers?.origin,
|
||||
secFetchSite: req.headers?.['sec-fetch-site'],
|
||||
proxyHeaders: req.headers || {},
|
||||
});
|
||||
if (verdict.ok) return false;
|
||||
res.statusCode = verdict.status;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify({ error: verdict.error }));
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a middleware handler so cross-site requests are refused with 403
|
||||
* before the handler runs. Loopback tools and same-origin browser requests
|
||||
* pass straight through.
|
||||
* @param {(req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse, next?: Function) => unknown} handler
|
||||
* @returns {(req: import('node:http').IncomingMessage, res: import('node:http').ServerResponse, next?: Function) => unknown}
|
||||
*/
|
||||
export function sameSiteGated(handler) {
|
||||
return (req, res, next) => {
|
||||
if (admitSameSite(req, res)) return undefined;
|
||||
return handler(req, res, next);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
const STATUS_URL = 'https://www.nhc.noaa.gov/CurrentStorms.json';
|
||||
const GIS =
|
||||
'https://mapservices.weather.noaa.gov/tropical/rest/services/tropical/NHC_tropical_weather_summary/MapServer';
|
||||
const HOUR = 3600_000;
|
||||
const COVERAGE =
|
||||
'Atlantic and eastern/central North Pacific; not worldwide cyclone coverage.';
|
||||
const LAYERS = [
|
||||
{ id: 5, cap: 512 * 1024, count: 500, kind: 'points' },
|
||||
{ id: 6, cap: 512 * 1024, count: 32, kind: 'track' },
|
||||
{ id: 7, cap: 2 * 1024 * 1024, count: 32, kind: 'cone' },
|
||||
];
|
||||
|
||||
function invalid() {
|
||||
return new Error('invalid_cyclone_data');
|
||||
}
|
||||
function text(value, max = 80) {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!value.trim() ||
|
||||
value.length > max ||
|
||||
/[\u0000-\u001f<>]/.test(value)
|
||||
)
|
||||
throw invalid();
|
||||
return value.trim();
|
||||
}
|
||||
function advisory(value) {
|
||||
if (typeof value !== 'string' || !/^\d{1,3}[A-Z]?$/i.test(value))
|
||||
throw invalid();
|
||||
return value.replace(/^0+(?=\d)/, '').toUpperCase();
|
||||
}
|
||||
function number(value, min, max) {
|
||||
if (
|
||||
value === null ||
|
||||
value === undefined ||
|
||||
value === '' ||
|
||||
!['string', 'number'].includes(typeof value)
|
||||
)
|
||||
return null;
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) && n >= min && n <= max ? n : null;
|
||||
}
|
||||
function iso(value, now) {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{3})?Z$/.test(value)
|
||||
)
|
||||
throw invalid();
|
||||
const n = Date.parse(value);
|
||||
if (
|
||||
!Number.isFinite(n) ||
|
||||
n > now + 5 * 60_000 ||
|
||||
now - n > 12 * HOUR ||
|
||||
new Date(n).toISOString().replace('.000Z', 'Z') !==
|
||||
value.replace('.000Z', 'Z')
|
||||
)
|
||||
throw invalid();
|
||||
return new Date(n).toISOString();
|
||||
}
|
||||
function position(coordinates) {
|
||||
if (
|
||||
!Array.isArray(coordinates) ||
|
||||
coordinates.length !== 2 ||
|
||||
!coordinates.every((n) => typeof n === 'number' && Number.isFinite(n)) ||
|
||||
Math.abs(coordinates[0]) > 180 ||
|
||||
Math.abs(coordinates[1]) > 90
|
||||
)
|
||||
throw invalid();
|
||||
return { longitude: coordinates[0], latitude: coordinates[1] };
|
||||
}
|
||||
function officialLink(value) {
|
||||
if (typeof value !== 'string' || value.length > 256) return null;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.origin === 'https://www.nhc.noaa.gov' &&
|
||||
!url.username &&
|
||||
!url.password &&
|
||||
!url.search &&
|
||||
!url.hash &&
|
||||
/^\/text\/[A-Z0-9]+\.shtml$/.test(url.pathname)
|
||||
? url.href
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseCycloneStatus(payload, now = Date.now()) {
|
||||
if (!Array.isArray(payload?.activeStorms) || payload.activeStorms.length > 32)
|
||||
throw invalid();
|
||||
const seen = new Set();
|
||||
return payload.activeStorms.map((raw) => {
|
||||
if (
|
||||
typeof raw?.id !== 'string' ||
|
||||
!/^(?:al|ep|cp)\d{6}$/.test(raw.id) ||
|
||||
seen.has(raw.id)
|
||||
)
|
||||
throw invalid();
|
||||
seen.add(raw.id);
|
||||
const forecast = raw.forecastAdvisory;
|
||||
const issuedAt = iso(forecast?.issuance, now);
|
||||
const advisoryNumber = advisory(forecast?.advNum);
|
||||
const positionAt = iso(raw.lastUpdate, now);
|
||||
return {
|
||||
id: raw.id,
|
||||
name: text(raw.name),
|
||||
classification: text(raw.classification, 16),
|
||||
basin: raw.id.slice(0, 2).toUpperCase(),
|
||||
position: position([raw.longitudeNumeric, raw.latitudeNumeric]),
|
||||
positionAt,
|
||||
advisoryNumber,
|
||||
issuedAt,
|
||||
windKt: number(raw.intensity, 0, 300),
|
||||
pressureHpa: number(raw.pressure, 800, 1100),
|
||||
movement: {
|
||||
directionDegrees: number(raw.movementDir, 0, 360),
|
||||
speedKt: number(raw.movementSpeed, 0, 200),
|
||||
},
|
||||
advisoryUrl: officialLink(forecast.url),
|
||||
outlookUrl: raw.id.startsWith('al')
|
||||
? 'https://www.nhc.noaa.gov/gtwo.php?basin=atlc&fdays=7'
|
||||
: raw.id.startsWith('cp')
|
||||
? 'https://www.nhc.noaa.gov/gtwo.php?basin=cpac&fdays=7'
|
||||
: 'https://www.nhc.noaa.gov/gtwo.php?basin=epac&fdays=7',
|
||||
geometryStatus: 'pending',
|
||||
geometryAdvisoryNumber: null,
|
||||
forecastPoints: [],
|
||||
track: null,
|
||||
cone: null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function geometry(raw, kind, budget) {
|
||||
const types =
|
||||
kind === 'points'
|
||||
? ['Point']
|
||||
: kind === 'track'
|
||||
? ['LineString', 'MultiLineString']
|
||||
: ['Polygon', 'MultiPolygon'];
|
||||
if (!types.includes(raw?.type)) throw invalid();
|
||||
const point = (value) => {
|
||||
position(value);
|
||||
if (++budget.count > 25_000) throw invalid();
|
||||
return [...value];
|
||||
};
|
||||
const line = (value, ring = false) => {
|
||||
if (
|
||||
!Array.isArray(value) ||
|
||||
value.length < (ring ? 4 : 2) ||
|
||||
value.length > 10_000
|
||||
)
|
||||
throw invalid();
|
||||
const result = value.map(point);
|
||||
if (
|
||||
ring &&
|
||||
(result[0][0] !== result.at(-1)[0] || result[0][1] !== result.at(-1)[1])
|
||||
)
|
||||
throw invalid();
|
||||
return result;
|
||||
};
|
||||
const list = (value, read) => {
|
||||
if (!Array.isArray(value) || !value.length || value.length > 128)
|
||||
throw invalid();
|
||||
return value.map(read);
|
||||
};
|
||||
const polygon = (value) => list(value, (ring) => line(ring, true));
|
||||
const coordinates =
|
||||
raw.type === 'Point'
|
||||
? point(raw.coordinates)
|
||||
: raw.type === 'LineString'
|
||||
? line(raw.coordinates)
|
||||
: raw.type === 'MultiLineString'
|
||||
? list(raw.coordinates, (part) => line(part))
|
||||
: raw.type === 'Polygon'
|
||||
? polygon(raw.coordinates)
|
||||
: list(raw.coordinates, polygon);
|
||||
return { type: raw.type, coordinates };
|
||||
}
|
||||
|
||||
/** Never relabel earlier GIS geometry with the newer status advisory. */
|
||||
export function attachCycloneGeometry(storms, collections) {
|
||||
const budget = { count: 0 };
|
||||
const parsed = LAYERS.map((spec, index) => {
|
||||
const payload = collections[index];
|
||||
if (
|
||||
payload?.type !== 'FeatureCollection' ||
|
||||
payload.exceededTransferLimit ||
|
||||
!Array.isArray(payload.features) ||
|
||||
payload.features.length > spec.count
|
||||
)
|
||||
throw invalid();
|
||||
return payload.features.map((feature) => {
|
||||
if (feature?.type !== 'Feature') throw invalid();
|
||||
const p = feature.properties;
|
||||
const source =
|
||||
typeof p?.idp_source === 'string' &&
|
||||
p.idp_source.match(
|
||||
/^((?:al|ep|cp)\d{6})-(\d{1,3}[a-z]?)_5day_(pts|lin|pgn)$/i,
|
||||
);
|
||||
if (!source || source[3].toLowerCase() !== ['pts', 'lin', 'pgn'][index])
|
||||
throw invalid();
|
||||
const adv = advisory(p.advisnum);
|
||||
if (adv !== advisory(source[2])) throw invalid();
|
||||
return {
|
||||
id: source[1].toLowerCase(),
|
||||
advisoryNumber: adv,
|
||||
geometry: geometry(feature.geometry, spec.kind, budget),
|
||||
tauHours: number(p.tau, 0, 168),
|
||||
windKt: number(p.maxwind, 0, 300),
|
||||
gustKt: number(p.gust, 0, 350),
|
||||
};
|
||||
});
|
||||
});
|
||||
return storms.map((storm) => {
|
||||
const groups = parsed.map((items) =>
|
||||
items.filter(
|
||||
(item) =>
|
||||
item.id === storm.id && item.advisoryNumber === storm.advisoryNumber,
|
||||
),
|
||||
);
|
||||
if (groups[1].length > 1 || groups[2].length > 1) throw invalid();
|
||||
if (!groups[0].length || groups[1].length !== 1 || groups[2].length !== 1)
|
||||
return storm;
|
||||
const taus = new Set();
|
||||
const forecastPoints = groups[0]
|
||||
.map((item) => {
|
||||
if (item.tauHours === null || taus.has(item.tauHours)) throw invalid();
|
||||
taus.add(item.tauHours);
|
||||
return {
|
||||
position: position(item.geometry.coordinates),
|
||||
tauHours: item.tauHours,
|
||||
windKt: item.windKt,
|
||||
gustKt: item.gustKt,
|
||||
};
|
||||
})
|
||||
.sort((a, b) => a.tauHours - b.tauHours);
|
||||
return {
|
||||
...storm,
|
||||
geometryStatus: 'current',
|
||||
geometryAdvisoryNumber: storm.advisoryNumber,
|
||||
forecastPoints,
|
||||
track: groups[1][0].geometry,
|
||||
cone: groups[2][0].geometry,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Fixed official endpoints; one shared bounded refresh, no user destinations. */
|
||||
export function cycloneProxy({
|
||||
fetchImpl = fetch,
|
||||
now = () => Date.now(),
|
||||
timeoutMs = 12_000,
|
||||
} = {}) {
|
||||
let cache = null;
|
||||
let operation = null;
|
||||
let attemptedAt = -Infinity;
|
||||
async function upstream(url, cap, signal) {
|
||||
signal.throwIfAborted();
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/geo+json,application/json',
|
||||
'User-Agent': 'Gods Eye View (public NOAA weather context)',
|
||||
},
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('cyclone_upstream_unavailable');
|
||||
}
|
||||
const result = JSON.parse(
|
||||
await readResponseTextCapped(response, cap, signal),
|
||||
);
|
||||
signal.throwIfAborted();
|
||||
return result;
|
||||
}
|
||||
async function refresh(signal) {
|
||||
let storms = parseCycloneStatus(
|
||||
await upstream(STATUS_URL, 128 * 1024, signal),
|
||||
now(),
|
||||
);
|
||||
if (storms.length) {
|
||||
try {
|
||||
const results = await Promise.allSettled(
|
||||
LAYERS.map(async (spec) => {
|
||||
const url = new URL(`${GIS}/${spec.id}/query`);
|
||||
url.search = new URLSearchParams({
|
||||
where: '1=1',
|
||||
outFields:
|
||||
spec.kind === 'points'
|
||||
? 'idp_source,advisnum,tau,maxwind,gust'
|
||||
: 'idp_source,advisnum',
|
||||
outSR: '4326',
|
||||
resultRecordCount: String(spec.count),
|
||||
geometryPrecision: '4',
|
||||
f: 'geojson',
|
||||
}).toString();
|
||||
return upstream(url.href, spec.cap, signal);
|
||||
}),
|
||||
);
|
||||
if (results.some((result) => result.status === 'rejected'))
|
||||
throw invalid();
|
||||
storms = attachCycloneGeometry(
|
||||
storms,
|
||||
results.map((result) => result.value),
|
||||
);
|
||||
} catch {
|
||||
storms = storms.map((storm) => ({
|
||||
...storm,
|
||||
geometryStatus: 'unavailable',
|
||||
}));
|
||||
}
|
||||
}
|
||||
signal.throwIfAborted();
|
||||
cache = { storms, fetchedAt: now() };
|
||||
return cache;
|
||||
}
|
||||
async function acquire(signal) {
|
||||
signal.throwIfAborted();
|
||||
if (cache && now() - cache.fetchedAt < 300_000) return cache;
|
||||
if (operation?.controller.signal.aborted) operation = null;
|
||||
if (!operation) {
|
||||
if (now() - attemptedAt < 60_000) throw new Error('cyclone_retry_later');
|
||||
attemptedAt = now();
|
||||
const controller = new AbortController();
|
||||
const owned = { controller, waiters: 0 };
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
owned.promise = refresh(controller.signal).finally(() => {
|
||||
clearTimeout(timer);
|
||||
if (operation === owned) operation = null;
|
||||
});
|
||||
operation = owned;
|
||||
}
|
||||
const owned = operation;
|
||||
if (owned.waiters >= 32)
|
||||
throw Object.assign(new Error('cyclone_busy'), { status: 429 });
|
||||
owned.waiters++;
|
||||
let abort;
|
||||
const cancelled = new Promise((_, reject) => {
|
||||
abort = () => reject(signal.reason ?? new Error('cancelled'));
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
});
|
||||
try {
|
||||
return await Promise.race([owned.promise, cancelled]);
|
||||
} finally {
|
||||
signal.removeEventListener('abort', abort);
|
||||
if (--owned.waiters === 0 && operation === owned) {
|
||||
owned.controller.abort();
|
||||
if (signal.aborted) attemptedAt = -Infinity;
|
||||
}
|
||||
}
|
||||
}
|
||||
function describe(value, stale = false) {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
source: 'NOAA NHC / CPHC',
|
||||
attribution:
|
||||
'NOAA/NWS National Hurricane Center / Central Pacific Hurricane Center',
|
||||
coverage: COVERAGE,
|
||||
fetchedAt: value?.fetchedAt ?? null,
|
||||
stale: stale || !value,
|
||||
unavailable: !value,
|
||||
reason: !value
|
||||
? 'Cyclone data unavailable'
|
||||
: stale
|
||||
? 'Cached cyclone advisory; upstream unavailable'
|
||||
: null,
|
||||
storms: value?.storms ?? [],
|
||||
};
|
||||
}
|
||||
async function handler(req, res) {
|
||||
const controller = new AbortController();
|
||||
const close = () => controller.abort();
|
||||
res.once?.('close', close);
|
||||
const json = (status, value) => {
|
||||
if (controller.signal.aborted) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 429 ? { 'Retry-After': '2' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
try {
|
||||
if (req.method !== 'GET')
|
||||
return json(405, { error: 'method_not_allowed' });
|
||||
if (req.url !== '/' && req.url !== '')
|
||||
return json(400, { error: 'invalid_cyclone_query' });
|
||||
try {
|
||||
json(200, describe(await acquire(controller.signal)));
|
||||
} catch (error) {
|
||||
if (error.status === 429) return json(429, { error: 'cyclone_busy' });
|
||||
const usable =
|
||||
cache &&
|
||||
now() - cache.fetchedAt <= 12 * HOUR &&
|
||||
cache.storms.every(
|
||||
(storm) => now() - Date.parse(storm.issuedAt) <= 12 * HOUR,
|
||||
);
|
||||
json(200, describe(usable ? cache : null, true));
|
||||
}
|
||||
} finally {
|
||||
res.removeListener?.('close', close);
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: 'cyclones',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/cyclones', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/cyclones', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import { normalizeFirePerimeterSnapshot } from '../../src/layers/perimeters/records.js';
|
||||
import {
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
coalesceProxyRequest,
|
||||
} from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
|
||||
// NIFC WFIGS current interagency fire perimeters (public, keyless).
|
||||
// maxAllowableOffset trades ~100 m of boundary fidelity for a payload small
|
||||
// enough to refresh continuously (~1 MB for a typical fire season).
|
||||
const API_URL =
|
||||
'https://services3.arcgis.com/T4QMspbfLg3qTGWY/arcgis/rest/services/' +
|
||||
'WFIGS_Interagency_Perimeters_Current/FeatureServer/0/query?' +
|
||||
new URLSearchParams({
|
||||
where: '1=1',
|
||||
outFields: [
|
||||
'poly_IncidentName',
|
||||
'attr_UniqueFireIdentifier',
|
||||
'attr_IncidentSize',
|
||||
'attr_PercentContained',
|
||||
'attr_POOState',
|
||||
'attr_IncidentTypeCategory',
|
||||
'attr_FireDiscoveryDateTime',
|
||||
'poly_DateCurrent',
|
||||
'attr_FireCause',
|
||||
'attr_FireBehaviorGeneral',
|
||||
'attr_TotalIncidentPersonnel',
|
||||
'attr_POOCounty',
|
||||
'attr_EstimatedCostToDate',
|
||||
'attr_IncidentComplexityLevel',
|
||||
'attr_CpxName',
|
||||
].join(','),
|
||||
maxAllowableOffset: '0.001',
|
||||
outSR: '4326',
|
||||
f: 'geojson',
|
||||
}).toString();
|
||||
|
||||
// The service caps a single response at its maxRecordCount (2000); a peak
|
||||
// season can exceed that, so follow exceededTransferLimit with offset pages.
|
||||
const MAX_PAGES = 5;
|
||||
|
||||
const exceededTransferLimit = (payload) =>
|
||||
payload?.exceededTransferLimit === true ||
|
||||
payload?.properties?.exceededTransferLimit === true;
|
||||
|
||||
const MIB = 1024 * 1024;
|
||||
|
||||
/** Fixed-origin, bounded WFIGS and InciWeb routes for dev and preview. */
|
||||
export function firePerimetersProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const publications = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 60, globalMax: 1200 });
|
||||
|
||||
async function upstream(url, cap, timeout, options = {}) {
|
||||
const signal = AbortSignal.timeout(timeout);
|
||||
const response = await fetchImpl(url, {
|
||||
...options,
|
||||
signal,
|
||||
redirect: 'error',
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
return readResponseJsonCapped(response, cap, signal);
|
||||
}
|
||||
|
||||
async function fetchPerimeters() {
|
||||
const features = [];
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const url =
|
||||
page === 0 ? API_URL : `${API_URL}&resultOffset=${features.length}`;
|
||||
const payload = await upstream(url, 16 * MIB, 30_000);
|
||||
if (!Array.isArray(payload?.features))
|
||||
throw new Error('invalid_snapshot');
|
||||
features.push(...payload.features);
|
||||
if (!exceededTransferLimit(payload) || !payload.features.length) break;
|
||||
}
|
||||
const rows = normalizeFirePerimeterSnapshot({ features });
|
||||
return { fetchedAt: now(), rows };
|
||||
}
|
||||
|
||||
async function fetchIndex() {
|
||||
const rows = await upstream(
|
||||
'https://inciweb.wildfire.gov/api/single-publication/',
|
||||
4 * MIB,
|
||||
20_000,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ title: '' }),
|
||||
},
|
||||
);
|
||||
if (!Array.isArray(rows)) throw new Error('invalid_index');
|
||||
return rows;
|
||||
}
|
||||
|
||||
// InciWeb's publication API was retired; incident pages carry origin and update times.
|
||||
async function fetchPublication(id) {
|
||||
const signal = AbortSignal.timeout(15_000);
|
||||
let response = await fetchImpl(`https://inciweb.wildfire.gov/node/${id}`, {
|
||||
signal,
|
||||
redirect: 'manual',
|
||||
});
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
await response.body?.cancel();
|
||||
const location = response.headers.get('location');
|
||||
if (!location || /\.\.|[?%\\]/.test(location))
|
||||
throw new Error('invalid_redirect');
|
||||
const url = new URL(location, 'https://inciweb.wildfire.gov/');
|
||||
if (
|
||||
url.hostname !== 'inciweb.wildfire.gov' ||
|
||||
!/^\/[a-z0-9-]+(?:\/[a-z0-9-]+)?$/.test(url.pathname) ||
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
url.port ||
|
||||
url.username ||
|
||||
url.password
|
||||
)
|
||||
throw new Error('invalid_redirect');
|
||||
url.protocol = 'https:';
|
||||
url.search = '';
|
||||
url.hash = '';
|
||||
response = await fetchImpl(url.href, { signal, redirect: 'error' });
|
||||
}
|
||||
if (response.status !== 200) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
const html = await readResponseTextCapped(response, 2 * MIB, signal);
|
||||
const changed = Date.parse(
|
||||
html.match(/<meta\s+property="og:updated_time"\s+content="([^"]+)"/)?.[1],
|
||||
);
|
||||
const created = Date.parse(
|
||||
html.match(
|
||||
/Date of Origin<\/th>\s*<td[^>]*>\s*<time\s+datetime="([^"]+)"/,
|
||||
)?.[1],
|
||||
);
|
||||
const createdMs = Number.isFinite(created) ? created : null;
|
||||
const changedMs = Number.isFinite(changed) ? changed : null;
|
||||
if (createdMs === null && changedMs === null)
|
||||
throw new Error('invalid_incident_page');
|
||||
return { createdMs, changedMs };
|
||||
}
|
||||
|
||||
async function acquire(key, ttl, load, store) {
|
||||
const previous = store.get(key);
|
||||
if (previous && now() - previous.savedAt < ttl)
|
||||
return { value: previous.value, stale: false };
|
||||
// Bound distinct concurrent publication requests as well as retained entries.
|
||||
if (!inFlight.has(key) && inFlight.size >= 256)
|
||||
throw Object.assign(new Error('busy'), { status: 429 });
|
||||
try {
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
const value = await load();
|
||||
store.delete(key);
|
||||
store.set(key, { value, savedAt: now() });
|
||||
if (store === publications && store.size > 256)
|
||||
store.delete(store.keys().next().value);
|
||||
return value;
|
||||
});
|
||||
return { value: await promise, stale: false };
|
||||
} catch (error) {
|
||||
if (previous) return { value: previous.value, stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...(status === 429 ? { 'Retry-After': '60' } : {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
let key,
|
||||
ttl,
|
||||
load,
|
||||
store = cache;
|
||||
if (path === '/' || path === '') {
|
||||
key = 'perimeters';
|
||||
ttl = 300_000;
|
||||
load = fetchPerimeters;
|
||||
} else if (path === '/inciweb/index') {
|
||||
key = 'index';
|
||||
ttl = 3_600_000;
|
||||
load = fetchIndex;
|
||||
} else if (path.startsWith('/inciweb/publication/')) {
|
||||
const id = path.slice('/inciweb/publication/'.length);
|
||||
if (!/^\d{1,9}$/.test(id))
|
||||
return json(400, { error: 'invalid_publication_id' });
|
||||
key = `publication:${id}`;
|
||||
ttl = 1_800_000;
|
||||
load = () => fetchPublication(id);
|
||||
store = publications;
|
||||
} else return json(404, { error: 'unknown_route' });
|
||||
if (!allow(clientKey(req))) return json(429, { error: 'rate_limited' });
|
||||
try {
|
||||
const { value, stale } = await acquire(key, ttl, load, store);
|
||||
json(
|
||||
200,
|
||||
key === 'perimeters' && stale ? { ...value, stale: true } : value,
|
||||
stale,
|
||||
);
|
||||
} catch (error) {
|
||||
json(error.status === 429 ? 429 : 502, {
|
||||
error: 'fire_perimeters_unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'fire-perimeters',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/fire-perimeters', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/fire-perimeters', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -2,16 +2,18 @@ import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
/**
|
||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
|
||||
*
|
||||
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
|
||||
* satellites, no cross-source dedup) fetched sequentially with `days=2`
|
||||
* (`days=1` means "current UTC day", nearly empty just after 00:00Z) and
|
||||
* clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is
|
||||
* 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
|
||||
* satellites, no cross-source dedup) plus MODIS_NRT (combined Terra+Aqua,
|
||||
* ~1 km, numeric 0-100 confidence), fetched sequentially (quota courtesy)
|
||||
* with `days=2` (`days=1` means "current UTC day", nearly empty just after
|
||||
* 00:00Z) and clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS
|
||||
* quota is 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
|
||||
* TTL 30 min, single-flight refresh, serve-stale-on-failure, and a
|
||||
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
|
||||
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
|
||||
@@ -28,7 +30,18 @@ import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
export function firmsProxy() {
|
||||
const TTL_MS = 30 * 60_000;
|
||||
const STATUS_TTL_MS = 5 * 60_000;
|
||||
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT'];
|
||||
const SOURCES = [
|
||||
'VIIRS_NOAA20_NRT',
|
||||
'VIIRS_NOAA21_NRT',
|
||||
'VIIRS_SNPP_NRT',
|
||||
'MODIS_NRT',
|
||||
];
|
||||
/**
|
||||
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
|
||||
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
|
||||
* bound, not a budget.
|
||||
*/
|
||||
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
|
||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
|
||||
|
||||
@@ -77,9 +90,12 @@ export function firmsProxy() {
|
||||
*/
|
||||
async function fetchSource(key, source) {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
||||
const signal = AbortSignal.timeout(60_000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const records = parseFirmsCsv(await res.text());
|
||||
const records = parseFirmsCsv(
|
||||
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
|
||||
);
|
||||
if (records === null) throw new Error('non-CSV upstream response');
|
||||
return records;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
isAllowedGbfsHost,
|
||||
isAllowedGbfsPath,
|
||||
gbfsCacheControl,
|
||||
gbfsUpstreamHeaders,
|
||||
} from '../../src/data/gbfsSource.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -11,6 +12,9 @@ import {
|
||||
/** Upstream fetch timeout for GBFS requests (ms). */
|
||||
const GBFS_PROXY_TIMEOUT_MS = 12000;
|
||||
|
||||
/** Value of the Client-Identifier header sent with every upstream request. */
|
||||
export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view';
|
||||
|
||||
export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
|
||||
|
||||
function gbfsRedirectHost(location, requestUrl) {
|
||||
@@ -60,6 +64,10 @@ export async function fetchGbfsUpstream(
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
|
||||
// Urban Sharing (the Norwegian city-bike feeds) asks every client to
|
||||
// name itself in this header; other operators ignore it.
|
||||
'Client-Identifier': GBFS_CLIENT_IDENTIFIER,
|
||||
...gbfsUpstreamHeaders(new URL(url).hostname),
|
||||
},
|
||||
redirect: 'manual',
|
||||
signal: controller.signal,
|
||||
@@ -181,7 +189,7 @@ export function gbfsProxy() {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
|
||||
if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) {
|
||||
res.writeHead(400, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import {
|
||||
GNSS_CLASSIFIER,
|
||||
GNSS_WINDOW_MS,
|
||||
normalizeGnssAircraft,
|
||||
} from '../../src/layers/gnss/records.js';
|
||||
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
|
||||
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
|
||||
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
|
||||
// This is a separate read from the flights fallback on purpose: that point
|
||||
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
|
||||
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
|
||||
// whole-degree anchors held for a minute.
|
||||
const RADIUS_NM = 250;
|
||||
const CACHE_MS = 60_000;
|
||||
const CACHE_MAX = 64;
|
||||
/** Default 429 cooldown when upstream sends no usable Retry-After. */
|
||||
const COOLDOWN_MS = 60_000;
|
||||
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
|
||||
const COOLDOWN_MIN_MS = 5_000;
|
||||
const COOLDOWN_MAX_MS = 60 * 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
const MAX_IN_FLIGHT = 8;
|
||||
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
|
||||
export function gnssIntegrityAnchor(url) {
|
||||
const anchor = adsbLolFallbackAnchor({ url });
|
||||
if (!anchor) return null;
|
||||
return {
|
||||
lat: Math.round(anchor.latitude),
|
||||
lon: Math.round(anchor.longitude),
|
||||
};
|
||||
}
|
||||
|
||||
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
|
||||
export function gnssRetryCooldownMs(raw, nowMs) {
|
||||
if (raw) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
|
||||
const ms = seconds * 1000;
|
||||
if (Number.isFinite(seconds) && seconds > 0)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
|
||||
const at =
|
||||
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
|
||||
if (Number.isFinite(at) && at > nowMs)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
|
||||
export function gnssIntegrityProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
monotonicNow = () => performance.now(),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
|
||||
let cooldownUntil = 0;
|
||||
let cooldownStatus = 429;
|
||||
|
||||
async function load({ lat, lon }) {
|
||||
if (monotonicNow() < cooldownUntil)
|
||||
throw Object.assign(new Error('cooling_down'), {
|
||||
status: cooldownStatus,
|
||||
});
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
|
||||
{
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
if (response.status === 429) {
|
||||
const at = monotonicNow();
|
||||
const raw = response.headers.get('retry-after');
|
||||
const delay = gnssRetryCooldownMs(raw, now());
|
||||
if (delay === COOLDOWN_MAX_MS)
|
||||
warn('[gnss-integrity] upstream cooldown capped at one hour');
|
||||
cooldownStatus = 429;
|
||||
cooldownUntil = Math.max(cooldownUntil, at + delay);
|
||||
throw Object.assign(new Error('upstream_rate_limited'), {
|
||||
status: 429,
|
||||
});
|
||||
}
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const rows = normalizeGnssAircraft(payload);
|
||||
if (!rows) throw new Error('invalid_snapshot');
|
||||
const sourceNow = payload.now;
|
||||
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
|
||||
throw new Error('undated_snapshot');
|
||||
const fetchedAt = Math.min(
|
||||
now(),
|
||||
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
|
||||
);
|
||||
if (now() - fetchedAt > GNSS_WINDOW_MS)
|
||||
throw new Error('expired_snapshot');
|
||||
return {
|
||||
fetchedAt,
|
||||
anchor: { lat, lon },
|
||||
radiusNm: RADIUS_NM,
|
||||
// `degraded` on each row is this classifier's verdict, not gpsjam's.
|
||||
classifier: GNSS_CLASSIFIER.id,
|
||||
rows,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.status !== 429) {
|
||||
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
|
||||
if (until > cooldownUntil) {
|
||||
cooldownUntil = until;
|
||||
cooldownStatus = 502;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function age(entry) {
|
||||
if (!entry) return Infinity;
|
||||
const elapsed = monotonicNow() - entry.savedAt;
|
||||
// A backwards injected clock cannot rejuvenate a snapshot.
|
||||
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
|
||||
}
|
||||
|
||||
async function acquire(anchor, peer) {
|
||||
const key = `${anchor.lat},${anchor.lon}`;
|
||||
const previous = cache.get(key);
|
||||
if (age(previous) < CACHE_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: false };
|
||||
try {
|
||||
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
|
||||
throw Object.assign(new Error('busy'), { status: 429, local: true });
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
if (monotonicNow() >= cooldownUntil && !allow(peer))
|
||||
throw Object.assign(new Error('rate_limited'), {
|
||||
status: 429,
|
||||
local: true,
|
||||
});
|
||||
const value = await load(anchor);
|
||||
const entry = {
|
||||
value,
|
||||
savedAt: monotonicNow(),
|
||||
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
|
||||
};
|
||||
cache.delete(key);
|
||||
cache.set(key, entry);
|
||||
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
|
||||
return entry;
|
||||
});
|
||||
const entry = await promise;
|
||||
const ageMs = age(entry);
|
||||
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
|
||||
} catch (error) {
|
||||
if (age(previous) <= GNSS_WINDOW_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false, retryAfterS = 60) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...([429, 502].includes(status)
|
||||
? { 'Retry-After': String(retryAfterS) }
|
||||
: {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
const anchor = gnssIntegrityAnchor(req.url);
|
||||
if (!anchor) return json(400, { error: 'invalid_anchor' });
|
||||
try {
|
||||
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
|
||||
// ageMs is measured on this server's clock, so the browser can date
|
||||
// the rows without comparing its clock to ours.
|
||||
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
|
||||
} catch (error) {
|
||||
if (error.status === 429) {
|
||||
// Tell the browser how long the shared upstream cooldown has left.
|
||||
const retryAfterS = Math.max(
|
||||
1,
|
||||
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
|
||||
);
|
||||
return json(
|
||||
429,
|
||||
{
|
||||
error: error.local ? error.message : 'gnss_integrity_rate_limited',
|
||||
},
|
||||
false,
|
||||
retryAfterS,
|
||||
);
|
||||
}
|
||||
json(
|
||||
502,
|
||||
{ error: 'gnss_integrity_unavailable' },
|
||||
false,
|
||||
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'gnss-integrity',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
import { lookup as dnsLookup } from 'node:dns/promises';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import { Readable } from 'node:stream';
|
||||
import { coalesceProxyRequest, readResponseTextCapped } from './common/http.js';
|
||||
import { isLocalIpv4, parseTapAddress } from '../../src/data/tapAddress.js';
|
||||
import { normalizeDump1090Aircraft } from '../../src/sources/adsbRecords.js';
|
||||
|
||||
/**
|
||||
* Local decoder feeds for the Local ADS-B layer.
|
||||
*
|
||||
* A user who runs dump1090 / readsb / tar1090 (1090 MHz) or dump978-fa +
|
||||
* skyaware978 (978 MHz UAT) exposes an `aircraft.json` over HTTP. This route
|
||||
* reads those documents and returns them as Local ADS-B records.
|
||||
*
|
||||
* The feed list comes only from the operator's `LOCAL_RECEIVER_FEEDS`
|
||||
* environment value; the browser never supplies an address. Every host must
|
||||
* pass the shared receiver-tap address rule (`src/data/tapAddress.js`:
|
||||
* loopback, RFC1918, `localhost`, `*.local`), the scheme must be http(s) and
|
||||
* the path must end in `aircraft.json`. Invalid entries are logged at startup,
|
||||
* reported as `invalid` and never fetched.
|
||||
*
|
||||
* A name (`localhost`, `*.local`) is resolved before every read; each address
|
||||
* it resolves to must itself be loopback or RFC1918, and the connection is
|
||||
* pinned to exactly those validated addresses, so a later re-resolution
|
||||
* cannot redirect it anywhere else.
|
||||
*/
|
||||
|
||||
export const LOCAL_RECEIVERS_ROUTE = '/api/local-receivers/aircraft';
|
||||
export const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
|
||||
export const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
|
||||
export const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
|
||||
export const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
|
||||
export const LOCAL_RECEIVER_CACHE_MS = 1_000;
|
||||
/** A feed whose own `now` is older than this is reported stale. */
|
||||
export const LOCAL_RECEIVER_STALE_MS = 10_000;
|
||||
const MAX_FEEDS = 8;
|
||||
const LOG_PREFIX = '[local-receivers]';
|
||||
|
||||
const BAND_LABELS = Object.freeze({ 1090: '1090 MHz', 978: '978 MHz UAT' });
|
||||
/** Statuses whose `Response` must be constructed without a body. */
|
||||
const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
|
||||
|
||||
function defaultPort(protocol) {
|
||||
return protocol === 'https:' ? 443 : 80;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate one `band=url` entry.
|
||||
* @param {string} entry Raw entry text.
|
||||
* @returns {{band:string, url:string}|{band:string|null, reason:string}}
|
||||
*/
|
||||
function parseEntry(entry) {
|
||||
const separator = entry.indexOf('=');
|
||||
if (separator <= 0) return { band: null, reason: 'expected band=url' };
|
||||
const band = entry.slice(0, separator).trim();
|
||||
const rawUrl = entry.slice(separator + 1).trim();
|
||||
if (!LOCAL_RECEIVER_BANDS.includes(band))
|
||||
return { band: null, reason: 'band must be 1090 or 978' };
|
||||
let url;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch {
|
||||
return { band, reason: 'not a valid URL' };
|
||||
}
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
||||
return { band, reason: 'scheme must be http or https' };
|
||||
if (url.username || url.password)
|
||||
return { band, reason: 'credentials are not allowed in the URL' };
|
||||
if (url.search || url.hash || rawUrl.includes('?') || rawUrl.includes('#'))
|
||||
return { band, reason: 'query and fragment are not allowed' };
|
||||
const port = url.port ? Number(url.port) : defaultPort(url.protocol);
|
||||
const address = parseTapAddress(`${url.hostname}:${port}`);
|
||||
if (!address)
|
||||
return {
|
||||
band,
|
||||
reason: 'host must be loopback, RFC1918, localhost or *.local',
|
||||
};
|
||||
if (!/(?:^|\/)aircraft\.json$/.test(url.pathname))
|
||||
return { band, reason: 'path must end in aircraft.json' };
|
||||
return {
|
||||
band,
|
||||
url: `${url.protocol}//${address.host}:${address.port}${url.pathname}`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `LOCAL_RECEIVER_FEEDS` (`band=url[,band=url…]`).
|
||||
*
|
||||
* Labels name the band only (plus an ordinal when a band repeats), so feed
|
||||
* addresses never reach the browser.
|
||||
* @param {string|undefined} value Environment value.
|
||||
* @returns {{configured:boolean, feeds:object[]}} Each feed is
|
||||
* `{ id, band, label, url }` when valid or `{ id, band, label, reason }`
|
||||
* when invalid (never fetched).
|
||||
*/
|
||||
export function parseLocalReceiverFeeds(value) {
|
||||
const entries = String(value ?? '')
|
||||
.split(',')
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
const feeds = [];
|
||||
const perBand = new Map();
|
||||
entries.forEach((entry, index) => {
|
||||
const parsed =
|
||||
index < MAX_FEEDS
|
||||
? parseEntry(entry)
|
||||
: { band: null, reason: `at most ${MAX_FEEDS} feeds are read` };
|
||||
const band = parsed.band;
|
||||
const ordinal = (perBand.get(band) || 0) + 1;
|
||||
perBand.set(band, ordinal);
|
||||
const base = band ? BAND_LABELS[band] : `entry ${index + 1}`;
|
||||
const label = band && ordinal > 1 ? `${base} #${ordinal}` : base;
|
||||
feeds.push({
|
||||
id: `feed-${index + 1}`,
|
||||
band,
|
||||
label,
|
||||
...(parsed.url ? { url: parsed.url } : { reason: parsed.reason }),
|
||||
});
|
||||
});
|
||||
// Ordinals are only needed when a band actually repeats.
|
||||
for (const feed of feeds) {
|
||||
if (feed.band && perBand.get(feed.band) > 1 && !feed.label.includes('#'))
|
||||
feed.label = `${feed.label} #1`;
|
||||
}
|
||||
return { configured: feeds.length > 0, feeds };
|
||||
}
|
||||
|
||||
class FeedError extends Error {
|
||||
constructor(code) {
|
||||
super(code);
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a resolved address satisfies the receiver-tap rule: loopback or
|
||||
* RFC1918 IPv4, the IPv6 loopback, or an IPv4-mapped form of the former.
|
||||
* @param {string} address Resolved IP address.
|
||||
* @returns {boolean}
|
||||
*/
|
||||
export function isLocalReceiverAddress(address) {
|
||||
const value = String(address || '').toLowerCase();
|
||||
if (value === '::1') return true;
|
||||
const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(value);
|
||||
return isLocalIpv4(mapped ? mapped[1] : value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a feed host name and require every address to be local.
|
||||
* @param {string} hostname `localhost` or a `*.local` name.
|
||||
* @param {typeof dnsLookup} lookupImpl `dns.promises.lookup`-compatible.
|
||||
* @returns {Promise<Array<{address:string, family:number}>>}
|
||||
*/
|
||||
export async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
|
||||
let resolved;
|
||||
try {
|
||||
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
|
||||
} catch {
|
||||
throw new FeedError('UNRESOLVED');
|
||||
}
|
||||
const addresses = (Array.isArray(resolved) ? resolved : [resolved])
|
||||
.map((row) => ({
|
||||
address: String(row?.address || ''),
|
||||
family:
|
||||
Number(row?.family) || (String(row?.address).includes(':') ? 6 : 4),
|
||||
}))
|
||||
.filter((row) => row.address);
|
||||
if (!addresses.length) throw new FeedError('UNRESOLVED');
|
||||
if (addresses.some((row) => !isLocalReceiverAddress(row.address)))
|
||||
throw new FeedError('FORBIDDEN_ADDRESS');
|
||||
return addresses;
|
||||
}
|
||||
|
||||
/**
|
||||
* Settle with `promise`, or reject with an AbortError as soon as `signal`
|
||||
* aborts; a later settlement of `promise` is ignored.
|
||||
*/
|
||||
function untilAborted(promise, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
const error = new Error('aborted');
|
||||
error.name = 'AbortError';
|
||||
reject(error);
|
||||
};
|
||||
// Handlers are attached first so a late rejection is never unhandled.
|
||||
promise.then(
|
||||
(value) => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
if (signal.aborted) onAbort();
|
||||
else signal.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/** A `net`-style lookup that only ever answers the validated addresses. */
|
||||
function pinnedLookup(addresses) {
|
||||
return (_hostname, options, callback) => {
|
||||
const done = typeof options === 'function' ? options : callback;
|
||||
if (options?.all)
|
||||
done(
|
||||
null,
|
||||
addresses.map((row) => ({ ...row })),
|
||||
);
|
||||
else done(null, addresses[0].address, addresses[0].family);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Default transport. With `options.lookup` (a pinned feed name) the request
|
||||
* is made with node:http(s) so the socket connects only to the validated
|
||||
* addresses (TLS still verifies the name); otherwise it is a plain fetch of
|
||||
* an IP-literal URL. Redirects are never followed either way.
|
||||
* @param {string} url
|
||||
* @param {object} options
|
||||
* @returns {Promise<Response>}
|
||||
*/
|
||||
function fetchLocalReceiverFeed(url, options) {
|
||||
if (typeof options?.lookup !== 'function') return fetch(url, options);
|
||||
const client = new URL(url).protocol === 'https:' ? https : http;
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client.request(
|
||||
url,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: options.headers,
|
||||
signal: options.signal,
|
||||
lookup: options.lookup,
|
||||
// A fresh socket per read: a pooled one could outlive the pin.
|
||||
agent: false,
|
||||
},
|
||||
(response) => {
|
||||
// Nothing thrown here may escape: this callback runs outside the
|
||||
// promise, so an exception would be uncaught and crash the server.
|
||||
try {
|
||||
const status = response.statusCode;
|
||||
// A `Response` only represents final statuses 200–599.
|
||||
if (!Number.isInteger(status) || status < 200 || status > 599)
|
||||
throw new FeedError('HTTP_STATUS');
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
}
|
||||
const nullBody = NULL_BODY_STATUSES.has(status);
|
||||
const upstream = new Response(
|
||||
nullBody ? null : Readable.toWeb(response),
|
||||
{ status, statusText: response.statusMessage || '', headers },
|
||||
);
|
||||
if (nullBody) response.destroy();
|
||||
resolve(upstream);
|
||||
} catch (error) {
|
||||
response.destroy();
|
||||
reject(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one validated feed document without following redirects and with the
|
||||
* body cap enforced while it streams.
|
||||
* @param {string} url Validated feed URL.
|
||||
* @param {object} options
|
||||
* @returns {Promise<object>} Parsed JSON document.
|
||||
*/
|
||||
export async function fetchLocalReceiverDocument(
|
||||
url,
|
||||
{
|
||||
fetchImpl = fetchLocalReceiverFeed,
|
||||
lookupImpl = dnsLookup,
|
||||
timeoutMs = LOCAL_RECEIVER_TIMEOUT_MS,
|
||||
maxBytes = LOCAL_RECEIVER_MAX_BODY_BYTES,
|
||||
} = {},
|
||||
) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
let upstream;
|
||||
try {
|
||||
const { hostname } = new URL(url);
|
||||
// IP literals were validated when the feed list was parsed; a name is
|
||||
// validated by what it resolves to now, and the socket is pinned there.
|
||||
// Resolution runs against the same deadline as the read: a stalled
|
||||
// lookup fails this feed instead of holding the whole snapshot.
|
||||
const lookup = isLocalIpv4(hostname)
|
||||
? undefined
|
||||
: pinnedLookup(
|
||||
await untilAborted(
|
||||
resolveLocalReceiverAddresses(hostname, lookupImpl),
|
||||
controller.signal,
|
||||
),
|
||||
);
|
||||
upstream = await fetchImpl(url, {
|
||||
method: 'GET',
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: controller.signal,
|
||||
...(lookup ? { lookup } : {}),
|
||||
});
|
||||
if (upstream.status >= 300 && upstream.status < 400)
|
||||
throw new FeedError('REDIRECT_REFUSED');
|
||||
// Opaque redirect responses from `redirect: 'manual'` report status 0.
|
||||
if (upstream.type === 'opaqueredirect')
|
||||
throw new FeedError('REDIRECT_REFUSED');
|
||||
if (!upstream.ok) throw new FeedError('HTTP_STATUS');
|
||||
const text = await readResponseTextCapped(
|
||||
upstream,
|
||||
maxBytes,
|
||||
controller.signal,
|
||||
);
|
||||
let json;
|
||||
try {
|
||||
json = JSON.parse(text);
|
||||
} catch {
|
||||
throw new FeedError('BAD_JSON');
|
||||
}
|
||||
if (!json || typeof json !== 'object' || !Array.isArray(json.aircraft))
|
||||
throw new FeedError('BAD_DOCUMENT');
|
||||
return json;
|
||||
} catch (error) {
|
||||
controller.abort();
|
||||
if (upstream?.body && !upstream.body.locked)
|
||||
void upstream.body.cancel().catch(() => {});
|
||||
if (error instanceof FeedError) throw error;
|
||||
if (error?.code === 'RESPONSE_TOO_LARGE')
|
||||
throw new FeedError('RESPONSE_TOO_LARGE');
|
||||
if (error?.name === 'AbortError' || controller.signal.aborted)
|
||||
throw new FeedError('TIMEOUT');
|
||||
throw new FeedError('UNREACHABLE');
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one feed and describe it.
|
||||
* @returns {Promise<{status:object, records:object[]}>}
|
||||
*/
|
||||
async function readFeed(
|
||||
feed,
|
||||
{ fetchImpl, lookupImpl, now, timeoutMs, maxBytes, log },
|
||||
) {
|
||||
const summary = { band: feed.band, label: feed.label };
|
||||
if (!feed.url)
|
||||
return {
|
||||
status: { ...summary, status: 'invalid', aircraft: 0, ageMs: null },
|
||||
records: [],
|
||||
};
|
||||
let json;
|
||||
try {
|
||||
json = await fetchLocalReceiverDocument(feed.url, {
|
||||
fetchImpl,
|
||||
lookupImpl,
|
||||
timeoutMs,
|
||||
maxBytes,
|
||||
});
|
||||
} catch (error) {
|
||||
log(feed, 'unreachable', error.code);
|
||||
return {
|
||||
status: { ...summary, status: 'unreachable', aircraft: 0, ageMs: null },
|
||||
records: [],
|
||||
};
|
||||
}
|
||||
const receivedAt = now();
|
||||
const feedNowS = Number(json.now);
|
||||
// The feed's own clock decides staleness. A clock ahead of ours reads as
|
||||
// age 0; a document without `now` is treated as current.
|
||||
const ageMs = Number.isFinite(feedNowS)
|
||||
? Math.max(0, Math.round(receivedAt - feedNowS * 1000))
|
||||
: 0;
|
||||
const status = ageMs > LOCAL_RECEIVER_STALE_MS ? 'stale' : 'live';
|
||||
log(feed, status);
|
||||
// Anchor record ages at the document time so a stale feed ages out
|
||||
// naturally, without trusting a receiver clock that runs ahead of ours.
|
||||
const records = normalizeDump1090Aircraft(json, receivedAt - ageMs, {
|
||||
band: feed.band,
|
||||
});
|
||||
return {
|
||||
status: { ...summary, status, aircraft: records.length, ageMs },
|
||||
records,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the route handler. Exposed for tests; the plugin wraps it.
|
||||
* @param {object} [options]
|
||||
* @param {string} [options.feedsValue] `LOCAL_RECEIVER_FEEDS` value.
|
||||
* @param {typeof fetch} [options.fetchImpl] Receives `options.lookup` when
|
||||
* the feed host is a name; the connection must resolve only through it.
|
||||
* @param {Function} [options.lookupImpl] `dns.promises.lookup`-compatible.
|
||||
* @param {() => number} [options.now]
|
||||
* @param {{warn:Function, info?:Function}} [options.logger]
|
||||
* @returns {{handle:Function, config:object}}
|
||||
*/
|
||||
export function createLocalReceiversHandler({
|
||||
feedsValue = process.env[LOCAL_RECEIVER_FEEDS_ENV],
|
||||
fetchImpl = fetchLocalReceiverFeed,
|
||||
lookupImpl = dnsLookup,
|
||||
now = Date.now,
|
||||
logger = console,
|
||||
timeoutMs = LOCAL_RECEIVER_TIMEOUT_MS,
|
||||
maxBytes = LOCAL_RECEIVER_MAX_BODY_BYTES,
|
||||
cacheMs = LOCAL_RECEIVER_CACHE_MS,
|
||||
} = {}) {
|
||||
const config = parseLocalReceiverFeeds(feedsValue);
|
||||
for (const feed of config.feeds) {
|
||||
if (feed.reason)
|
||||
logger.warn(
|
||||
`${LOG_PREFIX} ${LOCAL_RECEIVER_FEEDS_ENV} ${feed.id} (${feed.label}) rejected: ${feed.reason}. It will not be fetched.`,
|
||||
);
|
||||
}
|
||||
const lastStatus = new Map();
|
||||
// Log a feed only when its status changes, so a feed that is down does not
|
||||
// write a line every second.
|
||||
const log = (feed, status, code) => {
|
||||
if (lastStatus.get(feed.id) === status) return;
|
||||
lastStatus.set(feed.id, status);
|
||||
if (status === 'live') logger.info?.(`${LOG_PREFIX} ${feed.label} live`);
|
||||
else
|
||||
logger.warn(
|
||||
`${LOG_PREFIX} ${feed.label} ${status}${code ? ` (${code})` : ''}`,
|
||||
);
|
||||
};
|
||||
const inFlight = new Map();
|
||||
let cached = null;
|
||||
|
||||
async function snapshot() {
|
||||
const results = await Promise.all(
|
||||
config.feeds.map((feed) =>
|
||||
readFeed(feed, {
|
||||
fetchImpl,
|
||||
lookupImpl,
|
||||
now,
|
||||
timeoutMs,
|
||||
maxBytes,
|
||||
log,
|
||||
}),
|
||||
),
|
||||
);
|
||||
return {
|
||||
configured: true,
|
||||
generatedAt: now(),
|
||||
feeds: results.map((result) => result.status),
|
||||
records: results.flatMap((result) => result.records),
|
||||
};
|
||||
}
|
||||
|
||||
async function payload() {
|
||||
if (!config.configured)
|
||||
return { configured: false, feeds: [], records: [] };
|
||||
if (cached && now() - cached.at < cacheMs) return cached.payload;
|
||||
const { promise } = coalesceProxyRequest(inFlight, 'all', snapshot);
|
||||
const result = await promise;
|
||||
cached = { at: now(), payload: result };
|
||||
return result;
|
||||
}
|
||||
|
||||
async function handle(req, res) {
|
||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||
res.writeHead(405, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const body = await payload();
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify(body));
|
||||
} catch {
|
||||
res.writeHead(502, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Local receiver feeds unavailable' }));
|
||||
}
|
||||
}
|
||||
|
||||
return { handle, config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Vite plugin mounting `GET /api/local-receivers/aircraft`.
|
||||
*
|
||||
* The environment is read when the server starts, so construction alone
|
||||
* neither parses configuration nor fetches anything.
|
||||
* @param {object} [options] Passed to {@link createLocalReceiversHandler}.
|
||||
* @returns {import('vite').Plugin}
|
||||
*/
|
||||
export function localReceiversProxy(options = {}) {
|
||||
const install = (server) => {
|
||||
const { handle } = createLocalReceiversHandler(options);
|
||||
server.middlewares.use(LOCAL_RECEIVERS_ROUTE, (req, res, next) => {
|
||||
const path = new URL(req.url || '/', 'http://localhost').pathname;
|
||||
if (path !== '/' && path !== '') {
|
||||
if (typeof next === 'function') return next();
|
||||
return undefined;
|
||||
}
|
||||
return handle(req, res);
|
||||
});
|
||||
};
|
||||
return {
|
||||
name: 'local-receivers-proxy',
|
||||
configureServer: install,
|
||||
configurePreviewServer: install,
|
||||
};
|
||||
}
|
||||
@@ -13,16 +13,26 @@ import { cctvProxy } from './cctv.js';
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
import { radioBrowserProxy } from './radio.js';
|
||||
import { gbfsProxy } from './gbfs.js';
|
||||
import { localReceiversProxy } from './local-receivers.js';
|
||||
import { transitProxy } from './transit.js';
|
||||
import { adsbLolProxy } from './aircraft/adsb-lol.js';
|
||||
import { aisLiveProxy } from './vessels/ais-live.js';
|
||||
import { trackBackfillProxies } from './aircraft/tracks.js';
|
||||
import { openAiRealtimeProxy } from './openai.js';
|
||||
import { googlePlacesContextProxy } from './places.js';
|
||||
import { mapillaryProxy } from './mapillary.js';
|
||||
import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
/** Construct the local provider plugins in their established order. */
|
||||
function localProviderPlugins() {
|
||||
/**
|
||||
* Construct the local provider plugins in their established order.
|
||||
* `realtime` configures the voice session token endpoint.
|
||||
*/
|
||||
function localProviderPlugins({ realtime } = {}) {
|
||||
return [
|
||||
openSkyProxy(),
|
||||
celestrakProxy(),
|
||||
@@ -39,12 +49,19 @@ function localProviderPlugins() {
|
||||
cctvProxy({ sourceRoot: defaultSourceRoot }),
|
||||
radioBrowserProxy(),
|
||||
gbfsProxy(),
|
||||
localReceiversProxy(),
|
||||
transitProxy(),
|
||||
adsbLolProxy(),
|
||||
aisLiveProxy(),
|
||||
trackBackfillProxies(),
|
||||
openAiRealtimeProxy(),
|
||||
openAiRealtimeProxy({ realtime }),
|
||||
googlePlacesContextProxy(),
|
||||
windProxy(),
|
||||
weatherProxy(),
|
||||
cycloneProxy(),
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
@@ -66,6 +83,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
|
||||
export { googlePlacesContextProxy } from './places.js';
|
||||
export { googleServerApiKey } from './places.js';
|
||||
export { keylessGooglePlacesResponse } from './places.js';
|
||||
export { keylessGoogleGeocodeResponse } from './places.js';
|
||||
export { validateGeocodeAddress } from './places.js';
|
||||
export { geocodeBounds } from './places.js';
|
||||
export { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
export { readResponseTextCapped } from './common/http.js';
|
||||
export { readResponseJsonCapped } from './common/http.js';
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { installMapillaryRoutes } from './mapillary/routes.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: cached Mapillary coverage tiles (token added server-side) and a
|
||||
* status route that says whether a token is configured.
|
||||
*/
|
||||
function mapillaryProxy() {
|
||||
return {
|
||||
name: 'mapillary-proxy',
|
||||
configureServer(server) {
|
||||
installMapillaryRoutes(server.middlewares);
|
||||
},
|
||||
configurePreviewServer(server) {
|
||||
installMapillaryRoutes(server.middlewares);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export { mapillaryProxy };
|
||||
@@ -0,0 +1,72 @@
|
||||
import path from 'node:path';
|
||||
|
||||
/** Mapillary vector tile root; the path template is /{layer}/2/{z}/{x}/{y}. */
|
||||
export const MAPILLARY_TILE_HOST = 'https://tiles.mapillary.com/maps/vtp';
|
||||
|
||||
/** Public tile layer names this proxy exposes, mapped to Mapillary's ids. */
|
||||
export const TILE_LAYERS = Object.freeze({
|
||||
// Region-sized z6–10 tiles are refused; the app never asks for them. The
|
||||
// z14 `image` layer is ~98% of a tile and unused (positions come from the
|
||||
// graph API), so it is dropped in transit.
|
||||
coverage: Object.freeze({
|
||||
upstream: 'mly1_public',
|
||||
zoomRanges: Object.freeze([Object.freeze([0, 5]), Object.freeze([11, 14])]),
|
||||
dropLayers: Object.freeze(['image']),
|
||||
}),
|
||||
});
|
||||
|
||||
/** Disk cache root, alongside the other providers' caches. */
|
||||
export const MAPILLARY_CACHE_DIR = path.join(
|
||||
process.cwd(),
|
||||
'.gev-cache',
|
||||
'mapillary',
|
||||
);
|
||||
export const TILE_DISK_DIR = path.join(MAPILLARY_CACHE_DIR, 'tiles');
|
||||
|
||||
/**
|
||||
* Longer than the 15-minute upstream Cache-Control: tiles change only when new
|
||||
* imagery is processed, and a day avoids re-downloading a city per camera move.
|
||||
*/
|
||||
export const TILE_DISK_TTL_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* A tile write starts a background sweep at most this often: expired files go
|
||||
* first, then the oldest until the cache fits in TILE_DISK_MAX_BYTES.
|
||||
*/
|
||||
export const TILE_DISK_SWEEP_INTERVAL_MS = 15 * 60 * 1000;
|
||||
export const TILE_DISK_MAX_BYTES = 1024 * 1024 * 1024;
|
||||
/** Per-file cost on top of its size, so empty tiles still count. */
|
||||
export const TILE_DISK_FILE_OVERHEAD_BYTES = 4096;
|
||||
|
||||
/** A z14 image tile over a dense city is ~11 MB; anything past this is wrong. */
|
||||
export const TILE_MAX_BYTES = 48 * 1024 * 1024;
|
||||
|
||||
export const TILE_MEMORY_BUDGET_BYTES = 96 * 1024 * 1024;
|
||||
/** Per-entry cost on top of its bytes, so empty tiles still count. */
|
||||
export const TILE_MEMORY_ENTRY_OVERHEAD_BYTES = 1024;
|
||||
export const TILE_FETCH_TIMEOUT_MS = 60_000;
|
||||
/**
|
||||
* Upstream fetches at once; further misses queue. A view needs at most 25
|
||||
* tiles, and the cap stops a burst of misses opening thousands of requests.
|
||||
*/
|
||||
export const TILE_UPSTREAM_CONCURRENCY = 6;
|
||||
/**
|
||||
* Tile requests per client IP per minute, cache hits included. 600 is 24 new
|
||||
* 25-tile views a minute, more than a person flying the camera reaches, but it
|
||||
* stops one page draining the shared token into a Mapillary rate limit.
|
||||
*/
|
||||
export const TILE_ROUTE_MAX_PER_MIN = 600;
|
||||
/**
|
||||
* How long a 401/403 answers tile misses before asking Mapillary again. A
|
||||
* changed token is tried at once.
|
||||
*/
|
||||
export const TILE_KEY_REJECTED_HOLD_MS = 5 * 60 * 1000;
|
||||
/** After a 429 without a usable Retry-After, hold tile misses this long. */
|
||||
export const TILE_RATE_LIMIT_HOLD_MS = 60_000;
|
||||
/** The longest Retry-After honoured, so a bad header cannot stall coverage. */
|
||||
export const TILE_RATE_LIMIT_MAX_HOLD_MS = 10 * 60 * 1000;
|
||||
|
||||
/** The client token lives in the browser by design; the server adds it to tile URLs too. */
|
||||
export function mapillaryToken() {
|
||||
return String(process.env.MAPILLARY_CLIENT_TOKEN || '').trim();
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { sameSiteGated } from '../common/same-site.js';
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { fetchTile, TileRequestError, TileUpstreamError } from './tiles.js';
|
||||
import { mapillaryToken, TILE_ROUTE_MAX_PER_MIN } from './constants.js';
|
||||
|
||||
function sendJson(res, status, payload) {
|
||||
res.statusCode = status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
/** GET /api/mapillary/status — whether a token is configured, never its value. */
|
||||
function handleStatus(req, res) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
sendJson(res, 200, { configured: Boolean(mapillaryToken()) });
|
||||
}
|
||||
|
||||
/** GET /api/mapillary/tiles/coverage/{z}/{x}/{y} — cached protobuf tile. */
|
||||
async function handleTile(req, res, allow) {
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, 405, { error: 'Method not allowed' });
|
||||
if (!allow(clientKey(req))) {
|
||||
res.setHeader('Retry-After', '5');
|
||||
return sendJson(res, 429, {
|
||||
error: 'Too many tile requests',
|
||||
retryAfter: 5,
|
||||
});
|
||||
}
|
||||
const match = /^\/(coverage)\/(\d{1,2})\/(\d{1,6})\/(\d{1,6})$/.exec(
|
||||
(req.url || '').split('?')[0],
|
||||
);
|
||||
if (!match)
|
||||
return sendJson(res, 400, {
|
||||
error: 'Tile path must be /coverage/{z}/{x}/{y}',
|
||||
});
|
||||
if (!mapillaryToken())
|
||||
return sendJson(res, 503, { error: 'no_key', keyRequired: true });
|
||||
const abandoned = new AbortController();
|
||||
req.on?.('aborted', () => abandoned.abort());
|
||||
res.on?.('close', () => abandoned.abort());
|
||||
try {
|
||||
const { bytes, source } = await fetchTile(
|
||||
{ layer: match[1], z: match[2], x: match[3], y: match[4] },
|
||||
{ signal: abandoned.signal },
|
||||
);
|
||||
if (res.writableEnded) return;
|
||||
res.statusCode = bytes.length ? 200 : 204;
|
||||
res.setHeader('Content-Type', 'application/x-protobuf');
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
res.setHeader('X-Gev-Cache', source);
|
||||
res.end(bytes.length ? bytes : undefined);
|
||||
} catch (error) {
|
||||
if (res.writableEnded || abandoned.signal.aborted) return;
|
||||
if (error instanceof TileRequestError)
|
||||
return sendJson(res, error.status, { error: error.message });
|
||||
if (error instanceof TileUpstreamError) {
|
||||
// A rejected token is a key problem the panel can name, not a fault.
|
||||
if (error.keyRejected)
|
||||
return sendJson(res, 403, {
|
||||
error: 'Mapillary rejected the access token',
|
||||
keyRejected: true,
|
||||
});
|
||||
if (error.status === 429) {
|
||||
const retryAfter = error.retryAfterSec || 60;
|
||||
res.setHeader('Retry-After', String(retryAfter));
|
||||
return sendJson(res, 429, {
|
||||
error: 'Mapillary is rate-limiting tile requests',
|
||||
retryAfter,
|
||||
});
|
||||
}
|
||||
// An upstream 400 must not read as a malformed request to this proxy.
|
||||
return sendJson(res, 502, { error: error.message });
|
||||
}
|
||||
sendJson(res, 502, { error: error?.message || 'Tile fetch failed' });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach the Mapillary routes. Both refuse cross-site requests so another page
|
||||
* cannot spend the token or provoke a 429 that holds every tile miss.
|
||||
*/
|
||||
export function installMapillaryRoutes(middlewares) {
|
||||
const allow = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: TILE_ROUTE_MAX_PER_MIN,
|
||||
globalMax: TILE_ROUTE_MAX_PER_MIN * 4,
|
||||
});
|
||||
middlewares.use('/api/mapillary/status', sameSiteGated(handleStatus));
|
||||
middlewares.use(
|
||||
'/api/mapillary/tiles',
|
||||
sameSiteGated((req, res) => handleTile(req, res, allow)),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { readResponseBytesCapped } from '../common/http.js';
|
||||
import { stripTileLayers } from './trim.js';
|
||||
import {
|
||||
MAPILLARY_TILE_HOST,
|
||||
TILE_LAYERS,
|
||||
TILE_DISK_DIR,
|
||||
TILE_DISK_TTL_MS,
|
||||
TILE_DISK_SWEEP_INTERVAL_MS,
|
||||
TILE_DISK_MAX_BYTES,
|
||||
TILE_DISK_FILE_OVERHEAD_BYTES,
|
||||
TILE_MAX_BYTES,
|
||||
TILE_MEMORY_BUDGET_BYTES,
|
||||
TILE_MEMORY_ENTRY_OVERHEAD_BYTES,
|
||||
TILE_FETCH_TIMEOUT_MS,
|
||||
TILE_UPSTREAM_CONCURRENCY,
|
||||
TILE_KEY_REJECTED_HOLD_MS,
|
||||
TILE_RATE_LIMIT_HOLD_MS,
|
||||
TILE_RATE_LIMIT_MAX_HOLD_MS,
|
||||
mapillaryToken,
|
||||
} from './constants.js';
|
||||
|
||||
/** Log without the stack; the token is never in scope here. */
|
||||
function warn(what, error) {
|
||||
console.warn(`[Mapillary Proxy] ${what}:`, error?.message || error);
|
||||
}
|
||||
|
||||
/** Thrown for a request this proxy refuses before contacting Mapillary. */
|
||||
export class TileRequestError extends Error {
|
||||
constructor(message, status = 400) {
|
||||
super(message);
|
||||
this.name = 'TileRequestError';
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
/** Mapillary answered with an error status; 401/403 set `keyRejected`. */
|
||||
export class TileUpstreamError extends Error {
|
||||
constructor(status, message, { retryAfterSec = null } = {}) {
|
||||
super(message || `Mapillary tiles HTTP ${status}`);
|
||||
this.name = 'TileUpstreamError';
|
||||
this.status = status;
|
||||
this.keyRejected = status === 401 || status === 403;
|
||||
this.retryAfterSec = retryAfterSec;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A rejected token or rate limit that answers every tile miss until it expires
|
||||
* (or the token changes). Cached tiles are still served meanwhile.
|
||||
* @type {{status: number, until: number, token: string}|null}
|
||||
*/
|
||||
let _upstreamHold = null;
|
||||
|
||||
/** Milliseconds to wait from a Retry-After header (seconds or an HTTP date). */
|
||||
function retryAfterMs(header, now = Date.now()) {
|
||||
const text = String(header ?? '').trim();
|
||||
let ms = NaN;
|
||||
if (/^\d+$/.test(text)) ms = Number(text) * 1000;
|
||||
else if (text) ms = Date.parse(text) - now;
|
||||
if (!Number.isFinite(ms) || ms <= 0) return TILE_RATE_LIMIT_HOLD_MS;
|
||||
return Math.min(ms, TILE_RATE_LIMIT_MAX_HOLD_MS);
|
||||
}
|
||||
|
||||
function heldRefusal(token) {
|
||||
const hold = _upstreamHold;
|
||||
if (!hold) return null;
|
||||
const left = hold.until - Date.now();
|
||||
if (left <= 0 || hold.token !== token) {
|
||||
_upstreamHold = null;
|
||||
return null;
|
||||
}
|
||||
return new TileUpstreamError(hold.status, undefined, {
|
||||
retryAfterSec: Math.ceil(left / 1000),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a tile address against the layer's zoom ranges.
|
||||
* @returns {{layer:string, upstream:string, z:number, x:number, y:number, key:string}}
|
||||
*/
|
||||
export function normalizeTileAddress({ layer, z, x, y }) {
|
||||
const spec = Object.hasOwn(TILE_LAYERS, layer) ? TILE_LAYERS[layer] : null;
|
||||
if (!spec) throw new TileRequestError(`Unknown tile layer: ${layer}`);
|
||||
const zi = Number(z);
|
||||
const xi = Number(x);
|
||||
const yi = Number(y);
|
||||
if (
|
||||
![zi, xi, yi].every((v) => Number.isInteger(v) && v >= 0) ||
|
||||
!spec.zoomRanges.some(([min, max]) => zi >= min && zi <= max)
|
||||
)
|
||||
throw new TileRequestError(
|
||||
`Tile zoom for ${layer} must be ${spec.zoomRanges
|
||||
.map(([min, max]) => `${min}–${max}`)
|
||||
.join(' or ')}`,
|
||||
);
|
||||
const n = 2 ** zi;
|
||||
if (xi >= n || yi >= n)
|
||||
throw new TileRequestError('Tile address out of range');
|
||||
return {
|
||||
layer,
|
||||
upstream: spec.upstream,
|
||||
dropLayers: spec.dropLayers || [],
|
||||
z: zi,
|
||||
x: xi,
|
||||
y: yi,
|
||||
key: `${layer}/${zi}/${xi}/${yi}`,
|
||||
};
|
||||
}
|
||||
|
||||
/** @type {Map<string, {bytes: Buffer, at: number}>} insertion-ordered LRU */
|
||||
const _memory = new Map();
|
||||
let _memoryBytes = 0;
|
||||
const memoryCost = (bytes) => bytes.length + TILE_MEMORY_ENTRY_OVERHEAD_BYTES;
|
||||
/**
|
||||
* Upstream fetches shared by every request for the same tile; the last waiter
|
||||
* to leave cancels it (see `joinFlight`).
|
||||
* @type {Map<string, {controller: AbortController, promise: Promise<Buffer>, waiters: number}>}
|
||||
*/
|
||||
const _inFlight = new Map();
|
||||
|
||||
function memoryGet(key) {
|
||||
const hit = _memory.get(key);
|
||||
if (!hit) return null;
|
||||
// Same life as the disk copy, so a long-running server never serves stale
|
||||
// coverage.
|
||||
if (Date.now() - hit.at > TILE_DISK_TTL_MS) {
|
||||
_memory.delete(key);
|
||||
_memoryBytes -= memoryCost(hit.bytes);
|
||||
return null;
|
||||
}
|
||||
_memory.delete(key);
|
||||
_memory.set(key, hit);
|
||||
return hit.bytes;
|
||||
}
|
||||
|
||||
/** `at` is when Mapillary served the bytes (a disk tile's mtime). */
|
||||
function memoryPut(key, bytes, at = Date.now()) {
|
||||
if (memoryCost(bytes) > TILE_MEMORY_BUDGET_BYTES / 2) return;
|
||||
const existing = _memory.get(key);
|
||||
if (existing) _memoryBytes -= memoryCost(existing.bytes);
|
||||
_memory.set(key, { bytes, at });
|
||||
_memoryBytes += memoryCost(bytes);
|
||||
while (_memoryBytes > TILE_MEMORY_BUDGET_BYTES && _memory.size) {
|
||||
const [oldest, entry] = _memory.entries().next().value;
|
||||
_memory.delete(oldest);
|
||||
_memoryBytes -= memoryCost(entry.bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/** Disk cache root; tests point it at a temporary directory. */
|
||||
let _diskDir = TILE_DISK_DIR;
|
||||
|
||||
function diskPath({ layer, z, x, y }, root = _diskDir) {
|
||||
return path.join(root, layer, String(z), `${x}-${y}.pbf`);
|
||||
}
|
||||
|
||||
async function readDisk(address) {
|
||||
const file = diskPath(address);
|
||||
try {
|
||||
const stat = await fsp.stat(file);
|
||||
if (Date.now() - stat.mtimeMs > TILE_DISK_TTL_MS) return null;
|
||||
return { bytes: await fsp.readFile(file), at: stat.mtimeMs };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Unsettled background writes and sweeps; requests never wait on them. */
|
||||
const _background = new Set();
|
||||
|
||||
function track(task) {
|
||||
const settled = task
|
||||
.catch(() => {})
|
||||
.finally(() => _background.delete(settled));
|
||||
_background.add(settled);
|
||||
return settled;
|
||||
}
|
||||
|
||||
/** @type {Map<string, number>} cache file to its newest write's sequence */
|
||||
const _diskWrites = new Map();
|
||||
let _diskWriteSeq = 0;
|
||||
|
||||
function diskWritePending(address) {
|
||||
return _diskWrites.has(diskPath(address));
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a tile in the background. `at` keeps a rewrite's original fetch time
|
||||
* so it does not extend the tile's life. An overtaken write steps aside.
|
||||
*/
|
||||
function writeDisk(address, bytes, at = null) {
|
||||
// Sweep this write's root even if the cache dir has moved since.
|
||||
const root = _diskDir;
|
||||
const file = diskPath(address, root);
|
||||
const seq = ++_diskWriteSeq;
|
||||
const tmp = `${file}.${process.pid}-${seq}.tmp`;
|
||||
const newest = () => _diskWrites.get(file) === seq;
|
||||
_diskWrites.set(file, seq);
|
||||
return track(
|
||||
fsp
|
||||
.mkdir(path.dirname(file), { recursive: true })
|
||||
.then(() => fsp.writeFile(tmp, bytes))
|
||||
.then(() => at != null && fsp.utimes(tmp, new Date(), new Date(at)))
|
||||
.then(() => {
|
||||
if (!newest()) return fsp.rm(tmp, { force: true });
|
||||
return fsp.rename(tmp, file).then(() => scheduleSweep(root));
|
||||
})
|
||||
.catch(async (error) => {
|
||||
await fsp.rm(tmp, { force: true }).catch(() => {});
|
||||
warn('tile cache write failed', error);
|
||||
})
|
||||
.finally(() => {
|
||||
if (newest()) _diskWrites.delete(file);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
let _lastSweepAt = 0;
|
||||
let _sweeping = null;
|
||||
let _sweepWarned = false;
|
||||
|
||||
function scheduleSweep(root) {
|
||||
if (_sweeping || Date.now() - _lastSweepAt < TILE_DISK_SWEEP_INTERVAL_MS)
|
||||
return;
|
||||
_lastSweepAt = Date.now();
|
||||
const sweep = sweepTileDisk({ root })
|
||||
.catch((error) => {
|
||||
if (_sweepWarned) return;
|
||||
_sweepWarned = true;
|
||||
warn('tile cache sweep failed', error);
|
||||
})
|
||||
.finally(() => {
|
||||
if (_sweeping === tracked) _sweeping = null;
|
||||
});
|
||||
const tracked = track(sweep);
|
||||
_sweeping = tracked;
|
||||
}
|
||||
|
||||
/** Every file under `dir` with size and mtime; a missing dir is empty. */
|
||||
async function listCacheFiles(dir) {
|
||||
let entries;
|
||||
try {
|
||||
entries = await fsp.readdir(dir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return [];
|
||||
throw error;
|
||||
}
|
||||
const files = [];
|
||||
for (const entry of entries) {
|
||||
const file = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) files.push(...(await listCacheFiles(file)));
|
||||
else if (entry.isFile()) {
|
||||
const stat = await fsp.stat(file).catch(() => null);
|
||||
if (stat) files.push({ file, size: stat.size, at: stat.mtimeMs });
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete expired tiles, then the oldest until the rest fit in `maxBytes`.
|
||||
* @returns {Promise<{removed: number, bytes: number}>} files deleted, charged bytes kept
|
||||
*/
|
||||
export async function sweepTileDisk({
|
||||
root = _diskDir,
|
||||
maxBytes = TILE_DISK_MAX_BYTES,
|
||||
} = {}) {
|
||||
const now = Date.now();
|
||||
const files = (await listCacheFiles(root)).sort((a, b) => a.at - b.at);
|
||||
const cost = (size) => size + TILE_DISK_FILE_OVERHEAD_BYTES;
|
||||
let bytes = files.reduce((sum, { size }) => sum + cost(size), 0);
|
||||
let removed = 0;
|
||||
for (const { file, size, at } of files) {
|
||||
if (now - at <= TILE_DISK_TTL_MS && bytes <= maxBytes) break;
|
||||
await fsp.rm(file, { force: true });
|
||||
bytes -= cost(size);
|
||||
removed++;
|
||||
}
|
||||
return { removed, bytes };
|
||||
}
|
||||
|
||||
/** The one origin the tile token may be sent to, at every redirect hop. */
|
||||
const TILE_ORIGIN = new URL(MAPILLARY_TILE_HOST).origin;
|
||||
export const TILE_MAX_REDIRECTS = 3;
|
||||
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
/**
|
||||
* Follow redirects by hand so every hop stays HTTPS on the tile origin and the
|
||||
* token in the query never reaches another host. A bad redirect is a 502.
|
||||
*/
|
||||
async function fetchPinned(url, init) {
|
||||
let next = url;
|
||||
for (let hop = 0; ; hop++) {
|
||||
const response = await fetch(next, { ...init, redirect: 'manual' });
|
||||
if (!REDIRECT_STATUSES.has(response.status)) return response;
|
||||
await response.body?.cancel().catch(() => {});
|
||||
if (hop >= TILE_MAX_REDIRECTS)
|
||||
throw new TileUpstreamError(502, 'Mapillary tile redirected too often');
|
||||
let target = null;
|
||||
try {
|
||||
target = new URL(response.headers.get('location') ?? '', next);
|
||||
} catch {
|
||||
/* unparseable Location */
|
||||
}
|
||||
if (
|
||||
!response.headers.get('location') ||
|
||||
target?.protocol !== 'https:' ||
|
||||
target.origin !== TILE_ORIGIN ||
|
||||
target.username ||
|
||||
target.password
|
||||
)
|
||||
throw new TileUpstreamError(
|
||||
502,
|
||||
'Mapillary tile redirect left the tile origin',
|
||||
);
|
||||
next = target.href;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchUpstream(address, signal) {
|
||||
const token = mapillaryToken();
|
||||
if (!token) throw new TileRequestError('Mapillary token not configured', 503);
|
||||
const held = heldRefusal(token);
|
||||
if (held) throw held;
|
||||
const url = `${MAPILLARY_TILE_HOST}/${address.upstream}/2/${address.z}/${address.x}/${address.y}?access_token=${encodeURIComponent(token)}`;
|
||||
const timeout = AbortSignal.timeout(TILE_FETCH_TIMEOUT_MS);
|
||||
const oversize = new AbortController();
|
||||
const response = await fetchPinned(url, {
|
||||
signal: AbortSignal.any([signal, timeout, oversize.signal].filter(Boolean)),
|
||||
headers: { Accept: 'application/x-protobuf' },
|
||||
});
|
||||
if (response.status === 404 || response.status === 204) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
const error = new TileUpstreamError(response.status);
|
||||
if (error.keyRejected)
|
||||
_upstreamHold = {
|
||||
status: response.status,
|
||||
until: Date.now() + TILE_KEY_REJECTED_HOLD_MS,
|
||||
token,
|
||||
};
|
||||
else if (response.status === 429) {
|
||||
const wait = retryAfterMs(response.headers.get('retry-after'));
|
||||
_upstreamHold = { status: 429, until: Date.now() + wait, token };
|
||||
error.retryAfterSec = Math.ceil(wait / 1000);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// Chunked or compressed bodies have no Content-Length: the cap applies as
|
||||
// the body streams, and an oversized one also aborts the request.
|
||||
try {
|
||||
const bytes = await readResponseBytesCapped(
|
||||
response,
|
||||
TILE_MAX_BYTES,
|
||||
signal,
|
||||
);
|
||||
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
} catch (error) {
|
||||
if (error?.code !== 'RESPONSE_TOO_LARGE') throw error;
|
||||
oversize.abort();
|
||||
throw new TileUpstreamError(502, 'Mapillary tile exceeds size cap');
|
||||
}
|
||||
}
|
||||
|
||||
/** @type {{active: number, queue: Array<() => void>}} upstream slots */
|
||||
let _slots = { active: 0, queue: [] };
|
||||
|
||||
/**
|
||||
* Resolve with a release function once a slot is free. An abort while
|
||||
* queued leaves the queue.
|
||||
* @returns {Promise<() => void>}
|
||||
*/
|
||||
function acquireUpstreamSlot(signal) {
|
||||
const slots = _slots;
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal.aborted) return reject(signal.reason);
|
||||
const onAbort = () => {
|
||||
const index = slots.queue.indexOf(grant);
|
||||
if (index !== -1) slots.queue.splice(index, 1);
|
||||
reject(signal.reason);
|
||||
};
|
||||
function grant() {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
slots.active++;
|
||||
let released = false;
|
||||
resolve(() => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
slots.active--;
|
||||
slots.queue.shift()?.();
|
||||
});
|
||||
}
|
||||
if (slots.active < TILE_UPSTREAM_CONCURRENCY) return grant();
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
slots.queue.push(grant);
|
||||
});
|
||||
}
|
||||
|
||||
function trim(address, bytes) {
|
||||
if (!address.dropLayers.length || !bytes.length) return bytes;
|
||||
try {
|
||||
return stripTileLayers(bytes, address.dropLayers);
|
||||
} catch (error) {
|
||||
warn('tile trim failed, serving raw', error);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one tile from memory, disk, a shared flight, then Mapillary. Empty
|
||||
* bytes mean no coverage.
|
||||
* @param {{layer:string,z:number|string,x:number|string,y:number|string}} request
|
||||
* @param {{signal?: AbortSignal}} [options]
|
||||
* @returns {Promise<{bytes: Buffer, source: 'memory'|'disk'|'inflight'|'upstream', address: object}>}
|
||||
*/
|
||||
export async function fetchTile(request, { signal } = {}) {
|
||||
const address = normalizeTileAddress(request);
|
||||
const memory = memoryGet(address.key);
|
||||
if (memory) return { bytes: memory, source: 'memory', address };
|
||||
const disk = await readDisk(address);
|
||||
if (disk) {
|
||||
// Older cache files may be untrimmed: rewrite once, keeping the fetch time.
|
||||
const bytes = trim(address, disk.bytes);
|
||||
if (bytes !== disk.bytes && !diskWritePending(address))
|
||||
writeDisk(address, bytes, disk.at);
|
||||
memoryPut(address.key, bytes, disk.at);
|
||||
return { bytes, source: 'disk', address };
|
||||
}
|
||||
// A caller gone while the disk was read must not start a fetch nobody joins.
|
||||
signal?.throwIfAborted();
|
||||
let flight = _inFlight.get(address.key);
|
||||
const joined = Boolean(flight) && !flight.controller.signal.aborted;
|
||||
if (!joined) flight = startFlight(address);
|
||||
const bytes = await joinFlight(flight, signal);
|
||||
return { bytes, source: joined ? 'inflight' : 'upstream', address };
|
||||
}
|
||||
|
||||
/** Start the shared upstream fetch; it holds a slot until the body is read. */
|
||||
function startFlight(address) {
|
||||
const controller = new AbortController();
|
||||
const flight = { controller, waiters: 0, promise: null };
|
||||
flight.promise = acquireUpstreamSlot(controller.signal)
|
||||
.then((release) =>
|
||||
fetchUpstream(address, controller.signal).finally(release),
|
||||
)
|
||||
.then((raw) => trim(address, raw))
|
||||
.then((bytes) => {
|
||||
memoryPut(address.key, bytes);
|
||||
writeDisk(address, bytes);
|
||||
return bytes;
|
||||
})
|
||||
.finally(() => {
|
||||
if (_inFlight.get(address.key) === flight) _inFlight.delete(address.key);
|
||||
});
|
||||
// Every waiter may have left before it settles; nobody awaits it then.
|
||||
flight.promise.catch(() => {});
|
||||
_inFlight.set(address.key, flight);
|
||||
return flight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait on a shared flight. A caller's abort rejects only that caller; the
|
||||
* fetch is cancelled when the last waiter leaves.
|
||||
*/
|
||||
function joinFlight(flight, signal) {
|
||||
signal?.throwIfAborted();
|
||||
flight.waiters++;
|
||||
return new Promise((resolve, reject) => {
|
||||
let done = false;
|
||||
const leave = () => {
|
||||
if (done) return false;
|
||||
done = true;
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
flight.waiters--;
|
||||
return true;
|
||||
};
|
||||
const onAbort = () => {
|
||||
if (!leave()) return;
|
||||
if (flight.waiters === 0) flight.controller.abort();
|
||||
reject(signal.reason);
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
flight.promise.then(
|
||||
(bytes) => leave() && resolve(bytes),
|
||||
(error) => leave() && reject(error),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Test seam: forget every cached tile held in memory. */
|
||||
export function _resetTileMemoryForTest() {
|
||||
_memory.clear();
|
||||
_memoryBytes = 0;
|
||||
_inFlight.clear();
|
||||
_upstreamHold = null;
|
||||
_slots = { active: 0, queue: [] };
|
||||
}
|
||||
|
||||
/** Test seam: how many tiles memory holds and the bytes charged for them. */
|
||||
export function _tileMemoryForTest() {
|
||||
return { entries: _memory.size, bytes: _memoryBytes };
|
||||
}
|
||||
|
||||
/** Test seam: wait for every background write and the sweeps they start. */
|
||||
export async function _settleTileWritesForTest() {
|
||||
while (_background.size) await Promise.allSettled([..._background]);
|
||||
}
|
||||
|
||||
/** Test seam: keep the disk cache in `dir` (null restores the default). */
|
||||
export function _setTileCacheDirForTest(dir) {
|
||||
_diskDir = dir || TILE_DISK_DIR;
|
||||
_lastSweepAt = 0;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { PbfReader, PbfWriter } from 'pbf';
|
||||
|
||||
/** Field number of `layers` in a Mapbox Vector Tile, and of `name` in a layer. */
|
||||
const TILE_LAYER_FIELD = 3;
|
||||
const LAYER_NAME_FIELD = 1;
|
||||
|
||||
/** Read a layer's name without touching its features. */
|
||||
function layerName(layerBytes) {
|
||||
const pbf = new PbfReader(layerBytes);
|
||||
let name = '';
|
||||
pbf.readFields((tag, _result, reader) => {
|
||||
if (tag !== LAYER_NAME_FIELD) return;
|
||||
name = reader.readString();
|
||||
reader.pos = reader.length; // the name is all we need
|
||||
}, null);
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop whole layers from a vector tile without decoding kept layers, so a
|
||||
* 12 MB tile trims in milliseconds. Returns the input when nothing was dropped.
|
||||
* @param {Buffer|Uint8Array} bytes
|
||||
* @param {Iterable<string>} dropNames
|
||||
*/
|
||||
export function stripTileLayers(bytes, dropNames) {
|
||||
const drop = new Set(dropNames || []);
|
||||
if (!bytes?.length || !drop.size) return bytes;
|
||||
const reader = new PbfReader(bytes);
|
||||
const writer = new PbfWriter();
|
||||
let dropped = false;
|
||||
reader.readFields((tag, _result, pbf) => {
|
||||
if (tag !== TILE_LAYER_FIELD) return;
|
||||
const end = pbf.readVarint() + pbf.pos;
|
||||
const layerBytes = bytes.subarray(pbf.pos, end);
|
||||
pbf.pos = end;
|
||||
if (drop.has(layerName(layerBytes))) {
|
||||
dropped = true;
|
||||
return;
|
||||
}
|
||||
writer.writeBytesField(TILE_LAYER_FIELD, layerBytes);
|
||||
}, null);
|
||||
return dropped ? Buffer.from(writer.finish()) : bytes;
|
||||
}
|
||||
|
||||
/** Names of the layers inside a vector tile, in order. */
|
||||
export function listTileLayers(bytes) {
|
||||
const names = [];
|
||||
if (!bytes?.length) return names;
|
||||
new PbfReader(bytes).readFields((tag, _result, pbf) => {
|
||||
if (tag !== TILE_LAYER_FIELD) return;
|
||||
const end = pbf.readVarint() + pbf.pos;
|
||||
names.push(layerName(bytes.subarray(pbf.pos, end)));
|
||||
pbf.pos = end;
|
||||
}, null);
|
||||
return names;
|
||||
}
|
||||
@@ -5,7 +5,14 @@ import {
|
||||
MILITARY_INSTALLATION_DISK_TTL_MS,
|
||||
MILITARY_INSTALLATION_STALE_MS,
|
||||
} from './military-installations/constants.js';
|
||||
import { fetchOverpassPayload } from './overpass/transport.js';
|
||||
import {
|
||||
fetchOverpassPayload,
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import {
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
} from './overpass/constants.js';
|
||||
import {
|
||||
_militaryInstallationCache,
|
||||
trimMilitaryInstallationCache,
|
||||
@@ -45,6 +52,7 @@ function militaryInstallationsProxy() {
|
||||
throw Object.assign(
|
||||
new Error('Mapped installation upstream unavailable'),
|
||||
{
|
||||
retryAfterMs: upstream.retryAfterMs,
|
||||
installationReason: upstream.rateLimited
|
||||
? 'rate_limited'
|
||||
: upstream.status === 504
|
||||
@@ -69,7 +77,13 @@ function militaryInstallationsProxy() {
|
||||
retrievedAt: new Date().toISOString(),
|
||||
status: 'ready',
|
||||
};
|
||||
const entry = { payload, cachedAt: Date.now() };
|
||||
// Tagged with the configured upstream set so the disk layer does not serve
|
||||
// this answer back after the operator repoints (see overpass/cache.js).
|
||||
const entry = {
|
||||
payload,
|
||||
cachedAt: Date.now(),
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
};
|
||||
_militaryInstallationCache.set(key, entry);
|
||||
trimMilitaryInstallationCache();
|
||||
writeMilitaryInstallationDisk(key, entry);
|
||||
@@ -83,14 +97,6 @@ function militaryInstallationsProxy() {
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
return;
|
||||
}
|
||||
if (!_militaryInstallationsRateLimiter(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '5',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
const requested = validMilitaryInstallationBox(url.searchParams);
|
||||
if (!requested) {
|
||||
@@ -117,6 +123,36 @@ function militaryInstallationsProxy() {
|
||||
: militaryInstallationCacheKey(box);
|
||||
const now = Date.now();
|
||||
const cached = _militaryInstallationCache.get(key);
|
||||
if (!resolveOverpassUpstreams().length) {
|
||||
const stale =
|
||||
cached || (await readMilitaryInstallationDisk(key, Infinity));
|
||||
const disabled = overpassNotConfigured();
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-Military-Installations': stale ? 'STALE' : 'DISABLED',
|
||||
});
|
||||
res.end(
|
||||
stale
|
||||
? JSON.stringify({
|
||||
...stale.payload,
|
||||
status: 'stale',
|
||||
retrievedAt:
|
||||
stale.payload.retrievedAt ||
|
||||
new Date(stale.cachedAt).toISOString(),
|
||||
})
|
||||
: disabled.body,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!_militaryInstallationsRateLimiter(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '5',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
const preflight = await resolveMilitaryInstallationTier({
|
||||
cacheKey: key,
|
||||
memoryCache: _militaryInstallationCache,
|
||||
@@ -176,6 +212,9 @@ function militaryInstallationsProxy() {
|
||||
return;
|
||||
}
|
||||
res.writeHead(503, {
|
||||
...(error?.retryAfterMs
|
||||
? { 'Retry-After': String(Math.ceil(error.retryAfterMs / 1000)) }
|
||||
: {}),
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
@@ -183,6 +222,7 @@ function militaryInstallationsProxy() {
|
||||
JSON.stringify({
|
||||
error: 'Mapped installation context is temporarily unavailable',
|
||||
reason: militaryInstallationFailureReason(error),
|
||||
retryAfterMs: error?.retryAfterMs,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
MILITARY_INSTALLATION_DISK_DIR,
|
||||
MILITARY_INSTALLATION_MAX_CACHE,
|
||||
} from './constants.js';
|
||||
import { cachedByConfiguredUpstream } from '../overpass/constants.js';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
@@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk(
|
||||
const entry = JSON.parse(
|
||||
await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'),
|
||||
);
|
||||
// Same reason as the Overpass disk cache: the key is the bbox alone, so an
|
||||
// entry written against a different OVERPASS_UPSTREAMS is otherwise a hit.
|
||||
if (!cachedByConfiguredUpstream(entry)) return null;
|
||||
if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null;
|
||||
return migrateMilitaryInstallationEntry(entry);
|
||||
} catch {
|
||||
|
||||
@@ -1,30 +1,51 @@
|
||||
import { defaultSourceRoot } from './common/source-root.js';
|
||||
import { handleHudSummary } from './openai/hud-summary.js';
|
||||
import { createDebugLogHandler } from './openai/debug-log.js';
|
||||
import { createRealtimeTokenHandler } from './openai/realtime.js';
|
||||
import {
|
||||
createRealtimeOAuthLoginHandler,
|
||||
createRealtimeOAuthStatusHandler,
|
||||
createRealtimeTokenHandler,
|
||||
} from './openai/realtime.js';
|
||||
import { sameSiteGated } from './common/same-site.js';
|
||||
import { createCodexOAuthLogin } from './openai/codex-auth.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: OpenAI Realtime ephemeral client secret.
|
||||
*
|
||||
* Keeps OPENAI_API_KEY server-side while the browser connects to the
|
||||
* Realtime API over WebRTC with a short-lived secret.
|
||||
* Keeps the selected cloud credential server-side while the browser connects
|
||||
* to the Realtime API over WebRTC with a short-lived secret.
|
||||
*/
|
||||
function openAiRealtimeProxy({
|
||||
sourceRoot = defaultSourceRoot,
|
||||
annotationGuidance,
|
||||
realtime = {},
|
||||
} = {}) {
|
||||
const oauthLogin = createCodexOAuthLogin(realtime);
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/openai/hud-summary', handleHudSummary);
|
||||
// Cost-bearing and log endpoints refuse cross-site browser requests
|
||||
// (see server/providers/common/same-site.js and SECURITY.md).
|
||||
middlewares.use('/api/openai/hud-summary', sameSiteGated(handleHudSummary));
|
||||
|
||||
middlewares.use(
|
||||
'/api/realtime/debug-log',
|
||||
createDebugLogHandler({ sourceRoot }),
|
||||
sameSiteGated(createDebugLogHandler({ sourceRoot })),
|
||||
);
|
||||
|
||||
middlewares.use(
|
||||
'/api/realtime/token',
|
||||
createRealtimeTokenHandler({ ...realtime, annotationGuidance }),
|
||||
sameSiteGated(
|
||||
createRealtimeTokenHandler({ ...realtime, annotationGuidance }),
|
||||
),
|
||||
);
|
||||
|
||||
middlewares.use(
|
||||
'/api/realtime/oauth-status',
|
||||
sameSiteGated(createRealtimeOAuthStatusHandler({ oauthLogin })),
|
||||
);
|
||||
|
||||
middlewares.use(
|
||||
'/api/realtime/oauth-login',
|
||||
sameSiteGated(createRealtimeOAuthLoginHandler({ oauthLogin })),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -32,10 +53,13 @@ function openAiRealtimeProxy({
|
||||
name: 'openai-realtime-proxy',
|
||||
configureServer(server) {
|
||||
install(server.middlewares);
|
||||
server.httpServer?.once('close', oauthLogin.dispose);
|
||||
},
|
||||
configurePreviewServer(server) {
|
||||
install(server.middlewares);
|
||||
server.httpServer?.once('close', oauthLogin.dispose);
|
||||
},
|
||||
closeBundle: oauthLogin.dispose,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import {
|
||||
accessSync,
|
||||
constants as fsConstants,
|
||||
readFileSync,
|
||||
statSync,
|
||||
} from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
export function codexAuthPath({
|
||||
environment = process.env,
|
||||
home = os.homedir(),
|
||||
} = {}) {
|
||||
const explicit = String(environment.CODEX_AUTH_JSON || '').trim();
|
||||
if (explicit) return path.resolve(explicit);
|
||||
const codexHome = String(environment.CODEX_HOME || '').trim();
|
||||
return path.join(
|
||||
codexHome ? path.resolve(codexHome) : path.join(home, '.codex'),
|
||||
'auth.json',
|
||||
);
|
||||
}
|
||||
|
||||
/** An optional operator lock that refuses the metered API-key voice lane. */
|
||||
export function preferCodexOAuth(environment = process.env) {
|
||||
if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false;
|
||||
const value = String(environment.GEV_PREFER_CODEX_OAUTH || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (['1', 'true', 'yes', 'on'].includes(value)) return true;
|
||||
if (['0', 'false', 'no', 'off'].includes(value)) return false;
|
||||
throw new Error(
|
||||
'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.',
|
||||
);
|
||||
}
|
||||
|
||||
function codexOAuthError(message, code = 'CODEX_OAUTH_UNAVAILABLE') {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
|
||||
export function readCodexOAuthAccessToken({
|
||||
authPath = codexAuthPath(),
|
||||
readFile = readFileSync,
|
||||
statFile = statSync,
|
||||
nowMs = Date.now(),
|
||||
} = {}) {
|
||||
const unavailable =
|
||||
'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.';
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(readFile(authPath, 'utf8'));
|
||||
} catch {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
const mode = String(parsed.auth_mode || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const chatGptMode = mode
|
||||
? mode === 'chatgpt' || mode === 'chatgptauthtokens'
|
||||
: typeof parsed.OPENAI_API_KEY !== 'string';
|
||||
const token = parsed.tokens?.access_token;
|
||||
if (!chatGptMode || typeof token !== 'string' || !token.trim()) {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
|
||||
// This only checks token shape and expiry. OpenAI verifies the signature when
|
||||
// minting the short-lived Realtime credential; we never modify auth.json.
|
||||
const parts = token.split('.');
|
||||
if (
|
||||
parts.length !== 3 ||
|
||||
parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part))
|
||||
) {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
|
||||
} catch {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
let expiresAtMs;
|
||||
if (Object.hasOwn(payload, 'exp')) {
|
||||
if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
expiresAtMs = payload.exp * 1000;
|
||||
} else {
|
||||
try {
|
||||
expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000;
|
||||
} catch {
|
||||
throw codexOAuthError(unavailable);
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) {
|
||||
throw codexOAuthError(
|
||||
'ChatGPT sign-in has expired. Re-authenticate in Provider Settings or run codex login and try again.',
|
||||
'CODEX_OAUTH_REAUTH_REQUIRED',
|
||||
);
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
export function resolveCodexExecutable({
|
||||
environment = process.env,
|
||||
home = os.homedir(),
|
||||
access = accessSync,
|
||||
} = {}) {
|
||||
const explicit = String(environment.CODEX_BIN || '').trim();
|
||||
if (explicit) return explicit;
|
||||
|
||||
for (const candidate of [
|
||||
path.join(home, '.local', 'bin', 'codex'),
|
||||
path.join(
|
||||
home,
|
||||
'.codex',
|
||||
'packages',
|
||||
'standalone',
|
||||
'current',
|
||||
'bin',
|
||||
'codex',
|
||||
),
|
||||
]) {
|
||||
try {
|
||||
access(candidate, fsConstants.X_OK);
|
||||
return candidate;
|
||||
} catch {
|
||||
// Fall through to the next known location, then PATH.
|
||||
}
|
||||
}
|
||||
return 'codex';
|
||||
}
|
||||
|
||||
// What `codex login` needs to find its home, open a browser and reach OpenAI.
|
||||
// The server's own environment also holds every provider secret, so the
|
||||
// child receives only these names.
|
||||
const CODEX_LOGIN_ENV =
|
||||
/^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
|
||||
|
||||
/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
|
||||
export function codexLoginEnvironment(environment, codexHome) {
|
||||
const env = {};
|
||||
for (const [name, value] of Object.entries(environment || {}))
|
||||
if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
|
||||
env.CODEX_HOME = codexHome;
|
||||
return env;
|
||||
}
|
||||
|
||||
export function startCodexChatGptLogin({
|
||||
spawnImpl = spawn,
|
||||
environment = process.env,
|
||||
home = os.homedir(),
|
||||
executable = resolveCodexExecutable({ environment, home }),
|
||||
} = {}) {
|
||||
// Codex owns its credential storage. An explicit read path must agree with
|
||||
// the CLI's write path; never silently sign in to another credential store.
|
||||
const configuredHome = String(environment.CODEX_HOME || '').trim();
|
||||
const loginHome = configuredHome
|
||||
? path.resolve(configuredHome)
|
||||
: path.join(home, '.codex');
|
||||
if (
|
||||
codexAuthPath({ environment, home }) !== path.join(loginHome, 'auth.json')
|
||||
) {
|
||||
return Promise.reject(
|
||||
codexOAuthError(
|
||||
'CODEX_AUTH_JSON does not match the Codex login storage path. Set CODEX_HOME to the matching directory or sign in to Codex separately.',
|
||||
'CODEX_OAUTH_STORAGE_MISMATCH',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let child;
|
||||
let spawned = false;
|
||||
let finish;
|
||||
const completion = new Promise((done) => {
|
||||
finish = done;
|
||||
});
|
||||
const fail = () => {
|
||||
reject(
|
||||
codexOAuthError(
|
||||
'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
|
||||
'CODEX_OAUTH_LOGIN_FAILED',
|
||||
),
|
||||
);
|
||||
finish({ exitCode: null });
|
||||
};
|
||||
try {
|
||||
child = spawnImpl(executable, ['login'], {
|
||||
cwd: home,
|
||||
detached: true,
|
||||
stdio: 'ignore',
|
||||
env: codexLoginEnvironment(environment, loginHome),
|
||||
});
|
||||
} catch {
|
||||
fail();
|
||||
return;
|
||||
}
|
||||
|
||||
child.once('error', () => {
|
||||
if (!spawned) fail();
|
||||
else finish({ exitCode: null });
|
||||
});
|
||||
child.once('exit', (exitCode) => finish({ exitCode }));
|
||||
child.once('spawn', () => {
|
||||
spawned = true;
|
||||
child.unref?.();
|
||||
resolve({
|
||||
completion,
|
||||
cancel() {
|
||||
try {
|
||||
child.kill?.('SIGTERM');
|
||||
} catch {
|
||||
/* Already stopped. */
|
||||
}
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** One local login owner shared by the launch and status endpoints. */
|
||||
export function createCodexOAuthLogin({
|
||||
resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
|
||||
startOAuthLogin = () => startCodexChatGptLogin(),
|
||||
loginTimeoutMs = 120_000,
|
||||
setTimer = setTimeout,
|
||||
clearTimer = clearTimeout,
|
||||
} = {}) {
|
||||
let active = null;
|
||||
let failure = null;
|
||||
let disposed = false;
|
||||
|
||||
function credentials() {
|
||||
try {
|
||||
if (resolveOAuthAccessToken()) return { available: true };
|
||||
} catch (error) {
|
||||
const code = error?.code || 'CODEX_OAUTH_UNAVAILABLE';
|
||||
return {
|
||||
available: false,
|
||||
error: error?.message || 'ChatGPT sign-in is unavailable',
|
||||
code,
|
||||
reauthRequired: code === 'CODEX_OAUTH_REAUTH_REQUIRED',
|
||||
};
|
||||
}
|
||||
return {
|
||||
available: false,
|
||||
error: 'ChatGPT sign-in is unavailable',
|
||||
code: 'CODEX_OAUTH_UNAVAILABLE',
|
||||
reauthRequired: false,
|
||||
};
|
||||
}
|
||||
|
||||
function status() {
|
||||
const current = credentials();
|
||||
if (current.available) return current;
|
||||
if (failure) return { available: false, ...failure, loginFailed: true };
|
||||
return active ? { ...current, pending: true } : current;
|
||||
}
|
||||
|
||||
function finish(attempt, error) {
|
||||
if (active !== attempt) return;
|
||||
clearTimer(attempt.timer);
|
||||
active = null;
|
||||
failure = error;
|
||||
}
|
||||
|
||||
async function start() {
|
||||
if (disposed)
|
||||
throw codexOAuthError(
|
||||
'ChatGPT sign-in is unavailable because the server has stopped.',
|
||||
'CODEX_OAUTH_LOGIN_FAILED',
|
||||
);
|
||||
if (credentials().available)
|
||||
return { available: true, started: false, pending: false };
|
||||
if (active) return { available: false, started: false, pending: true };
|
||||
|
||||
// Reserve the attempt before awaiting spawn so concurrent POSTs share it.
|
||||
const attempt = { process: null, timer: null };
|
||||
active = attempt;
|
||||
failure = null;
|
||||
attempt.timer = setTimer(() => {
|
||||
finish(attempt, {
|
||||
code: 'CODEX_OAUTH_LOGIN_TIMEOUT',
|
||||
error:
|
||||
'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.',
|
||||
});
|
||||
attempt.process?.cancel();
|
||||
}, loginTimeoutMs);
|
||||
attempt.timer?.unref?.();
|
||||
|
||||
try {
|
||||
const login = await startOAuthLogin();
|
||||
if (!login?.completion || typeof login.cancel !== 'function')
|
||||
throw new Error('Invalid login process');
|
||||
attempt.process = login;
|
||||
if (active !== attempt) {
|
||||
login.cancel();
|
||||
return { ...status(), started: false, pending: false };
|
||||
}
|
||||
Promise.resolve(login.completion).then(
|
||||
(result) => {
|
||||
if (active !== attempt) return;
|
||||
if (result?.exitCode !== 0) {
|
||||
finish(attempt, {
|
||||
code: 'CODEX_OAUTH_LOGIN_FAILED',
|
||||
error:
|
||||
'ChatGPT sign-in did not complete. Try again in Provider Settings, or run codex login separately.',
|
||||
});
|
||||
} else if (!credentials().available) {
|
||||
finish(attempt, {
|
||||
code: 'CODEX_OAUTH_STORAGE_UNAVAILABLE',
|
||||
error:
|
||||
'Codex sign-in finished, but its configured auth.json has no usable ChatGPT token. Check CODEX_HOME and Codex credential storage. GEV can read file storage only.',
|
||||
});
|
||||
} else finish(attempt, null);
|
||||
},
|
||||
() =>
|
||||
finish(attempt, {
|
||||
code: 'CODEX_OAUTH_LOGIN_FAILED',
|
||||
error:
|
||||
'ChatGPT sign-in did not complete. Try again in Provider Settings.',
|
||||
}),
|
||||
);
|
||||
return { available: false, started: true, pending: true };
|
||||
} catch (error) {
|
||||
const code =
|
||||
error?.code === 'CODEX_OAUTH_STORAGE_MISMATCH'
|
||||
? error.code
|
||||
: 'CODEX_OAUTH_LOGIN_FAILED';
|
||||
finish(attempt, {
|
||||
code,
|
||||
error:
|
||||
code === 'CODEX_OAUTH_STORAGE_MISMATCH'
|
||||
? error.message
|
||||
: 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
|
||||
});
|
||||
return { ...status(), started: false, pending: false };
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status,
|
||||
start,
|
||||
dispose() {
|
||||
disposed = true;
|
||||
if (!active) return;
|
||||
const attempt = active;
|
||||
finish(attempt, null);
|
||||
attempt.process?.cancel();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -16,6 +16,9 @@ const OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT = 0.5;
|
||||
|
||||
const OPENAI_HUD_SUMMARY_MODEL_DEFAULT = 'gpt-5-nano';
|
||||
|
||||
/** Input transcription for voice captions; OPENAI_REALTIME_TRANSCRIBE_MODEL=off disables it. */
|
||||
const OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT = 'gpt-4o-mini-transcribe';
|
||||
|
||||
export {
|
||||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
|
||||
OPENAI_REALTIME_MODEL_MINI_DEFAULT,
|
||||
@@ -24,4 +27,5 @@ export {
|
||||
OPENAI_REALTIME_REASONING_DEFAULT,
|
||||
OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
|
||||
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
|
||||
};
|
||||
@@ -1,16 +1,117 @@
|
||||
import { defaultSourceRoot } from '../common/source-root.js';
|
||||
import path from 'node:path';
|
||||
import { readRequestBody } from '../common/request.js';
|
||||
import fs from 'node:fs';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
|
||||
/** Cap on one request body. */
|
||||
const REALTIME_DEBUG_LOG_MAX_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
|
||||
const REALTIME_DEBUG_LOG_DIR = path.join(sourceRoot, '.gev-logs');
|
||||
const REALTIME_DEBUG_LOG_FILE = path.join(
|
||||
REALTIME_DEBUG_LOG_DIR,
|
||||
'realtime-conversations.jsonl',
|
||||
);
|
||||
/**
|
||||
* Ceiling for the live log before it rotates. One generation is kept
|
||||
* (`…jsonl.1`), so the sink is bounded at twice this however long a session
|
||||
* runs. REALTIME_DEBUG_LOG_MAX_BYTES caps a single request; it never bounded
|
||||
* the file those requests accumulate into.
|
||||
*/
|
||||
const REALTIME_DEBUG_LOG_MAX_FILE_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Per-IP write ceiling. Always on, unlike the opt-in limiter the cost-bearing
|
||||
* OpenAI routes share: throttling those by default would change what a user
|
||||
* spends, while this sink spends disk and event-loop time. A voice session
|
||||
* writes far below this, so the cap needs no configuration to stay invisible.
|
||||
*/
|
||||
const REALTIME_DEBUG_LOG_MAX_PER_MIN = 120;
|
||||
|
||||
/**
|
||||
* Fields that carry what people said or heard: transcripts, typed and spoken
|
||||
* text, streamed deltas, tool arguments/results and the span's quoted
|
||||
* replies. They are dropped from the persisted log unless content logging is explicitly enabled
|
||||
* (GEV_VOICE_LOG_CONTENT=1), so timing and tool diagnostics never retain
|
||||
* speech by default.
|
||||
*/
|
||||
const VOICE_TEXT_FIELDS = new Set([
|
||||
'transcript',
|
||||
'text',
|
||||
'delta',
|
||||
'answer_text',
|
||||
'preamble_text',
|
||||
]);
|
||||
const VOICE_CONTENT_MAX_DEPTH = 12;
|
||||
const VOICE_CONTENT_MAX_DEPTH_MARKER = '[omitted: max depth]';
|
||||
|
||||
/** Replace spoken or typed content and complete tool argument/result bodies. */
|
||||
function omitVoiceContent(value, depth = 0) {
|
||||
if (depth > VOICE_CONTENT_MAX_DEPTH) return VOICE_CONTENT_MAX_DEPTH_MARKER;
|
||||
if (value === null || typeof value !== 'object') return value;
|
||||
if (Array.isArray(value))
|
||||
return value.map((item) => omitVoiceContent(item, depth + 1));
|
||||
const output = {};
|
||||
for (const [key, item] of Object.entries(value)) {
|
||||
// Tool bodies have open-ended schemas: omit the whole body rather than
|
||||
// chasing query, label, say, display, or future tool-specific fields.
|
||||
// Protocol response.output arrays still carry useful event metadata; only
|
||||
// string output is a serialized function result and must be opaque.
|
||||
if (
|
||||
key === 'arguments' ||
|
||||
key === 'result' ||
|
||||
(key === 'output' && typeof item === 'string')
|
||||
) {
|
||||
output[key] =
|
||||
typeof item === 'string'
|
||||
? `[omitted ${item.length} chars]`
|
||||
: '[omitted]';
|
||||
continue;
|
||||
}
|
||||
if (VOICE_TEXT_FIELDS.has(key) && typeof item === 'string') {
|
||||
output[key] = `[omitted ${item.length} chars]`;
|
||||
continue;
|
||||
}
|
||||
output[key] = omitVoiceContent(item, depth + 1);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function createDebugLogHandler({
|
||||
sourceRoot = defaultSourceRoot,
|
||||
includeContent = process.env.GEV_VOICE_LOG_CONTENT === '1',
|
||||
} = {}) {
|
||||
const logDir = path.join(sourceRoot, '.gev-logs');
|
||||
const logFile = path.join(logDir, 'realtime-conversations.jsonl');
|
||||
const allow = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: REALTIME_DEBUG_LOG_MAX_PER_MIN,
|
||||
globalMax: 400,
|
||||
});
|
||||
|
||||
// Appends are chained rather than fired concurrently because the rotation
|
||||
// check is a read-then-write on file size: run in parallel, every caller
|
||||
// stats a file still under the ceiling, none of them rotates, and the log
|
||||
// grows past it. The records themselves are O_APPEND and never interleave.
|
||||
let queue = Promise.resolve();
|
||||
|
||||
function append(line) {
|
||||
const done = queue.then(async () => {
|
||||
await fsp.mkdir(logDir, { recursive: true });
|
||||
const size = await fsp.stat(logFile).then(
|
||||
(stat) => stat.size,
|
||||
() => 0,
|
||||
);
|
||||
if (size + Buffer.byteLength(line) > REALTIME_DEBUG_LOG_MAX_FILE_BYTES) {
|
||||
// Node does not replace an existing destination on Windows. Remove the
|
||||
// retained generation explicitly so every rotation behaves the same on
|
||||
// supported platforms rather than failing after the first 64 MB.
|
||||
await fsp.rm(`${logFile}.1`, { force: true });
|
||||
await fsp.rename(logFile, `${logFile}.1`);
|
||||
}
|
||||
await fsp.appendFile(logFile, line);
|
||||
});
|
||||
// The caller still sees its own rejection, but one failed append must not
|
||||
// poison the chain for every request queued behind it.
|
||||
queue = done.catch(() => {});
|
||||
return done;
|
||||
}
|
||||
|
||||
return async (req, res) => {
|
||||
if (req.method !== 'POST') {
|
||||
res.statusCode = 405;
|
||||
@@ -19,29 +120,41 @@ function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allow(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Retry-After': '60',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await readRequestBody(req, REALTIME_DEBUG_LOG_MAX_BYTES);
|
||||
const record = JSON.parse(body || '{}');
|
||||
fs.mkdirSync(REALTIME_DEBUG_LOG_DIR, { recursive: true });
|
||||
fs.appendFileSync(
|
||||
REALTIME_DEBUG_LOG_FILE,
|
||||
const parsed = JSON.parse(body || '{}');
|
||||
const record = includeContent ? parsed : omitVoiceContent(parsed);
|
||||
// The server's own timestamp comes last, so a record cannot supply one.
|
||||
await append(
|
||||
`${JSON.stringify({
|
||||
loggedAt: new Date().toISOString(),
|
||||
...record,
|
||||
loggedAt: new Date().toISOString(),
|
||||
})}\n`,
|
||||
);
|
||||
res.statusCode = 204;
|
||||
res.end();
|
||||
} catch (error) {
|
||||
res.statusCode = 400;
|
||||
// A malformed record or an oversized body is the caller's fault; a failed
|
||||
// write is ours, and only the second is worth a log line. Neither answer
|
||||
// carries the error text, which for an fs failure is an errno and an
|
||||
// absolute path.
|
||||
const callerFault =
|
||||
error instanceof SyntaxError || error?.code === 'BODY_TOO_LARGE';
|
||||
if (!callerFault) console.warn('[realtime-debug-log] append failed');
|
||||
res.statusCode = callerFault ? 400 : 500;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: error?.message || 'Failed to write Realtime debug log',
|
||||
}),
|
||||
);
|
||||
res.end(JSON.stringify({ error: 'Failed to write Realtime debug log' }));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export { createDebugLogHandler };
|
||||
export { createDebugLogHandler, omitVoiceContent };
|
||||
@@ -1,5 +1,8 @@
|
||||
import { keylessHudSummaryResponse } from '../../../src/hudSummaryResponse.js';
|
||||
import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
import {
|
||||
HUD_SUMMARY_INSTRUCTIONS,
|
||||
keylessHudSummaryResponse,
|
||||
} from '../../../src/hudSummaryResponse.js';
|
||||
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
import { readRequestBody } from '../common/request.js';
|
||||
import { OPENAI_HUD_SUMMARY_MODEL_DEFAULT } from './constants.js';
|
||||
|
||||
@@ -43,10 +46,10 @@ async function handleHudSummary(req, res) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). Keyless HUD
|
||||
// fallback has no provider cost and resolves above without consuming a
|
||||
// paid-endpoint quota slot.
|
||||
if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
|
||||
// Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
|
||||
// Keyless HUD fallback has no provider cost and resolves above without
|
||||
// consuming a paid-endpoint quota slot.
|
||||
if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
|
||||
|
||||
try {
|
||||
const body = await readRequestBody(req, 64 * 1024);
|
||||
@@ -61,13 +64,7 @@ async function handleHudSummary(req, res) {
|
||||
model:
|
||||
process.env.OPENAI_HUD_SUMMARY_MODEL ||
|
||||
OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
|
||||
instructions: [
|
||||
"Write one concise intelligence-HUD summary for God's Eye View.",
|
||||
'Use only the supplied place, street, nearby-place, and enabled-layer text labels.',
|
||||
'Prefer the clearest named place and include a relevant enabled layer only when useful.',
|
||||
'Do not infer from coordinates or invent a place.',
|
||||
'Output exactly five words with no title, punctuation, markdown, or introductory phrase.',
|
||||
].join(' '),
|
||||
instructions: HUD_SUMMARY_INSTRUCTIONS,
|
||||
input: JSON.stringify(context),
|
||||
reasoning: { effort: 'minimal' },
|
||||
max_output_tokens: 100,
|
||||
@@ -78,20 +75,23 @@ async function handleHudSummary(req, res) {
|
||||
res.statusCode = response.ok && summary ? 200 : response.status || 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
if (!response.ok)
|
||||
console.warn(`[hud-summary] upstream HTTP ${response.status}`);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
summary: summary || null,
|
||||
error: response.ok
|
||||
? null
|
||||
: data.error?.message || 'OpenAI HUD summary request failed',
|
||||
// Never relay `data.error.message`: that is OpenAI's own wording, and
|
||||
// it carries request ids, organization hints and quota phrasing.
|
||||
error: response.ok ? null : 'OpenAI HUD summary request failed',
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
} catch {
|
||||
console.warn('[hud-summary] request failed');
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: error?.message || 'OpenAI HUD summary request failed',
|
||||
error: 'OpenAI HUD summary request failed',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
const DEFAULT_ANNOTATION_GUIDANCE =
|
||||
'Use a single annotate_map call with several annotations when you are describing multiple related places at once. Set flyTo true only when the user is not already looking at the place; if every mark in a call lands off-screen the app auto-frames them, so when unsure leave flyTo false. Do NOT say out loud that you are drawing, highlighting, or annotating — just speak naturally about the places while the marks appear. ANNOTATIONS ACCUMULATE AND PERSIST — keep adding marks as you explore; you can fly around, change topic, and jump between far-apart places and the marks STAY, so the user can build up the map and show people things. Do NOT clear on your own initiative: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
|
||||
'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
|
||||
|
||||
function realtimeInstructions(
|
||||
annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
|
||||
@@ -13,25 +13,24 @@ function realtimeInstructions(
|
||||
'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
|
||||
'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
|
||||
'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
|
||||
'For "what is this aircraft?" answers, read the callsign, operator, registration, type, and route only from get_entity_context selected.properties. Treat route, routeOrigin, and routeDestination as the only authoritative route fields. Every aircraft identity answer MUST explicitly cover operator, type, and route. When a route is present, repeat its endpoint codes exactly; do not expand airport codes into city names. For a missing field say exactly "Operator details are unavailable", "Aircraft type is unavailable", or "Route details are unavailable" as applicable. Never silently omit missing enrichment or infer it from the callsign.',
|
||||
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking automatically stops the tracking (the result says so): mention it briefly.',
|
||||
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
|
||||
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.',
|
||||
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
|
||||
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data: report the returned feed source briefly, never call it an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
|
||||
'For ANALYTICAL questions about layer data — how many / which / fastest / highest / biggest / nearest flights, ships, fires, or earthquakes ("how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Narrate the count plus two or three notable examples by name, and reflect the result\'s coverage note honestly: the answer covers data loaded by enabled layers, not the whole world. If the needed layer is disabled, say so and offer to enable it. For follow-ups about the same set ("which of THOSE is closest?"), call analyst_query with followUp=true and only the new filter/sort.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, "near / nearby / how many aircraft" means the Contacts window: answer from contactsWindow in the tool result — those are the exact numbers on the user\'s panel. set_context_mode, analyst_query, and get_current_view_state carry it after Contacts settles. For "Open Contacts and tell me how many aircraft are within 250 km", call set_context_mode{mode:"contacts"} first and answer from contactsWindow.aircraft; do not answer from a pre-Contacts analyst query. analyst_query\'s own count measures currently-loaded records and is usually lower; never give it as the window count. CENTER PRECEDENCE for a nearby/how-many ask, in order: an explicit place in the question ("over Texas", "near Austin") always wins and ignores Contacts state; else the CONTACTS SUBJECT when Contacts is active and has one — a selected datacenter, dam, fire, or cable does NOT silently become the center; else an entity the user explicitly names ("around this datacenter"); else the current view, said aloud ("nothing is selected, so this is the current view"). With Contacts active but NO subject yet, use the view and say so; never read an empty panel. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two different numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge ("a few", "less than a dozen"): if a tool returns 46, say 46. (5) When it matters, add once: counts cover loaded data, and the flights layer loads where you look.',
|
||||
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
|
||||
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
|
||||
'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
|
||||
'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
|
||||
'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
|
||||
'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
|
||||
'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
|
||||
'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
|
||||
'If get_entity_context has no selected object or overlay entities, use its basemap context: Google Photorealistic 3D Tiles/Cesium source, center target coordinates, reverse-geocoded place, camera altitude, active style, and enabled layers. Do not say there is nothing unless the basemap target is also unavailable.',
|
||||
'If basemap context includes knownLandmarks, prefer the nearest known landmark by name for "what am I looking at" answers. For example, if knownLandmarks includes Eiffel Tower, say Eiffel Tower.',
|
||||
'At local zoom, use basemap nearbyPlaces, place.labels, viewportPlaces.visibleLabels, and viewportPlaces.streetLabels to identify the building, premises, roads, and named places visible around the screen target.',
|
||||
'If basemap context includes viewportPlaces, prefer dominantCountry, dominantRegion, and dominantLocality over raw coordinates.',
|
||||
'When basemap context includes viewportSamples or an inferred country, trust that over a single reverse-geocoded address. If most samples indicate Iran, say Iran, not the United States.',
|
||||
'When a viewport screenshot is attached after get_entity_context, read clearly legible street, building, and place labels from it and combine them with structured label context. Respect scene viewScale: at global/continental/regional scale, avoid naming a precise street/city from one center pixel.',
|
||||
'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
|
||||
'Do not mention disabled layers or stale selections.',
|
||||
'When a request requires a tool call, do not speak in the same response as the tool call. Call the tool first.',
|
||||
'When a single user request contains MULTIPLE changes (e.g. "switch to operator layout, use balanced detection at density 50, and switch to Bing aerial"), call ALL the corresponding tools — multiple tool calls in sequence — before speaking. Never confirm a partial subset. If a later tool fails, say which parts succeeded and which failed.',
|
||||
'After receiving tool output, speak exactly one short confirmation. Do not repeat the confirmation.',
|
||||
'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
|
||||
'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
|
||||
'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
|
||||
'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
|
||||
'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
|
||||
// INSTRUCTION-ONLY mapping for the two globe-scale named views.
|
||||
//
|
||||
@@ -47,26 +46,24 @@ function realtimeInstructions(
|
||||
// string is the whole rollback.
|
||||
'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
|
||||
'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
|
||||
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any mention of "satellite" or "satellites" ALWAYS means the satellites DATA LAYER via set_layer_visibility, never a basemap. "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
|
||||
'HUD requests ("hud on/off", "switch to operator/minimal/tactical layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
|
||||
'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
|
||||
'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
|
||||
'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
|
||||
'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
|
||||
'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
|
||||
'"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
|
||||
'"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
|
||||
"After frame_overhead, speak ONLY from the tool result's count field — e.g. 'Framed fourteen aircraft, labels on'; never reassess or second-guess the count aloud.",
|
||||
'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
|
||||
'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
|
||||
'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
|
||||
'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
|
||||
'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
|
||||
'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
|
||||
'Keep spoken confirmations short, e.g. "Opening datacenters" or "Flying to London".',
|
||||
'WHITEBOARD THE WORLD: whenever you describe or explain a specific place, building, campus, district, boundary, or a spatial relationship between places, call annotate_map to mark it visually as you talk — like sketching on the map. To call out a specific building, campus, compound, park, or district, use type=area (it traces and encloses the real footprint — a building gets a glowing volume, a district gets a draped outline). Use type=highlight only for a transient pulse on a precise spot that has no meaningful footprint, and type=pin to drop a labeled marker. Examples: "what is the Palace of Fine Arts?" → an AREA on it; "the old military base next to it" → an AREA on the Presidio; "ILM is right here" → a pin; "it sits next to the Marina" → an arrow from one to the other. Prefer place NAMES so the app resolves real positions and outlines; never invent coordinates or pixel locations.',
|
||||
'On every annotation, also set entityKind to what the thing IS when you know it: building (one structure), compound (campus/grounds/mall/park), district (neighborhood/area of a city), street (a named road), or point_feature (a monument, statue, memorial, plaque, fountain, or other small point landmark). entityKind is a FACT about the target, independent of the mark type you chose — monuments and statues are point_feature even when you use type=area; the app then anchors them as precise points instead of guessing at a footprint.',
|
||||
'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
|
||||
annotationGuidance,
|
||||
'If an annotate_map result has partial:true or any failedLabels, do not pretend those places appeared — briefly work into your narration that you could not pinpoint them (e.g. "I couldn\'t place X"). If a route comes back as a direct line (no street route was found), describe it as a straight-line distance, not a walking/driving time. If an annotate_map result has capped:true, the map is full — ASK the user whether to clear before drawing more; do not clear unprompted. outlinePending:true is NOT a failure, but it is also NOT an outline: the anchor mark is placed and the boundary is still being traced in the background. Narrate it in progress — e.g. "tracing the boundary now" — and NEVER state the outline is already drawn or visible; it may yet come back as just a point. A later system item of type map_annotation_outline reports the final outcome per mark (status resolved or failed, with its label): use it to quietly confirm, or to correct yourself if you implied a boundary that stayed a point — an honest miss beats a misleading guess.',
|
||||
'PREFER NAMES. Only when you cannot name or geocode a place but you can clearly SEE the exact spot in the most recent viewport screenshot, fall back to screenX/screenY (normalized 0..1 from that image) to point at it; the app converts the pixel to a real world point. Never use screenX/screenY for something you could name.',
|
||||
'PATHS vs DISTANCES: for "walking/driving route from A to B" (or through several stops), use type=route with the ordered points and the matching mode (walking/driving/cycling) — the app draws the real street-following path on the map and reports distance and travel time, which you can read aloud. For "how far is X from Y", "is it nearby", or "X is next to Y", use type=arrow between the two — it draws a floating connector and shows the straight-line distance. Do NOT use route for a simple distance/proximity question.',
|
||||
'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
|
||||
'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
|
||||
@@ -1,33 +1,46 @@
|
||||
import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
|
||||
/**
|
||||
* Requests/min/IP applied when GEV_RATELIMIT_OPENAI_PER_MIN is unset. Both
|
||||
* places that already name a value agree on it: `.env.example` recommends 30
|
||||
* and the Pinokio build ships 30 (pinokio/_ENVIRONMENT), so the packaged app
|
||||
* is unaffected and only an unconfigured server changes. The HUD asks for a
|
||||
* summary every 15s (HUD_SUMMARY_INTERVAL_MS), so an open tab spends 4/min and
|
||||
* a voice session adds one token mint; the cap clears ordinary use several
|
||||
* times over and bites only a caller working the endpoint far harder than the
|
||||
* app does.
|
||||
*/
|
||||
const OPENAI_DEFAULT_PER_MIN = 30;
|
||||
|
||||
// Built LAZILY on first request, NOT at module load: `.env` values are applied to process.env later
|
||||
// (the plugin config hook calls loadEnv → process.env, AFTER this module is imported), so reading
|
||||
// process.env here at import time would always see them unset and silently stay unlimited even when
|
||||
// process.env here at import time would always see them unset and fall back to the default even when
|
||||
// configured via .env. Building on first request (like the OPENAI_API_KEY reads) sees the loaded env;
|
||||
// the result is cached so the limiter's per-IP window state persists. `null` = unlimited (default).
|
||||
// the result is cached so the limiter's per-IP window state persists. `null` = the explicit 0 opt-out.
|
||||
let _openAiRateLimiter;
|
||||
|
||||
/** OpenAI cost endpoints (realtime/token + hud-summary). Null = unlimited (default). */
|
||||
/** OpenAI cost endpoints (realtime/token + hud-summary). Null only when set to 0. */
|
||||
function openAiRateLimiter() {
|
||||
if (_openAiRateLimiter === undefined)
|
||||
_openAiRateLimiter = makeOptInRateLimiter(
|
||||
_openAiRateLimiter = makeCostRateLimiter(
|
||||
process.env.GEV_RATELIMIT_OPENAI_PER_MIN,
|
||||
OPENAI_DEFAULT_PER_MIN,
|
||||
);
|
||||
return _openAiRateLimiter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply an opt-in limiter to a request, writing a 429 when over the cap.
|
||||
* When `limiter` is null (unlimited, the default) this is a no-op returning
|
||||
* `true`, so the handler proceeds exactly as before.
|
||||
* Apply the limiter to a request, writing a 429 when over the cap.
|
||||
* When `limiter` is null — the explicit `0` opt-out — this is a no-op
|
||||
* returning `true`, so the handler proceeds unthrottled.
|
||||
*
|
||||
* @param {((key:string)=>boolean)|null} limiter
|
||||
* @param {import('http').IncomingMessage} req
|
||||
* @param {import('http').ServerResponse} res
|
||||
* @returns {boolean} True if the request may proceed; false if a 429 was sent.
|
||||
*/
|
||||
function enforceOptInRateLimit(limiter, req, res) {
|
||||
if (!limiter) return true; // unlimited (default) — no behavior change
|
||||
function enforceRateLimit(limiter, req, res) {
|
||||
if (!limiter) return true; // explicit 0 opt-out — unthrottled
|
||||
if (limiter(clientKey(req))) return true;
|
||||
res.statusCode = 429;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
@@ -36,4 +49,4 @@ function enforceOptInRateLimit(limiter, req, res) {
|
||||
return false;
|
||||
}
|
||||
|
||||
export { enforceOptInRateLimit, openAiRateLimiter };
|
||||
export { enforceRateLimit, openAiRateLimiter, OPENAI_DEFAULT_PER_MIN };
|
||||
@@ -1,4 +1,4 @@
|
||||
import { enforceOptInRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
import { enforceRateLimit, openAiRateLimiter } from './rate-limit.js';
|
||||
import {
|
||||
resolveVoiceModel,
|
||||
isKnownVoiceTier,
|
||||
@@ -10,16 +10,132 @@ import {
|
||||
OPENAI_REALTIME_REASONING_DEFAULT,
|
||||
OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
|
||||
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
|
||||
} from './constants.js';
|
||||
import { realtimeInstructions } from './instructions.js';
|
||||
import { GEV_REALTIME_TOOLS } from './tools.js';
|
||||
import {
|
||||
preferCodexOAuth,
|
||||
readCodexOAuthAccessToken,
|
||||
createCodexOAuthLogin,
|
||||
} from './codex-auth.js';
|
||||
|
||||
const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']);
|
||||
|
||||
function isLoopbackRequest(req) {
|
||||
return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || ''));
|
||||
}
|
||||
|
||||
function isSameOriginRequest(req) {
|
||||
const origin = String(req.headers?.origin || '').trim();
|
||||
if (!origin) return true;
|
||||
const host = String(req.headers?.host || '').trim();
|
||||
if (!host) return false;
|
||||
const protocol = req.socket?.encrypted ? 'https:' : 'http:';
|
||||
try {
|
||||
return new URL(origin).origin === new URL(`${protocol}//${host}`).origin;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function requestQuery(req) {
|
||||
try {
|
||||
return new URL(req.url || '', 'http://localhost').searchParams;
|
||||
} catch {
|
||||
return new URLSearchParams();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRealtimeAuthMode(query) {
|
||||
const requested = String(query.get('auth') || 'api-key').toLowerCase();
|
||||
if (requested === 'api-key' || requested === 'oauth') return requested;
|
||||
return null;
|
||||
}
|
||||
|
||||
function createRealtimeOAuthStatusHandler({
|
||||
oauthLogin = createCodexOAuthLogin(),
|
||||
} = {}) {
|
||||
return (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
if (req.method !== 'GET') {
|
||||
res.statusCode = 405;
|
||||
res.end(JSON.stringify({ error: 'Method not allowed' }));
|
||||
return;
|
||||
}
|
||||
if (!isLoopbackRequest(req)) {
|
||||
res.statusCode = 403;
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
available: false,
|
||||
error: 'ChatGPT OAuth is available only from this machine',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.statusCode = 200;
|
||||
res.end(JSON.stringify(oauthLogin.status()));
|
||||
};
|
||||
}
|
||||
|
||||
function createRealtimeOAuthLoginHandler({
|
||||
oauthLogin = createCodexOAuthLogin(),
|
||||
} = {}) {
|
||||
return async (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
if (req.method !== 'POST') {
|
||||
res.statusCode = 405;
|
||||
res.end(JSON.stringify({ error: 'Method not allowed' }));
|
||||
return;
|
||||
}
|
||||
if (!isLoopbackRequest(req)) {
|
||||
res.statusCode = 403;
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
available: false,
|
||||
error: 'ChatGPT OAuth sign-in is available only from this machine',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!isSameOriginRequest(req)) {
|
||||
res.statusCode = 403;
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
available: false,
|
||||
error: 'Cross-origin ChatGPT OAuth sign-in is not allowed',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await oauthLogin.start();
|
||||
res.statusCode = result.available ? 200 : result.loginFailed ? 503 : 202;
|
||||
res.end(JSON.stringify(result));
|
||||
} catch {
|
||||
res.statusCode = 503;
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
available: false,
|
||||
error: 'Could not start ChatGPT sign-in',
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function createRealtimeTokenHandler({
|
||||
annotationGuidance,
|
||||
endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
|
||||
fetchImpl = (...args) => fetch(...args),
|
||||
resolveApiKey = () => process.env.OPENAI_API_KEY,
|
||||
resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
|
||||
environment = process.env,
|
||||
models = {},
|
||||
tools = GEV_REALTIME_TOOLS,
|
||||
} = {}) {
|
||||
return async (req, res) => {
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
@@ -30,16 +146,85 @@ function createRealtimeTokenHandler({
|
||||
return;
|
||||
}
|
||||
|
||||
// Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset.
|
||||
if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return;
|
||||
const query = requestQuery(req);
|
||||
const authMode = resolveRealtimeAuthMode(query);
|
||||
if (!authMode) {
|
||||
res.statusCode = 400;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' }));
|
||||
return;
|
||||
}
|
||||
|
||||
const apiKey = resolveApiKey();
|
||||
if (!apiKey) {
|
||||
let oauthPinned;
|
||||
try {
|
||||
oauthPinned = preferCodexOAuth(environment);
|
||||
} catch (error) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
|
||||
res.end(JSON.stringify({ error: error.message }));
|
||||
return;
|
||||
}
|
||||
if (oauthPinned && authMode !== 'oauth') {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error:
|
||||
'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.',
|
||||
code: 'CODEX_OAUTH_REQUIRED',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
|
||||
if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
|
||||
|
||||
let bearer;
|
||||
if (authMode === 'oauth') {
|
||||
if (!isLoopbackRequest(req)) {
|
||||
res.statusCode = 403;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'ChatGPT OAuth voice is available only from this machine',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
bearer = resolveOAuthAccessToken();
|
||||
} catch (error) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: error?.message || 'ChatGPT sign-in is unavailable',
|
||||
code: error?.code || 'CODEX_OAUTH_UNAVAILABLE',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!bearer) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'ChatGPT sign-in is unavailable',
|
||||
code: 'CODEX_OAUTH_UNAVAILABLE',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
bearer = resolveApiKey();
|
||||
if (!bearer) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Voice model tier, requested by the client as ?tier=standard|mini.
|
||||
// resolveVoiceModel is total: an unknown, empty, or hostile value
|
||||
@@ -47,15 +232,7 @@ function createRealtimeTokenHandler({
|
||||
// bad querystring degrades to a normal session rather than a dead mic.
|
||||
// The env overrides stay authoritative per tier (see .env.example) —
|
||||
// a wrong upstream model id is then a config fix, not a code change.
|
||||
const requestedTier = (() => {
|
||||
try {
|
||||
return new URL(req.url || '', 'http://localhost').searchParams.get(
|
||||
'tier',
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
const requestedTier = query.get('tier');
|
||||
const tier = resolveVoiceModel(requestedTier).tier;
|
||||
const model =
|
||||
tier === 'mini'
|
||||
@@ -88,6 +265,14 @@ function createRealtimeTokenHandler({
|
||||
OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
|
||||
),
|
||||
);
|
||||
// User captions for the voice card. Billed separately from the session.
|
||||
const transcribeModel =
|
||||
process.env.OPENAI_REALTIME_TRANSCRIBE_MODEL ||
|
||||
OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT;
|
||||
const transcription =
|
||||
transcribeModel.toLowerCase() === 'off'
|
||||
? {}
|
||||
: { transcription: { model: transcribeModel } };
|
||||
const sessionConfig = {
|
||||
session: {
|
||||
type: 'realtime',
|
||||
@@ -103,6 +288,7 @@ function createRealtimeTokenHandler({
|
||||
audio: {
|
||||
input: {
|
||||
noise_reduction: { type: 'near_field' },
|
||||
...transcription,
|
||||
turn_detection: {
|
||||
type: 'semantic_vad',
|
||||
eagerness: 'low',
|
||||
@@ -113,7 +299,7 @@ function createRealtimeTokenHandler({
|
||||
output: { voice },
|
||||
},
|
||||
instructions: realtimeInstructions(annotationGuidance),
|
||||
tools: GEV_REALTIME_TOOLS,
|
||||
tools,
|
||||
tool_choice: 'auto',
|
||||
},
|
||||
};
|
||||
@@ -124,7 +310,7 @@ function createRealtimeTokenHandler({
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(30_000),
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Authorization: `Bearer ${bearer}`,
|
||||
'Content-Type': 'application/json',
|
||||
'OpenAI-Safety-Identifier': 'gev-local-dev',
|
||||
},
|
||||
@@ -132,30 +318,52 @@ function createRealtimeTokenHandler({
|
||||
});
|
||||
const body = await response.text();
|
||||
res.statusCode = response.status;
|
||||
// Which tier/model this secret was actually minted for. The upstream
|
||||
// success body is passed through untouched (the client parses it
|
||||
// verbatim), so these headers are the authoritative echo — including the
|
||||
// case where a bogus ?tier= was silently downgraded to standard.
|
||||
res.setHeader('X-GEV-Voice-Tier', tier);
|
||||
res.setHeader('X-GEV-Voice-Model', model);
|
||||
res.setHeader(
|
||||
'X-GEV-Voice-Auth',
|
||||
authMode === 'oauth' ? 'codex-oauth' : 'env',
|
||||
);
|
||||
// Captions are billed separately; the client meters them with this id.
|
||||
res.setHeader(
|
||||
'X-GEV-Voice-Transcribe-Model',
|
||||
transcription.transcription ? transcribeModel : 'off',
|
||||
);
|
||||
if (requestedTier && !isKnownVoiceTier(requestedTier)) {
|
||||
res.setHeader('X-GEV-Voice-Tier-Fallback', '1');
|
||||
}
|
||||
if (!response.ok) {
|
||||
console.warn(`[realtime-token] upstream HTTP ${response.status}`);
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(JSON.stringify({ error: 'Failed to create Realtime token' }));
|
||||
return;
|
||||
}
|
||||
res.setHeader(
|
||||
'Content-Type',
|
||||
response.headers.get('content-type') || 'application/json',
|
||||
);
|
||||
// Which tier/model this secret was actually minted for. The upstream
|
||||
// body is passed through untouched (the client parses it verbatim), so
|
||||
// these headers are the authoritative echo — including the case where a
|
||||
// bogus ?tier= was silently downgraded to standard.
|
||||
res.setHeader('X-GEV-Voice-Tier', tier);
|
||||
res.setHeader('X-GEV-Voice-Model', model);
|
||||
if (requestedTier && !isKnownVoiceTier(requestedTier)) {
|
||||
res.setHeader('X-GEV-Voice-Tier-Fallback', '1');
|
||||
}
|
||||
res.end(body);
|
||||
} catch (error) {
|
||||
} catch {
|
||||
// For a network fault this was a resolver message naming the upstream
|
||||
// host; the client only needs to know the mint failed.
|
||||
console.warn('[realtime-token] mint failed');
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: error?.message || 'Failed to create Realtime token',
|
||||
error: 'Failed to create Realtime token',
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export { createRealtimeTokenHandler };
|
||||
export {
|
||||
createRealtimeOAuthLoginHandler,
|
||||
createRealtimeOAuthStatusHandler,
|
||||
createRealtimeTokenHandler,
|
||||
};
|
||||
@@ -1,3 +1,8 @@
|
||||
import {
|
||||
voiceLayerAliasHint,
|
||||
voiceQueryFieldHint,
|
||||
} from '../../../src/voice/layerManifest.js';
|
||||
|
||||
export const ACTION_DESCRIPTIONS = {
|
||||
fly_to_location: {
|
||||
description:
|
||||
@@ -12,7 +17,11 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
query: {
|
||||
description:
|
||||
'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina".',
|
||||
'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina". "pointer" = the spot under the cursor.',
|
||||
$position: 1,
|
||||
},
|
||||
referent: {
|
||||
description: 'Numbered item n from the last result list.',
|
||||
$position: 1,
|
||||
},
|
||||
viewMode: {
|
||||
@@ -76,13 +85,13 @@ export const ACTION_DESCRIPTIONS = {
|
||||
$position: 1,
|
||||
},
|
||||
set_layer_visibility: {
|
||||
description: "Enable or disable one registered God's Eye View data layer.",
|
||||
description:
|
||||
"Enable or disable one God's Eye View data layer. Every shipped layer is in the enum; never tell the user a listed layer does not exist.",
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
layerId: {
|
||||
description:
|
||||
'Common-name mapping for the non-obvious ids: space mission(s) → rocket-launches; fires/wildfires/active fires → local-firms (NASA FIRMS); ships/vessels/boats → ais-live-vessels; undersea/submarine cables → telegeography-submarine-cables; datacenters → local-datacenters; dams → local-dams; bikes/bike share → bikeshare; street traffic/congestion → traffic; traffic cameras → cctv; internet radio/stations → radio; ALPR/license plate readers/Flock cameras → alpr-cameras.',
|
||||
description: `Spoken names: ${voiceLayerAliasHint()}.`,
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
@@ -154,9 +163,13 @@ export const ACTION_DESCRIPTIONS = {
|
||||
properties: {
|
||||
scope: {
|
||||
description:
|
||||
'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center.',
|
||||
'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center; pointer returns what the cursor is on.',
|
||||
$position: 2,
|
||||
},
|
||||
referent: {
|
||||
description: 'Numbered item n from the last result list.',
|
||||
$position: 1,
|
||||
},
|
||||
layerId: {
|
||||
description: 'Optional layer filter for visible entity context.',
|
||||
$position: 2,
|
||||
@@ -166,7 +179,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
get_current_view_state: {
|
||||
description:
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
|
||||
$position: 1,
|
||||
},
|
||||
set_hud: {
|
||||
@@ -182,6 +195,29 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
set_cyber_sonar: {
|
||||
description:
|
||||
'Adjust Cyber-only sonar controls. Requires the Cyber HUD layout; never switches layout or HUD visibility. Omitted fields stay unchanged. Returns actual settings and separate map/contact sweep activity. A saved setting does not mean the effect is active.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
enabled: { description: 'Explicitly turn sonar on or off.' },
|
||||
rings: { description: 'Number of decorative sonar rings, 3–12.' },
|
||||
rangePct: {
|
||||
description:
|
||||
'Visual ring range, 60–120 percent; not geographic distance.',
|
||||
},
|
||||
intensityPct: {
|
||||
description: 'Sonar Power slider, 0–100 percent. Zero is valid.',
|
||||
},
|
||||
opacityPct: {
|
||||
description:
|
||||
'Contact opacity floor between passes, 35–100 percent; labels have a derived floor. Not whole-scene dimming.',
|
||||
},
|
||||
sectorDeg: { description: 'Sonar sweep sector width, 8–60 degrees.' },
|
||||
},
|
||||
},
|
||||
},
|
||||
set_detection: {
|
||||
description:
|
||||
'Control the detection overlay: on/off, density-derived Sparse/Balanced/Dense profile, and Elastic/Weighted layer allocation.',
|
||||
@@ -326,12 +362,17 @@ export const ACTION_DESCRIPTIONS = {
|
||||
properties: {
|
||||
query: {
|
||||
description:
|
||||
'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id.',
|
||||
'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id. "pointer" = the contact under the cursor.',
|
||||
$position: 1,
|
||||
},
|
||||
referent: {
|
||||
description:
|
||||
'Numbered item n from the last result list (-1 = last); query may then be its label.',
|
||||
$position: 1,
|
||||
},
|
||||
layerId: {
|
||||
description:
|
||||
'Optional layer hint: flights | military | ais-live-vessels | satellites.',
|
||||
'Optional: flights, military, ais-live-vessels or satellites. Other layers are not trackable; for fires use query "biggest fire".',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
@@ -358,7 +399,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
annotate_map: {
|
||||
description:
|
||||
'Draw annotations on the 3D map to visually point out what you are talking about — like sketching on a whiteboard over the world. Use this whenever you mention a specific place, building, campus, boundary, district, or a relationship between two places, so the user can SEE what you mean. Give place NAMES (preferred) or explicit lat/lng; the app resolves them to real-world positions and real building/area outlines — never guess pixel positions. Call this as you begin describing something, and you may mark several places in one call.',
|
||||
'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -375,7 +416,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
target: {
|
||||
description:
|
||||
'Place name to resolve, e.g. "Palace of Fine Arts, San Francisco", "the Pentagon", "Presidio of San Francisco". Preferred over coordinates. For a specific monument/statue/feature that sits within a larger landmark, use its OWN name + city ("Tejano Monument, Austin", "Texas African American History Memorial, Austin") — do NOT phrase it as "X at the Texas State Capitol", which makes the geocoder collapse several of them onto the same centroid so they stack on one spot.',
|
||||
'Place name to resolve ("Palace of Fine Arts, San Francisco"), or "pointer" for the spot under the cursor. A monument inside a larger landmark gets its own name + city ("Tejano Monument, Austin"), never "X at the Texas State Capitol" (that stacks them on one centroid).',
|
||||
$position: 2,
|
||||
},
|
||||
points: {
|
||||
@@ -427,27 +468,24 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
entityKind: {
|
||||
description:
|
||||
'What KIND of thing the target IS — a fact, not a style choice: building = one structure; compound = campus/grounds/mall/park; district = neighborhood or area of a city; street = a named road/corridor; point_feature = monument/statue/memorial/plaque/fountain or other small point landmark. Set it whenever you know it — it routes the resolver to the right footprint source (point_feature anchors monuments as precise points instead of adopting a nearby building outline).',
|
||||
'What the target IS (a fact, not a style): building = one structure; compound = campus/grounds/mall/park; district = neighborhood; street = a named road; point_feature = monument/statue/memorial/fountain, anchored as a precise point. Set it whenever you know it.',
|
||||
$position: 2,
|
||||
},
|
||||
screenX: {
|
||||
description:
|
||||
'Fallback only: when you cannot name/geocode the place but can SEE it in the latest viewport screenshot, the normalized horizontal position (0=left, 1=right) of the spot. The app converts it back to a real world point under that pixel.',
|
||||
'Fallback only, for a spot visible in the latest screenshot that you cannot name: normalized x (0=left, 1=right).',
|
||||
$position: 3,
|
||||
},
|
||||
screenY: {
|
||||
description:
|
||||
'Fallback only: normalized vertical position (0=top, 1=bottom) of the spot in the latest viewport screenshot.',
|
||||
description: 'Fallback only: normalized y (0=top, 1=bottom).',
|
||||
$position: 3,
|
||||
},
|
||||
toScreenX: {
|
||||
description:
|
||||
'For type=arrow: normalized x of the arrow destination from the screenshot (pixel fallback).',
|
||||
description: 'For type=arrow: destination x (pixel fallback).',
|
||||
$position: 3,
|
||||
},
|
||||
toScreenY: {
|
||||
description:
|
||||
'For type=arrow: normalized y of the arrow destination from the screenshot (pixel fallback).',
|
||||
description: 'For type=arrow: destination y (pixel fallback).',
|
||||
$position: 3,
|
||||
},
|
||||
},
|
||||
@@ -505,18 +543,18 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Answer questions ABOUT the data currently loaded on the map — counts, lists, superlatives, and attribute filters over live layers (flights, military, ships, fires, earthquakes). Examples: "how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet", "fastest thing in view". Queries ONLY client-side data from ENABLED layers — if the needed layer is off, say so and offer to enable it. For a follow-up about the previous answer\'s set ("which of those is closest?"), set followUp=true and send only the new filters/sort.',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
layers: {
|
||||
description:
|
||||
'Layers to query. fires/wildfires → local-firms; ships/vessels → ais-live-vessels.',
|
||||
'Layers to query. fires → local-firms; ships → ais-live-vessels; buses/trains → transit; hurricanes → weather-cyclones; bases → military-installations; my receiver → local-adsb. Units differ per layer, so rank speed within one layer.',
|
||||
$position: 2,
|
||||
},
|
||||
scope: {
|
||||
description:
|
||||
'Spatial scope. Default: view (near the camera). Use kind=region for "over Texas"-style asks; kind=anywhere for global questions.',
|
||||
'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
|
||||
$position: 2,
|
||||
properties: {
|
||||
name: {
|
||||
@@ -525,23 +563,29 @@ export const ACTION_DESCRIPTIONS = {
|
||||
$position: 1,
|
||||
},
|
||||
km: {
|
||||
description: 'For kind=radius.',
|
||||
description:
|
||||
'For kind=radius or pointer: the actual requested kilometers. An active-subject 250 km request uses 250; another requested radius keeps that number.',
|
||||
$position: 1,
|
||||
},
|
||||
center: {
|
||||
description:
|
||||
'Omit for a radius around the active Contacts subject. Supply only an explicitly requested coordinate or a resolved named-place coordinate; never invent 0,0 or replace an explicit center with the subject.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: {
|
||||
description:
|
||||
'Attribute predicates, ANDed. ALTITUDE IS METERS (40,000 ft = 12192). Fields: altitudeM, speedMps, military, onGround, aircraftClass, callsign, operator, routeOrigin, routeDestination, originCountry (flights); speedKts, shipType, destination (ships); frp, confidence (fires); magnitude, depthKm, place (earthquakes).',
|
||||
description: `ANDed; values keep their type (numbers, true/false). Altitude is meters (40,000 ft = 12192). Time fields take ISO; ageHours = hours ago. Every layer has id, lat, lon. Fields — ${voiceQueryFieldHint()}.`,
|
||||
$position: 1,
|
||||
},
|
||||
sortBy: {
|
||||
description: 'Field to rank by, or "distance" for nearest-first.',
|
||||
description:
|
||||
'Field to rank by, or "distance" (nearest first unless sortDir=desc).',
|
||||
$position: 1,
|
||||
},
|
||||
followUp: {
|
||||
description:
|
||||
'true = re-query the PREVIOUS result set instead of fresh data.',
|
||||
'true = re-filter the last answer (its layers) instead of fresh data.',
|
||||
$position: 1,
|
||||
},
|
||||
},
|
||||
@@ -549,7 +593,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
next_iss_pass: {
|
||||
description:
|
||||
"When the user asks when the ISS / the space station will next fly over: returns the next visible ISS pass for the current camera location (or an explicit lat/lon) — rise time (ISO + minutes from now), rise compass direction, peak elevation, and duration. Requires the satellites layer to have loaded its catalog at least once this session; if it hasn't, tell the user to enable the satellites layer and try again.",
|
||||
'Next ISS pass over the camera location (or lat/lon): rise time, direction, peak elevation, duration and estimated visibility. Needs the satellites layer to have loaded once; if not, ask the user to enable it.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
@@ -571,4 +615,9 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
next_satellite_pass: {
|
||||
description:
|
||||
'Predict the next pass in 24 hours for one satellite in the loaded catalog, identified by exact NORAD ID or name. Ambiguous names return candidates: ask for a choice rather than selecting one. Defaults to geometric passes; visibleOnly requires estimated illumination and a dark observer sky, which does not guarantee naked-eye visibility. Uses camera location unless coordinates are supplied.',
|
||||
$position: 1,
|
||||
},
|
||||
};
|
||||
@@ -2,3 +2,15 @@ import { createActionTools } from '../../../src/voice/actionSchemas.js';
|
||||
import { ACTION_DESCRIPTIONS } from './toolDescriptions.js';
|
||||
|
||||
export const GEV_REALTIME_TOOLS = createActionTools(ACTION_DESCRIPTIONS);
|
||||
|
||||
/**
|
||||
* The session's tools: the app actions, then additional function tools
|
||||
* whose names no action already uses.
|
||||
*/
|
||||
export function realtimeSessionTools(additional = []) {
|
||||
const names = new Set(GEV_REALTIME_TOOLS.map((tool) => tool.name));
|
||||
return [
|
||||
...GEV_REALTIME_TOOLS,
|
||||
...additional.filter((tool) => !names.has(tool.name)),
|
||||
];
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { readRequestBodyCapped } from './common/request.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
resolveOverpassUpstreams,
|
||||
} from './overpass/constants.js';
|
||||
import { sanitizeOverpassBody } from './overpass/query.js';
|
||||
import {
|
||||
@@ -17,6 +18,7 @@ import {
|
||||
import {
|
||||
overpassPayloadIsData,
|
||||
fetchOverpassPayload,
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import { installRouteMiddleware } from './places/routes.js';
|
||||
|
||||
@@ -41,9 +43,16 @@ const _overpassRateLimiter = makeRateLimiter({
|
||||
function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
|
||||
res.writeHead(payload.status, {
|
||||
'Content-Type': payload.contentType || 'application/json',
|
||||
'Cache-Control': 'public, max-age=15',
|
||||
'Cache-Control': overpassPayloadIsData(payload)
|
||||
? 'public, max-age=15'
|
||||
: 'no-store',
|
||||
'X-Overpass-Cache': cacheStatus,
|
||||
'X-Overpass-Upstream': payload.endpoint || 'unknown',
|
||||
...(Number.isFinite(payload.cachedAt)
|
||||
? { 'X-Overpass-Cached-At': new Date(payload.cachedAt).toISOString() }
|
||||
: {}),
|
||||
...(payload.retryAfterMs
|
||||
? { 'Retry-After': String(Math.ceil(payload.retryAfterMs / 1000)) }
|
||||
: {}),
|
||||
});
|
||||
res.end(payload.body || '');
|
||||
}
|
||||
@@ -52,7 +61,7 @@ function sendOverpassResponse(res, payload, cacheStatus = 'MISS') {
|
||||
* Vite plugin: Overpass API proxy with response caching and request coalescing.
|
||||
*
|
||||
* Accepts POST requests at /api/overpass, normalizes the query body for
|
||||
* cache keying, and fans out to multiple Overpass mirrors with per-upstream
|
||||
* cache keying, and uses only operator-configured endpoints with per-upstream
|
||||
* timeout and rate-limit detection. Successful responses are cached for
|
||||
* OVERPASS_CACHE_MS. Concurrent identical queries share a single upstream
|
||||
* request via the in-flight map.
|
||||
@@ -66,6 +75,19 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
// (a body-read failure would otherwise hit an out-of-scope reference).
|
||||
let cacheKey = null;
|
||||
try {
|
||||
// Capability probe: lets clients skip queries when nothing is configured.
|
||||
if (req.method === 'GET' && req.url?.split('?')[0] === '/status') {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
configured: resolveOverpassUpstreams().length > 0,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (req.method !== 'POST') {
|
||||
res.writeHead(405, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||
@@ -104,12 +126,21 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
|
||||
// Normalize whitespace so semantically identical Overpass QL queries share cache entries
|
||||
cacheKey = safeBody.replace(/\s+/g, ' ').trim();
|
||||
if (!resolveOverpassUpstreams().length) {
|
||||
const cached = await readStaleOverpass(cacheKey);
|
||||
sendOverpassResponse(
|
||||
res,
|
||||
cached || overpassNotConfigured(),
|
||||
cached ? 'STALE' : 'DISABLED',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const preflight = await resolveOverpassPreflight({
|
||||
cacheKey,
|
||||
memoryCache: _overpassCache,
|
||||
inFlight: _overpassInFlight,
|
||||
// Fresh-enough disk entries survive restarts and skip the public
|
||||
// mirrors; boundary-class queries keep their month-long TTL.
|
||||
// Fresh-enough disk entries survive restarts and skip upstream
|
||||
// requests; boundary-class queries keep their month-long TTL.
|
||||
readDisk: () =>
|
||||
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
|
||||
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
|
||||
@@ -194,7 +225,7 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
sendOverpassResponse(res, stale, 'STALE');
|
||||
return;
|
||||
}
|
||||
console.error('[Overpass Proxy]', e.message);
|
||||
console.error('[Overpass Proxy] request failed');
|
||||
res.writeHead(502, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ error: 'Overpass proxy error' }));
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
OVERPASS_DISK_DIR,
|
||||
OVERPASS_CACHE_MS,
|
||||
OVERPASS_CACHE_MAX_ENTRIES,
|
||||
cachedByConfiguredUpstream,
|
||||
} from './constants.js';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
@@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
|
||||
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
|
||||
// them on both fresh and stale reads so an upgrade can recover immediately.
|
||||
if (!overpassPayloadIsData(payload)) return null;
|
||||
// The key is the query alone, so a file written by a different upstream —
|
||||
// including the public instances that were the default before #742 — is a
|
||||
// hit for the same query. Refuse it: the operator's configured instance
|
||||
// must be what answers, on the stale path too.
|
||||
if (!cachedByConfiguredUpstream(payload)) return null;
|
||||
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
|
||||
return payload;
|
||||
} catch {
|
||||
|
||||
@@ -1,32 +1,94 @@
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overpass API proxy constants and cache state
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* User-Agent sent to every Overpass mirror.
|
||||
*
|
||||
* The OSM API usage policy asks for a "Valid User-Agent identifying application
|
||||
* and version"; a generic proxy label is not one. A mirror is free to refuse a
|
||||
* client it cannot identify, and `src/overpassProxy.test.mjs` pins what that
|
||||
* costs: a refusal is never data, so the query falls through to whatever
|
||||
* mirrors are left. Keep this honest and stable — if it is ever refused, the
|
||||
* answer is less query volume, not a new name.
|
||||
*/
|
||||
/** Stable application identity for operator-configured Overpass instances. */
|
||||
const OVERPASS_USER_AGENT =
|
||||
'gods-eye-view/0.1 (+https://github.com/bilawalsidhu/gods-eye-view)';
|
||||
|
||||
/** Ordered list of Overpass API mirrors; tried sequentially on failure/rate-limit. */
|
||||
const OVERPASS_UPSTREAMS = [
|
||||
'https://overpass-api.de/api/interpreter',
|
||||
'https://overpass.kumi.systems/api/interpreter',
|
||||
'https://lz4.overpass-api.de/api/interpreter',
|
||||
// Community full-planet instance (privateforge nonprofit) — added 2026-07-30
|
||||
// when all three mirrors above refused this IP (likely a dev-traffic rate
|
||||
// ban; refused connections fail in ms, so healthy mirrors above still win).
|
||||
// Verified: planet coverage (Texas query), CORS *, ~5-20 s cold latency.
|
||||
'https://overpass.private.coffee/api/interpreter',
|
||||
];
|
||||
/** Parse only operator-supplied HTTP(S) endpoints; private instances are allowed. */
|
||||
function parseOverpassUpstreams(raw) {
|
||||
const endpoints = [];
|
||||
for (const token of String(raw || '').split(',')) {
|
||||
try {
|
||||
const url = new URL(token.trim());
|
||||
if (
|
||||
!['http:', 'https:'].includes(url.protocol) ||
|
||||
!url.hostname ||
|
||||
url.hash
|
||||
)
|
||||
continue;
|
||||
if (!endpoints.includes(url.href)) endpoints.push(url.href);
|
||||
} catch {
|
||||
// Invalid configuration never becomes an upstream or appears in logs.
|
||||
}
|
||||
}
|
||||
return endpoints.slice(0, 8);
|
||||
}
|
||||
|
||||
let upstreamMemo = { raw: null, endpoints: [], tag: 'none' };
|
||||
|
||||
/**
|
||||
* Short, stable identifier for the configured upstream set. Cached responses
|
||||
* carry it so an answer produced by a different instance is not served after
|
||||
* the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the
|
||||
* configuration change by 7 days, a month for boundary queries, and forever on
|
||||
* the serve-stale path. It is a hash, never the URLs, and it is only ever
|
||||
* compared on read: it is not sent to clients and does not appear in errors.
|
||||
*/
|
||||
function upstreamTag(endpoints) {
|
||||
if (!endpoints.length) return 'none';
|
||||
// Sorted, so reordering the same instances for failover keeps the cache.
|
||||
return createHash('sha256')
|
||||
.update([...endpoints].sort().join('\n'))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
/** Resolve after environment loading. Public Overpass instances are not used by default. */
|
||||
function resolveOverpassUpstreams() {
|
||||
readUpstreamMemo();
|
||||
return [...upstreamMemo.endpoints];
|
||||
}
|
||||
|
||||
/** The configured-upstream tag stored alongside, and checked against, cache entries. */
|
||||
function resolveOverpassUpstreamTag() {
|
||||
return readUpstreamMemo().tag;
|
||||
}
|
||||
|
||||
let reportedForeignCache = false;
|
||||
|
||||
/**
|
||||
* Was this cached entry produced by the upstream set configured now?
|
||||
*
|
||||
* With nothing configured there is no instance to contradict, and serving
|
||||
* last-good data at any age is the point of the cache (the proxy's
|
||||
* not-configured path depends on it), so any entry is accepted. Once an
|
||||
* operator does configure an instance, only its own answers are served: an
|
||||
* entry with no tag, or one from a different instance, is refused rather than
|
||||
* matched on the query alone. Shared by both caches fed by these upstreams.
|
||||
*/
|
||||
function cachedByConfiguredUpstream(entry) {
|
||||
const tag = resolveOverpassUpstreamTag();
|
||||
if (tag === 'none' || entry?.upstreams === tag) return true;
|
||||
if (!reportedForeignCache) {
|
||||
reportedForeignCache = true;
|
||||
console.warn(
|
||||
'[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.',
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function readUpstreamMemo() {
|
||||
const raw = process.env.OVERPASS_UPSTREAMS || '';
|
||||
if (upstreamMemo.raw !== raw) {
|
||||
const endpoints = parseOverpassUpstreams(raw);
|
||||
upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) };
|
||||
}
|
||||
return upstreamMemo;
|
||||
}
|
||||
|
||||
/**
|
||||
* TTL for FRESH cached Overpass responses (ms). Road geometry is static for
|
||||
@@ -142,7 +204,10 @@ export {
|
||||
OVERPASS_SIMPLIFY_MIN_POINTS,
|
||||
OVERPASS_SIMPLIFY_TOLERANCE_DEG,
|
||||
OVERPASS_MAX_RESPONSE_BYTES,
|
||||
OVERPASS_UPSTREAMS,
|
||||
parseOverpassUpstreams,
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
cachedByConfiguredUpstream,
|
||||
OVERPASS_USER_AGENT,
|
||||
OVERPASS_TIMEOUT_MS,
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
OVERPASS_MAX_RESPONSE_BYTES,
|
||||
OVERPASS_UPSTREAMS,
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
OVERPASS_USER_AGENT,
|
||||
OVERPASS_TIMEOUT_MS,
|
||||
} from './constants.js';
|
||||
@@ -60,99 +61,150 @@ function overpassPayloadIsData(payload) {
|
||||
);
|
||||
}
|
||||
|
||||
const cooldowns = new Map();
|
||||
|
||||
/** A stable, non-retryable capability response shared by every Overpass route. */
|
||||
export function overpassNotConfigured() {
|
||||
return {
|
||||
status: 503,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
error: 'Detailed OpenStreetMap queries are not configured',
|
||||
code: 'OVERPASS_NOT_CONFIGURED',
|
||||
retryable: false,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Honor Retry-After dates/seconds; absent values use bounded exponential backoff. */
|
||||
function retryDelay(value, failures, now) {
|
||||
const seconds = Number(value);
|
||||
const explicit =
|
||||
value && Number.isFinite(seconds)
|
||||
? seconds * 1000
|
||||
: Date.parse(value) - now;
|
||||
return Number.isFinite(explicit)
|
||||
? Math.max(1000, explicit)
|
||||
: Math.min(300_000, 30_000 * 2 ** Math.min(failures, 4));
|
||||
}
|
||||
|
||||
function refusal(status, retryAfterMs) {
|
||||
return {
|
||||
status,
|
||||
contentType: 'application/json',
|
||||
rateLimited: status === 429 || status === 406,
|
||||
retryAfterMs,
|
||||
body: JSON.stringify({
|
||||
error: 'Configured Overpass upstream unavailable',
|
||||
code: 'OVERPASS_UNAVAILABLE',
|
||||
retryable: true,
|
||||
retryAfterMs,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Try each mirror once, retaining response-size and per-mirror timeout caps.
|
||||
* Refusals and body-level failures rotate; total failure returns the last
|
||||
* rate-limit payload, otherwise the first refusal, or throws a network error.
|
||||
* @param {string} body URL-encoded Overpass QL query body.
|
||||
* @param {number} [maxResponseBytes] Endpoint-specific response cap.
|
||||
* @param {object} [options] Server-only endpoint and I/O overrides for tests.
|
||||
* @returns {Promise<{status:number,body:string,contentType:string,endpoint:string,rateLimited:boolean}>}
|
||||
* Query only the configured chain with capped reads, timeouts and per-endpoint cooldowns.
|
||||
* Explicit endpoint/I/O overrides are server-only test seams. Empty data is valid.
|
||||
*/
|
||||
async function fetchOverpassPayload(
|
||||
body,
|
||||
maxResponseBytes = OVERPASS_MAX_RESPONSE_BYTES,
|
||||
{
|
||||
endpoints = OVERPASS_UPSTREAMS,
|
||||
endpoints = resolveOverpassUpstreams(),
|
||||
fetchImpl = fetch,
|
||||
readBody = readResponseTextCapped,
|
||||
simplify = simplifyOverpassPayloadBody,
|
||||
now = Date.now,
|
||||
} = {},
|
||||
) {
|
||||
let lastError = null;
|
||||
let lastRateLimitPayload = null;
|
||||
let lastRefusalPayload = null;
|
||||
|
||||
if (!endpoints.length) return overpassNotConfigured();
|
||||
let failure = refusal(502, 30_000);
|
||||
for (const endpoint of endpoints) {
|
||||
const previous = cooldowns.get(endpoint);
|
||||
if (previous?.until > now()) {
|
||||
failure = refusal(previous.status, previous.until - now());
|
||||
continue;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), OVERPASS_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const upstream = await fetchImpl(endpoint, {
|
||||
const requestUrl = new URL(endpoint);
|
||||
const authorization =
|
||||
requestUrl.username || requestUrl.password
|
||||
? 'Basic ' +
|
||||
Buffer.from(
|
||||
`${decodeURIComponent(requestUrl.username)}:${decodeURIComponent(requestUrl.password)}`,
|
||||
).toString('base64')
|
||||
: null;
|
||||
requestUrl.username = '';
|
||||
requestUrl.password = '';
|
||||
const upstream = await fetchImpl(requestUrl.href, {
|
||||
method: 'POST',
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'User-Agent': OVERPASS_USER_AGENT,
|
||||
...(authorization ? { Authorization: authorization } : {}),
|
||||
},
|
||||
body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const responseBody = await readBody(upstream, maxResponseBytes);
|
||||
const contentType =
|
||||
upstream.headers.get('content-type') || 'application/json';
|
||||
const status = upstream.status;
|
||||
const rateLimited =
|
||||
status === 429 || overpassLooksRateLimited(responseBody);
|
||||
const runtimeError = overpassLooksRuntimeError(responseBody);
|
||||
const payload = {
|
||||
status,
|
||||
body: responseBody,
|
||||
contentType,
|
||||
endpoint,
|
||||
rateLimited,
|
||||
runtimeError,
|
||||
};
|
||||
|
||||
if (rateLimited) {
|
||||
lastRateLimitPayload = payload;
|
||||
continue;
|
||||
}
|
||||
// A 200 body carrying a runtime error / timeout is a transient upstream
|
||||
// failure — skip to the next mirror rather than returning or caching it.
|
||||
if (runtimeError) {
|
||||
lastError = new Error(`Overpass runtime error (${endpoint})`);
|
||||
continue;
|
||||
}
|
||||
// Anything but 2xx is this mirror declining, not an answer. Only 5xx used
|
||||
// to rotate, so a 4xx ended the fan-out and was returned — and cached —
|
||||
// as data: a mirror refusing this client answers 406 while the others
|
||||
// answer 200 to the very same request, so every Overpass-backed layer
|
||||
// failed on an error page with healthy mirrors untried. The first
|
||||
// refusal is kept so a genuinely bad query still reports what upstream
|
||||
// said, but only after every mirror has had the chance to answer it.
|
||||
if (status < 200 || status >= 300) {
|
||||
if (!lastRefusalPayload) lastRefusalPayload = payload;
|
||||
lastError = new Error(
|
||||
`Overpass upstream returned ${status} (${endpoint})`,
|
||||
upstream.status === 429 ||
|
||||
upstream.status === 406 ||
|
||||
overpassLooksRateLimited(responseBody);
|
||||
if (
|
||||
rateLimited ||
|
||||
upstream.status < 200 ||
|
||||
upstream.status >= 300 ||
|
||||
overpassLooksRuntimeError(responseBody)
|
||||
) {
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const delay = retryDelay(
|
||||
upstream.headers.get('retry-after'),
|
||||
failures - 1,
|
||||
now(),
|
||||
);
|
||||
const status = rateLimited
|
||||
? upstream.status === 406
|
||||
? 406
|
||||
: 429
|
||||
: 502;
|
||||
cooldowns.set(endpoint, { until: now() + delay, failures, status });
|
||||
while (cooldowns.size > 64)
|
||||
cooldowns.delete(cooldowns.keys().next().value);
|
||||
failure = refusal(status, delay);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Success: decimate giant boundary geometry before it reaches the cache,
|
||||
// the disk, or the client (what makes the 32 MB read cap safe to hold).
|
||||
payload.body = simplify(payload.body);
|
||||
return payload;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
const parsed = JSON.parse(responseBody);
|
||||
if (!Array.isArray(parsed?.elements) || parsed.remark)
|
||||
throw new Error('Malformed Overpass response');
|
||||
cooldowns.delete(endpoint);
|
||||
return {
|
||||
status: upstream.status,
|
||||
body: simplify(responseBody),
|
||||
contentType: 'application/json',
|
||||
// Never retain a secret-bearing endpoint in cache or response metadata.
|
||||
endpoint: 'configured',
|
||||
// Which configured instance produced this, as a hash rather than a URL,
|
||||
// so a cached answer is not reused after the operator repoints.
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
rateLimited: false,
|
||||
};
|
||||
} catch {
|
||||
const failures = (previous?.failures || 0) + 1;
|
||||
const delay = retryDelay(null, failures - 1, now());
|
||||
cooldowns.set(endpoint, { until: now() + delay, failures, status: 502 });
|
||||
while (cooldowns.size > 64)
|
||||
cooldowns.delete(cooldowns.keys().next().value);
|
||||
failure = refusal(502, delay);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
if (lastRateLimitPayload) return lastRateLimitPayload;
|
||||
if (lastRefusalPayload) return lastRefusalPayload;
|
||||
throw lastError || new Error('All Overpass upstreams failed');
|
||||
return failure;
|
||||
}
|
||||
|
||||
export { overpassPayloadIsData, fetchOverpassPayload };
|
||||
@@ -1,13 +1,18 @@
|
||||
/** Place-search and route middleware for Node development servers. */
|
||||
export { googlePlacesContextProxy } from './places/google.js';
|
||||
export {
|
||||
googlePlacesContextProxy,
|
||||
validateGeocodeAddress,
|
||||
geocodeBounds,
|
||||
} from './places/google.js';
|
||||
export {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './places/google-key.js';
|
||||
export { installRouteMiddleware } from './places/routes.js';
|
||||
export {
|
||||
makeRateLimiter,
|
||||
makeOptInRateLimiter,
|
||||
makeCostRateLimiter,
|
||||
clientKey,
|
||||
} from './common/rate-limit.js';
|
||||
export { haversineKm } from './common/geo.js';
|
||||
@@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search, the
|
||||
* CCTV Street View fallback). These never reach the browser, so this key can
|
||||
* be restricted by server IP and scoped to Places API + Street View Static
|
||||
* API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles +
|
||||
* Geocoding for the browser (#33). Splitting them is opt-in: unset, this
|
||||
* falls back to the shared browser key and nothing changes.
|
||||
* Geocoding speaks Google's own response shape rather than the `places: []`
|
||||
* contract its two siblings use, because the browser adapters read `status`
|
||||
* and `results` directly. The keyless capability reads the same way: a status
|
||||
* that is not `OK` and no results, which the adapters already treat as "this
|
||||
* provider did not answer" and pass to the next geocoder in the chain.
|
||||
*/
|
||||
export function keylessGoogleGeocodeResponse(apiKey) {
|
||||
if (String(apiKey ?? '').trim()) return null;
|
||||
return {
|
||||
statusCode: 200,
|
||||
payload: {
|
||||
configured: false,
|
||||
status: 'REQUEST_DENIED',
|
||||
results: [],
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search,
|
||||
* geocoding, the CCTV Street View fallback). These never reach the browser, so
|
||||
* this key can be restricted by server IP and scoped to Places API + Geocoding
|
||||
* API + Street View Static API — while GOOGLE_MAPS_API_KEY stays
|
||||
* referrer-restricted to Map Tiles + Maps JavaScript API for the browser
|
||||
* (#33, #363). Splitting them is opt-in: unset, this falls back to the shared
|
||||
* browser key, which then has to carry every API both sides use and cannot
|
||||
* take a referrer restriction.
|
||||
*/
|
||||
export function googleServerApiKey() {
|
||||
return resolveGoogleServerKey(process.env);
|
||||
|
||||
@@ -1,22 +1,38 @@
|
||||
import {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './google-key.js';
|
||||
import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { admitSameSite } from '../common/same-site.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import {
|
||||
projectNearbyPlaces,
|
||||
projectTextSearchPlaces,
|
||||
} from '../../../src/data/placeProviderPayloads.js';
|
||||
|
||||
// Construct lazily after the standalone environment has loaded.
|
||||
// undefined = not built yet; null = unlimited; fn = active limiter
|
||||
// undefined = not built yet; null = the explicit 0 opt-out; fn = active limiter
|
||||
let _googleRateLimiter;
|
||||
|
||||
/** Google cost endpoint (nearby-places). Null = unlimited (default). */
|
||||
/**
|
||||
* Requests/min/IP applied when GEV_RATELIMIT_GOOGLE_PER_MIN is unset. This is
|
||||
* the value the Pinokio build already ships (pinokio/_ENVIRONMENT), so the
|
||||
* packaged app keeps behaving exactly as it does today and only an
|
||||
* unconfigured server changes — from unlimited to what the product already
|
||||
* runs with. Places calls are user-driven (a search, a nearby lookup, an
|
||||
* installation probe) rather than polled, so this sits far above what the app
|
||||
* generates and bites only a caller enumerating the endpoint. `.env.example`
|
||||
* suggests a tighter 60 when the host is not localhost; that still applies.
|
||||
*/
|
||||
export const GOOGLE_DEFAULT_PER_MIN = 120;
|
||||
|
||||
/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */
|
||||
function googleRateLimiter() {
|
||||
if (_googleRateLimiter === undefined)
|
||||
_googleRateLimiter = makeOptInRateLimiter(
|
||||
_googleRateLimiter = makeCostRateLimiter(
|
||||
process.env.GEV_RATELIMIT_GOOGLE_PER_MIN,
|
||||
GOOGLE_DEFAULT_PER_MIN,
|
||||
);
|
||||
return _googleRateLimiter;
|
||||
}
|
||||
@@ -42,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) {
|
||||
return { ok: true, latitude, longitude };
|
||||
}
|
||||
|
||||
/** Longest accepted geocoder query. Real place names are far shorter. */
|
||||
const GEOCODE_MAX_QUERY = 200;
|
||||
|
||||
/**
|
||||
* Upstream deadline and body ceiling for the geocode routes. The deadline stays
|
||||
* armed through body consumption, so a stalled upstream cannot hold a handler
|
||||
* open, and the cap bounds a chunked or length-omitted reply. A geocode answer
|
||||
* is a handful of results; 256 KiB is far above any honest one.
|
||||
*/
|
||||
const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000;
|
||||
|
||||
const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024;
|
||||
|
||||
/** Validate a free-text geocoder query before consuming request quota. */
|
||||
export function validateGeocodeAddress(searchParams) {
|
||||
const address = String(searchParams.get('address') || '').trim();
|
||||
if (!address) return { ok: false, error: 'address is required' };
|
||||
if (address.length > GEOCODE_MAX_QUERY)
|
||||
return { ok: false, error: 'address is too long' };
|
||||
return { ok: true, address };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the optional viewport bias, which Google reads as
|
||||
* `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than
|
||||
* refused: the bias only ranks results, so a malformed one costs relevance,
|
||||
* not an answer.
|
||||
*/
|
||||
export function geocodeBounds(searchParams) {
|
||||
const raw = String(searchParams.get('bounds') || '').trim();
|
||||
if (!raw) return null;
|
||||
const corners = raw.split('|');
|
||||
if (corners.length !== 2) return null;
|
||||
const parts = corners.flatMap((corner) => corner.split(','));
|
||||
// A blank component must be refused before Number() turns it into 0 and a
|
||||
// half-written corner like `30.4,` reads as the equator.
|
||||
if (parts.length !== 4 || parts.some((part) => !part.trim())) return null;
|
||||
const numbers = parts.map(Number);
|
||||
if (!numbers.every(Number.isFinite)) return null;
|
||||
const [swLat, swLon, neLat, neLon] = numbers;
|
||||
if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null;
|
||||
if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null;
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** Nearby place labels and view-biased text search, with request-time key resolution. */
|
||||
export function googlePlacesContextProxy({
|
||||
resolveApiKey = googleServerApiKey,
|
||||
@@ -50,6 +111,9 @@ export function googlePlacesContextProxy({
|
||||
} = {}) {
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/google/nearby-places', async (req, res) => {
|
||||
// Gate first, like the OpenAI routes: a cross-site caller learns nothing
|
||||
// about this endpoint's method surface.
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
res.statusCode = 405;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
@@ -80,7 +144,7 @@ export function googlePlacesContextProxy({
|
||||
}
|
||||
const { latitude, longitude } = coordinates;
|
||||
|
||||
// Opt-in per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN). No-op when unset.
|
||||
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN). On by default; 0 disables.
|
||||
// Inlined (not the shared helper) so the 429 body keeps this endpoint's
|
||||
// `places: []` contract that the client expects on every error response.
|
||||
const _grl = googleRateLimiter();
|
||||
@@ -161,6 +225,8 @@ export function googlePlacesContextProxy({
|
||||
// a view-biased Text Search lands on the actual feature. Same key, field
|
||||
// mask, throttle, and `places: []` error contract as nearby-places above.
|
||||
middlewares.use('/api/google/text-search', async (req, res) => {
|
||||
// Paid like nearby-places, so gated the same way, first.
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
res.statusCode = 405;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
@@ -200,7 +266,7 @@ export function googlePlacesContextProxy({
|
||||
}
|
||||
const { latitude, longitude } = coordinates;
|
||||
|
||||
// Opt-in per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN). No-op when unset.
|
||||
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN). On by default; 0 disables.
|
||||
// Inlined (like nearby-places) so the 429 body keeps the `places: []`
|
||||
// contract the client expects on every error response.
|
||||
const _grl = googleRateLimiter();
|
||||
@@ -274,6 +340,149 @@ export function googlePlacesContextProxy({
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Geocoding belongs on the server, not in the page. Google's Geocoding web
|
||||
* service refuses referrer-restricted keys, so calling it from the browser
|
||||
* forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be
|
||||
* left unrestricted (#363). Here the request carries the server key, and
|
||||
* the browser key no longer needs the Geocoding API at all.
|
||||
*
|
||||
* Unlike the two Places routes above, these answer in Google's own shape
|
||||
* (`status` + `results`) rather than the `places: []` contract: the browser
|
||||
* normalizers read that shape directly, and keeping it means the proxy adds
|
||||
* a hop without adding a translation nobody asked for.
|
||||
*/
|
||||
function installGeocodeRoute(path, readQuery) {
|
||||
middlewares.use(path, async (req, res) => {
|
||||
const refuse = (statusCode, error) => {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }),
|
||||
);
|
||||
};
|
||||
|
||||
// Gate first, like the Places routes above: a cross-site caller learns
|
||||
// nothing about this endpoint's method surface.
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
refuse(405, 'Method not allowed');
|
||||
return;
|
||||
}
|
||||
|
||||
// A keyless geocode has no provider cost, so it resolves before the
|
||||
// paid-endpoint limiter can consume or exhaust quota (mirrors the
|
||||
// Places routes above).
|
||||
const apiKey = resolveApiKey();
|
||||
const keyless = keylessGoogleGeocodeResponse(apiKey);
|
||||
if (keyless) {
|
||||
res.statusCode = keyless.statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(keyless.payload));
|
||||
return;
|
||||
}
|
||||
|
||||
const requestUrl = new URL(req.url || '', 'http://localhost');
|
||||
const query = readQuery(requestUrl.searchParams);
|
||||
if (!query.ok) {
|
||||
refuse(400, query.error);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the
|
||||
// Places routes because both spend the same Google budget.
|
||||
const _grl = googleRateLimiter();
|
||||
if (_grl && !_grl(clientKey(req))) {
|
||||
res.statusCode = 429;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Retry-After', '5');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'OVER_QUERY_LIMIT',
|
||||
results: [],
|
||||
error: 'Rate limit exceeded',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const upstream = new URL(
|
||||
endpoints.geocode ||
|
||||
'https://maps.googleapis.com/maps/api/geocode/json',
|
||||
);
|
||||
for (const [name, value] of Object.entries(query.params))
|
||||
upstream.searchParams.set(name, value);
|
||||
upstream.searchParams.set('key', apiKey);
|
||||
const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS);
|
||||
const response = await fetchImpl(upstream.toString(), {
|
||||
redirect: 'error',
|
||||
signal: deadline,
|
||||
});
|
||||
// An unreadable, oversized or timed-out body is not an answer: it
|
||||
// falls through to the same shaped failure as a refusal below.
|
||||
const data = await readResponseJsonCapped(
|
||||
response,
|
||||
GEOCODE_MAX_RESPONSE_BYTES,
|
||||
deadline,
|
||||
).catch(() => null);
|
||||
|
||||
// A body we could not read is an upstream failure, not an answer with
|
||||
// a status of its own, so it does not borrow the upstream's code.
|
||||
if (!data) {
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = response.ok ? 200 : response.status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'private, max-age=300');
|
||||
res.end(JSON.stringify(data));
|
||||
} catch {
|
||||
// Fixed public text: a transport, URL or runtime message would
|
||||
// describe the server to the browser (SECURITY.md, sanitized errors).
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
installGeocodeRoute('/api/google/geocode', (searchParams) => {
|
||||
const address = validateGeocodeAddress(searchParams);
|
||||
if (!address.ok) return address;
|
||||
const bounds = geocodeBounds(searchParams);
|
||||
return {
|
||||
ok: true,
|
||||
params: {
|
||||
address: address.address,
|
||||
...(bounds ? { bounds } : {}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => {
|
||||
const coordinates = validatePlacesCoordinates(searchParams);
|
||||
if (!coordinates.ok) return coordinates;
|
||||
const { latitude, longitude } = coordinates;
|
||||
return { ok: true, params: { latlng: `${latitude},${longitude}` } };
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import https from 'node:https';
|
||||
import { Readable } from 'node:stream';
|
||||
import { isNonGlobalIpv4 } from './stations.js';
|
||||
|
||||
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
|
||||
|
||||
export function radioMirrorOrigin(value) {
|
||||
const hostname = String(value ?? '')
|
||||
.toLowerCase()
|
||||
@@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) {
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
// Nothing thrown here may escape: this callback runs outside the
|
||||
// promise, so an exception would be uncaught and crash the server.
|
||||
try {
|
||||
const status = response.statusCode || 500;
|
||||
// A `Response` only represents statuses up to 599; Node never hands
|
||||
// this callback a 1xx.
|
||||
if (status > 599)
|
||||
throw new Error(
|
||||
`Radio Browser returned unsupported status ${status}`,
|
||||
);
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
}
|
||||
const nullBody = NULL_BODY_STATUSES.has(status);
|
||||
const upstream = new Response(
|
||||
nullBody ? null : Readable.toWeb(response),
|
||||
{ status, statusText: response.statusMessage || '', headers },
|
||||
);
|
||||
if (nullBody) response.resume();
|
||||
resolve(upstream);
|
||||
} catch (error) {
|
||||
response.destroy();
|
||||
reject(error);
|
||||
}
|
||||
resolve(
|
||||
new Response(Readable.toWeb(response), {
|
||||
status: response.statusCode || 500,
|
||||
statusText: response.statusMessage || '',
|
||||
headers,
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { makeRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { coalesceProxyRequest } from '../common/http.js';
|
||||
import { fetchRegionalJson } from './http.js';
|
||||
import { normalizeRegionalPlace } from '../../../src/data/regionalModel.js';
|
||||
import { naturalRegionAtPoint } from '../../../src/data/naturalEarthRegions.js';
|
||||
import {
|
||||
nominatimToGeocodeResult,
|
||||
nominatimViewboxFromBounds,
|
||||
@@ -51,8 +51,8 @@ const NOMINATIM_SEARCH_MAX_QUERY = 200;
|
||||
// The pacer is deliberately module state while everything else here is
|
||||
// per-instance. One request per second is a budget for the whole application,
|
||||
// not for each provider object: two instances each pacing themselves would
|
||||
// send two requests a second between them. Reverse lookups and searches
|
||||
// therefore queue together.
|
||||
// send two requests a second between them. Only last-resort forward searches
|
||||
// use this queue.
|
||||
let _nominatimQueue = Promise.resolve();
|
||||
|
||||
let _nominatimLastRequestAt = 0;
|
||||
@@ -77,9 +77,7 @@ function abandonedError() {
|
||||
* Run one piece of upstream work, never closer than the policy spacing to the
|
||||
* last one.
|
||||
*
|
||||
* `bounded` applies the queue-depth and staleness limits. The cockpit's reverse
|
||||
* lookups are a slow background trickle and stay unbounded; the search route,
|
||||
* which a person can fire as fast as they can type, does not.
|
||||
* `bounded` applies queue-depth and staleness limits to interactive forward searches.
|
||||
*
|
||||
* @param {() => Promise<unknown>} work
|
||||
* @param {{bounded?: boolean, signal?: AbortSignal}} [options]
|
||||
@@ -111,30 +109,9 @@ function enqueueNominatim(work, { bounded = false, signal } = {}) {
|
||||
return task;
|
||||
}
|
||||
|
||||
/** Construct the serialized Nominatim reverse adapter with a trusted endpoint. */
|
||||
export function createRegionalPlaceProvider({
|
||||
endpoint = 'https://nominatim.openstreetmap.org/reverse',
|
||||
requestJson = fetchRegionalJson,
|
||||
} = {}) {
|
||||
function fetchRegionalPlace(point) {
|
||||
return enqueueNominatim(async () => {
|
||||
const params = new URLSearchParams({
|
||||
format: 'jsonv2',
|
||||
lat: point.latitude.toFixed(5),
|
||||
lon: point.longitude.toFixed(5),
|
||||
zoom: '10',
|
||||
addressdetails: '1',
|
||||
'accept-language': 'en',
|
||||
});
|
||||
const payload = await requestJson(`${endpoint}?${params}`, {
|
||||
headers: NOMINATIM_HEADERS,
|
||||
redirect: 'error',
|
||||
});
|
||||
return normalizeRegionalPlace(payload);
|
||||
});
|
||||
}
|
||||
|
||||
return fetchRegionalPlace;
|
||||
/** Construct the offline regional context provider from bundled Natural Earth polygons. */
|
||||
export function createRegionalPlaceProvider() {
|
||||
return (point) => naturalRegionAtPoint(point.latitude, point.longitude);
|
||||
}
|
||||
|
||||
export const fetchRegionalPlace = createRegionalPlaceProvider();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: CelesTrak TLE proxy.
|
||||
@@ -48,10 +49,14 @@ export function celestrakProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
|
||||
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
async function fetchUpstream(group) {
|
||||
const url = celestrakTleUrl(group);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
const res = await fetch(url.toString(), {
|
||||
signal: AbortSignal.timeout(20000),
|
||||
signal,
|
||||
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
|
||||
// descriptive User-Agent with a contact point.
|
||||
headers: {
|
||||
@@ -60,7 +65,11 @@ export function celestrakProxy() {
|
||||
},
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const body = await res.text();
|
||||
const body = await readResponseTextCapped(
|
||||
res,
|
||||
CELESTRAK_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
|
||||
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
|
||||
return { at: Date.now(), body };
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { readResponseBytesCapped } from './common/http.js';
|
||||
import {
|
||||
isValidTileCoord as isValidTomTomTile,
|
||||
utcDayKey as tomtomUtcDayKey,
|
||||
@@ -23,9 +24,16 @@ import {
|
||||
* Budget governor (mirrors the OpenSky credit-governor philosophy — last-good
|
||||
* data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json,
|
||||
* keyed by UTC date, reset on day change) counts upstream fetch attempts
|
||||
* against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free
|
||||
* tier's ~50k/day). Over the cap the proxy serves stale tiles when available,
|
||||
* else 429 {error:'budget'}.
|
||||
* against a soft cap (TOMTOM_DAILY_TILE_BUDGET). Over the cap the proxy
|
||||
* serves stale tiles when available, else 429 {error:'budget'}.
|
||||
*
|
||||
* The default is derived from TomTom's published free allowance, which is
|
||||
* granted MONTHLY, not daily: 200,000 Traffic Flow & Incidents tile requests
|
||||
* per month (https://docs.tomtom.com/pricing/). The previous 40,000/day
|
||||
* default — and the "~50k/day" figure this comment used to cite — exhausted
|
||||
* a month's entire allowance in five days, leaving the traffic layer dead
|
||||
* for the rest of the billing period. 6,000/day keeps a full 31-day month
|
||||
* inside the allowance (186,000) with headroom to spare.
|
||||
*
|
||||
* GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode:
|
||||
* status reports hasKey:false and the tile endpoint 503s {error:'no_key'}
|
||||
@@ -37,9 +45,12 @@ export function tomtomProxy() {
|
||||
const TILE_TTL_MS = 120_000;
|
||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom');
|
||||
const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json');
|
||||
const DEFAULT_DAILY_BUDGET = 40000;
|
||||
// 200,000/month free tier / 31 days = 6,451. Rounded down for headroom.
|
||||
const DEFAULT_DAILY_BUDGET = 6000;
|
||||
const MEM_MAX_ENTRIES = 256;
|
||||
const UPSTREAM_TIMEOUT_MS = 15000;
|
||||
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
|
||||
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
|
||||
const mem = new Map();
|
||||
@@ -135,11 +146,15 @@ export function tomtomProxy() {
|
||||
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
|
||||
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
|
||||
recordUpstreamFetch(); // attempts count — upstream bills the request either way
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
|
||||
});
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
const bytes = await readResponseBytesCapped(
|
||||
res,
|
||||
TOMTOM_TILE_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (buf.length === 0) throw new Error('empty tile body');
|
||||
return buf;
|
||||
}
|
||||
|
||||
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
const rows = Array.from({ length: 15000 }, (_, i) =>
|
||||
row(now, `vehicle-${i}`),
|
||||
);
|
||||
const start = performance.now();
|
||||
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
|
||||
const start = process.threadCpuUsage();
|
||||
h.ingest(feed, rows, now);
|
||||
times.push(performance.now() - start);
|
||||
const usage = process.threadCpuUsage(start);
|
||||
times.push((usage.user + usage.system) / 1000);
|
||||
}
|
||||
assert.equal(h.diagnostics().keys, 15000);
|
||||
for (let i = 0; i < 15000; i++) {
|
||||
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
}
|
||||
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
|
||||
t.diagnostic(
|
||||
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
);
|
||||
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
|
||||
// well under two seconds on a loaded laptop or a slow CI runner. The
|
||||
// per-run figure above is the diagnostic to read.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
|
||||
// CPU time measures ingestion work without counting scheduler delays on
|
||||
// loaded runners. Keep the two-second ceiling as a guard against quadratic
|
||||
// work, not a benchmark; the per-poll figures above show any slowdown.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
|
||||
});
|
||||
|
||||
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {
|
||||
|
||||
@@ -71,14 +71,15 @@ let _aisWebSocketImpl;
|
||||
*/
|
||||
export function aisLiveProxy() {
|
||||
function install(middlewares) {
|
||||
middlewares.use('/api/ais-live', async (req, res) => {
|
||||
middlewares.use('/api/vessels', async (req, res) => {
|
||||
try {
|
||||
res.setHeader('X-Feed-Source', 'AISStream');
|
||||
ensureAisStreamConnection();
|
||||
const incoming = new URL(req.url || '', 'http://localhost');
|
||||
|
||||
// Track sub-route MUST be handled before the rows snapshot — this
|
||||
// mount prefix-matches every subpath, so without this branch
|
||||
// /api/ais-live/track would be silently answered with vessel rows.
|
||||
// /api/vessels/track would be silently answered with vessel rows.
|
||||
if (
|
||||
incoming.pathname === '/track' ||
|
||||
incoming.pathname.startsWith('/track/')
|
||||
@@ -288,7 +289,7 @@ function aisKeyFingerprint() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive the watchdog once. Called on every /api/ais-live request and on the
|
||||
* Drive the watchdog once. Called on every /api/vessels request and on the
|
||||
* background interval, so recovery does not depend on browser traffic.
|
||||
*/
|
||||
function ensureAisStreamConnection() {
|
||||
@@ -307,7 +308,7 @@ function ensureAisStreamConnection() {
|
||||
keyFingerprint: aisKeyFingerprint(),
|
||||
});
|
||||
}
|
||||
/** Status metadata for /api/ais-live, safe to call before the first connect. */
|
||||
/** Status metadata for /api/vessels, safe to call before the first connect. */
|
||||
function aisStreamStatusSnapshot() {
|
||||
const snapshot = _aisAdapter ? _aisAdapter.snapshot() : null;
|
||||
if (snapshot) return snapshot;
|
||||
|
||||
@@ -72,8 +72,8 @@ export function ingestAisStreamEnvelope(envelope) {
|
||||
imo: stringValue(message.ImoNumber ?? message.IMO ?? staticData.imo),
|
||||
type: vesselTypeFromAis(message, staticData),
|
||||
destination: stringValue(message.Destination ?? staticData.destination),
|
||||
speed: numberValue(message.Sog ?? message.SOG),
|
||||
course: numberValue(message.Cog ?? message.COG),
|
||||
speed: normalizedSpeedOverGround(message.Sog ?? message.SOG),
|
||||
course: normalizedCourseOverGround(message.Cog ?? message.COG),
|
||||
heading: normalizedHeading(message.TrueHeading ?? message.Heading),
|
||||
last_position_UTC: normalizeAisTimestamp(
|
||||
metadata.time_utc ?? metadata.TimeUtc,
|
||||
@@ -277,6 +277,28 @@ function normalizedHeading(value) {
|
||||
return heading !== null && heading >= 0 && heading <= 360 ? heading : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Speed over ground in knots, or null when the report carries AIS's "not
|
||||
* available" code. ITU-R M.1371 encodes SOG in 0.1-knot units where 1022 means
|
||||
* "102.2 knots or higher" and 1023 means unavailable, so 102.3 is a sentinel
|
||||
* and not a reading. It is finite, so every consumer would otherwise treat it
|
||||
* as a genuine speed.
|
||||
*/
|
||||
function normalizedSpeedOverGround(value) {
|
||||
const speed = numberValue(value);
|
||||
return speed !== null && speed >= 0 && speed < 102.3 ? speed : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Course over ground in degrees, or null when the report carries AIS's "not
|
||||
* available" code (3600 in 0.1-degree units, i.e. 360). Valid courses stop at
|
||||
* 359.9; heading keeps its own separate sentinel of 511.
|
||||
*/
|
||||
function normalizedCourseOverGround(value) {
|
||||
const course = numberValue(value);
|
||||
return course !== null && course >= 0 && course < 360 ? course : null;
|
||||
}
|
||||
|
||||
function normalizeAisTimestamp(value) {
|
||||
const text = stringValue(value);
|
||||
if (!text) return new Date().toISOString();
|
||||
|
||||
@@ -0,0 +1,610 @@
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
import { readWindBody as readBytesCapped } from '../../src/sources/windBody.js';
|
||||
|
||||
const BASE = 'https://nowcoast.noaa.gov/geoserver/observations/';
|
||||
const HOUR = 3600_000;
|
||||
const PRODUCTS = Object.freeze({
|
||||
lightning: Object.freeze({
|
||||
service: 'lightning_detection',
|
||||
layer: 'ldn_lightning_strike_density',
|
||||
style: 'lightning_density',
|
||||
title: 'Lightning density · 15 min',
|
||||
coverage:
|
||||
'Pacific and Americas: 110°E across the dateline to 0°, 25°S–80°N; not global coverage.',
|
||||
description:
|
||||
'Observed 15-minute lightning strike density on an approximately 8 km grid, scaled as strikes/km²/min ×10³. Ground-network density, not individual GLM flashes.',
|
||||
attribution: 'NOAA/NWS nowCOAST; derived from Vaisala NLDN/GLD360',
|
||||
metadataTtlMs: 600_000,
|
||||
image: Object.freeze({ width: 4096, height: 2048 }),
|
||||
}),
|
||||
radar: Object.freeze({
|
||||
service: 'weather_radar',
|
||||
layer: 'conus_base_reflectivity_mosaic',
|
||||
style: 'weather_radar_base_reflectivity',
|
||||
title: 'CONUS radar reflectivity',
|
||||
coverage:
|
||||
'Contiguous United States; gaps do not establish absence of precipitation.',
|
||||
description:
|
||||
'Observed MRMS radar base reflectivity (dBZ), approximately 1 km and 4-minute updates; not a rainfall forecast.',
|
||||
image: Object.freeze({ width: 4096, height: 2048 }),
|
||||
}),
|
||||
clouds: Object.freeze({
|
||||
service: 'satellite',
|
||||
layer: 'global_longwave_imagery_mosaic',
|
||||
style: 'reflectance',
|
||||
title: 'Global satellite infrared',
|
||||
coverage:
|
||||
'Global mosaic with incomplete polar coverage; nominal coverage 60°S–60°N.',
|
||||
description:
|
||||
'Longwave infrared cloud and land/sea temperature patterns, approximately 3 km; hourly updates with 2–3 hour source latency. Not a cloud-only mask.',
|
||||
image: Object.freeze({ width: 2048, height: 1024 }),
|
||||
}),
|
||||
'clouds-regional': Object.freeze({
|
||||
service: 'satellite',
|
||||
layer: 'goes_longwave_imagery',
|
||||
style: 'goes-lir',
|
||||
title: 'GOES regional satellite infrared',
|
||||
coverage:
|
||||
'GOES East/West regional North American coverage; not a global image.',
|
||||
description:
|
||||
'GOES-19/18 longwave infrared Band 14 cloud and surface temperature patterns, approximately 2 km and 5-minute updates. Not a cloud-only mask.',
|
||||
image: Object.freeze({ width: 4096, height: 2048 }),
|
||||
}),
|
||||
});
|
||||
|
||||
// Whole-extent images: 2:1 sizes up to each product's largest (the default).
|
||||
const IMAGE_SIZES = Object.freeze(['1024x512', '2048x1024', '4096x2048']);
|
||||
// Detail windows: any product up to 4096×2048 (the default).
|
||||
const DETAIL_IMAGE = Object.freeze({ width: 4096, height: 2048 });
|
||||
const BBOX_STEP = 0.25;
|
||||
const MAX_IMAGE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
function failure(code, status = 503) {
|
||||
return Object.assign(new Error(code), { code, status });
|
||||
}
|
||||
|
||||
/** Canonicalize only explicit UTC observations; never expand time intervals. */
|
||||
function observationTime(value) {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value)
|
||||
)
|
||||
return null;
|
||||
const date = new Date(value);
|
||||
if (!Number.isFinite(date.getTime())) return null;
|
||||
const canonical = date.toISOString();
|
||||
return canonical.replace('.000Z', 'Z') === value.replace('.000Z', 'Z')
|
||||
? canonical
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Read the named leaf from bounded capabilities without resolving XML entities. */
|
||||
export function parseWeatherCapabilities(xml, product, nowMs = Date.now()) {
|
||||
const spec = PRODUCTS[product];
|
||||
if (
|
||||
!spec ||
|
||||
typeof xml !== 'string' ||
|
||||
xml.length > 512 * 1024 ||
|
||||
/<!DOCTYPE|<!ENTITY/i.test(xml)
|
||||
)
|
||||
throw failure('invalid_weather_metadata');
|
||||
const stack = [];
|
||||
let leaf = null;
|
||||
let tags = 0;
|
||||
for (const match of xml.matchAll(/<\/?(?:[\w.-]+:)?Layer\b[^>]*>/g)) {
|
||||
if (++tags > 2048) throw failure('invalid_weather_metadata');
|
||||
if (!match[0].startsWith('</')) {
|
||||
if (stack.length) stack.at(-1).nested = true;
|
||||
if (stack.length >= 16 || match[0].endsWith('/>'))
|
||||
throw failure('invalid_weather_metadata');
|
||||
stack.push({ start: match.index + match[0].length, nested: false });
|
||||
} else {
|
||||
const opened = stack.pop();
|
||||
if (!opened) throw failure('invalid_weather_metadata');
|
||||
if (opened.nested) continue;
|
||||
const candidate = xml.slice(opened.start, match.index);
|
||||
const name = candidate
|
||||
.match(/<(?:[\w.-]+:)?Name\s*>([^<]+)<\/(?:[\w.-]+:)?Name>/)?.[1]
|
||||
?.trim();
|
||||
if (name === spec.layer) {
|
||||
if (leaf !== null) throw failure('invalid_weather_metadata');
|
||||
leaf = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (stack.length || !leaf) throw failure('invalid_weather_metadata');
|
||||
const box = leaf.match(
|
||||
/<(?:[\w.-]+:)?EX_GeographicBoundingBox\s*>([\s\S]*?)<\/(?:[\w.-]+:)?EX_GeographicBoundingBox>/,
|
||||
)?.[1];
|
||||
const bounds = {};
|
||||
for (const [key, tag] of Object.entries({
|
||||
west: 'westBoundLongitude',
|
||||
south: 'southBoundLatitude',
|
||||
east: 'eastBoundLongitude',
|
||||
north: 'northBoundLatitude',
|
||||
})) {
|
||||
const value = box
|
||||
?.match(
|
||||
new RegExp(`<(?:[\\w.-]+:)?${tag}\\s*>([^<]+)</(?:[\\w.-]+:)?${tag}>`),
|
||||
)?.[1]
|
||||
?.trim();
|
||||
bounds[key] = value ? Number(value) : NaN;
|
||||
}
|
||||
if (
|
||||
!Object.values(bounds).every(Number.isFinite) ||
|
||||
bounds.west < -180 ||
|
||||
bounds.east > 180 ||
|
||||
bounds.south < -90 ||
|
||||
bounds.north > 90 ||
|
||||
bounds.west >= bounds.east ||
|
||||
bounds.south >= bounds.north
|
||||
)
|
||||
throw failure('invalid_weather_metadata');
|
||||
const dimensions = [
|
||||
...leaf.matchAll(
|
||||
/<(?:[\w.-]+:)?Dimension\b([^>]*)>([^<]*)<\/(?:[\w.-]+:)?Dimension>/g,
|
||||
),
|
||||
].filter((entry) => /\bname\s*=\s*["']time["']/.test(entry[1]));
|
||||
if (
|
||||
dimensions.length !== 1 ||
|
||||
!/\bunits\s*=\s*["']ISO8601["']/.test(dimensions[0][1])
|
||||
)
|
||||
throw failure('invalid_weather_metadata');
|
||||
const raw = dimensions[0][2].trim().split(',');
|
||||
if (!raw.length || raw.length > 512)
|
||||
throw failure('invalid_weather_metadata');
|
||||
const times = raw.map((value) => observationTime(value.trim()));
|
||||
if (times.some((value) => !value || Date.parse(value) > nowMs + 5 * 60_000))
|
||||
throw failure('invalid_weather_metadata');
|
||||
const defaultTime = observationTime(
|
||||
dimensions[0][1].match(/\bdefault\s*=\s*["']([^"']+)["']/)?.[1],
|
||||
);
|
||||
if (!defaultTime || !times.includes(defaultTime))
|
||||
throw failure('invalid_weather_metadata');
|
||||
const recent = [...new Set(times)]
|
||||
.filter((value) => nowMs - Date.parse(value) <= 24 * HOUR)
|
||||
.sort()
|
||||
.slice(-26);
|
||||
if (!recent.length) throw failure('weather_observations_expired');
|
||||
return { bounds, times: recent.slice(-13), allowedTimes: recent };
|
||||
}
|
||||
|
||||
/** A detail window `west,south,east,north` in degrees, rounded to 0.25° so cache
|
||||
* keys repeat, with a 2:1 aspect within 1 %; null when absent. Containment in
|
||||
* the product bounds is checked once the metadata is known. */
|
||||
export function weatherImageBbox(value) {
|
||||
if (value === null) return null;
|
||||
const parts = value.split(',');
|
||||
if (
|
||||
parts.length !== 4 ||
|
||||
parts.some((part) => !/^-?\d{1,3}(?:\.\d{1,6})?$/.test(part))
|
||||
)
|
||||
throw failure('invalid_weather_bbox', 400);
|
||||
// `+ 0` turns a rounded -0 into 0 so equal windows share one key.
|
||||
const [west, south, east, north] = parts.map(
|
||||
(part) => Math.round(Number(part) / BBOX_STEP) * BBOX_STEP + 0,
|
||||
);
|
||||
if (
|
||||
west < -180 ||
|
||||
east > 180 ||
|
||||
south < -90 ||
|
||||
north > 90 ||
|
||||
west >= east ||
|
||||
south >= north ||
|
||||
Math.abs((east - west) / (north - south) / 2 - 1) > 0.01
|
||||
)
|
||||
throw failure('invalid_weather_bbox', 400);
|
||||
return [west, south, east, north];
|
||||
}
|
||||
|
||||
/** GeographicTilingScheme's two longitude tiles and one latitude tile at level zero. */
|
||||
export function weatherTileBounds(z, x, y) {
|
||||
if (
|
||||
![z, x, y].every(Number.isInteger) ||
|
||||
z < 0 ||
|
||||
z > 6 ||
|
||||
x < 0 ||
|
||||
y < 0 ||
|
||||
x >= 2 ** (z + 1) ||
|
||||
y >= 2 ** z
|
||||
)
|
||||
throw failure('invalid_weather_tile', 400);
|
||||
const span = 180 / 2 ** z;
|
||||
return [
|
||||
-180 + x * span,
|
||||
90 - (y + 1) * span,
|
||||
-180 + (x + 1) * span,
|
||||
90 - y * span,
|
||||
];
|
||||
}
|
||||
|
||||
function validatePng(bytes, width, height) {
|
||||
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (
|
||||
buffer.length < 33 ||
|
||||
!buffer
|
||||
.subarray(0, 8)
|
||||
.equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ||
|
||||
buffer.readUInt32BE(8) !== 13 ||
|
||||
buffer.toString('ascii', 12, 16) !== 'IHDR' ||
|
||||
buffer.readUInt32BE(16) !== width ||
|
||||
buffer.readUInt32BE(20) !== height
|
||||
)
|
||||
throw failure('invalid_weather_image');
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/** Fixed NOAA observed-weather metadata and WMS tiles; no client-supplied destinations. */
|
||||
export function weatherProxy({
|
||||
fetchImpl = fetch,
|
||||
now = () => Date.now(),
|
||||
timeoutMs = 12_000,
|
||||
} = {}) {
|
||||
const metadata = new Map();
|
||||
const attempts = new Map();
|
||||
const operations = new Map();
|
||||
const pending = [];
|
||||
const tiles = new Map();
|
||||
const tileFailures = new Map();
|
||||
let tileBytes = 0;
|
||||
let active = 0;
|
||||
|
||||
function pump() {
|
||||
while (active < 8 && pending.length) {
|
||||
const operation = pending.shift();
|
||||
if (operation.controller.signal.aborted) continue;
|
||||
active++;
|
||||
operation.started = true;
|
||||
Promise.resolve()
|
||||
.then(() => operation.work(operation.controller.signal))
|
||||
.then(operation.resolve, operation.reject)
|
||||
.finally(() => {
|
||||
active--;
|
||||
pump();
|
||||
});
|
||||
}
|
||||
}
|
||||
async function shared(key, work, clientSignal) {
|
||||
clientSignal.throwIfAborted();
|
||||
let operation = operations.get(key);
|
||||
if (operation?.controller.signal.aborted) {
|
||||
operations.delete(key);
|
||||
operation = null;
|
||||
}
|
||||
if (!operation) {
|
||||
if (operations.size >= 120 || pending.length >= 96)
|
||||
throw failure('weather_busy', 429);
|
||||
const controller = new AbortController();
|
||||
operation = { controller, work, waiters: 0, started: false };
|
||||
operation.promise = new Promise((resolve, reject) => {
|
||||
operation.resolve = resolve;
|
||||
operation.reject = reject;
|
||||
});
|
||||
const cancel = () => {
|
||||
const index = pending.indexOf(operation);
|
||||
if (index >= 0) pending.splice(index, 1);
|
||||
operation.reject(failure('weather_request_cancelled'));
|
||||
};
|
||||
controller.signal.addEventListener('abort', cancel, { once: true });
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
operation.promise = operation.promise.finally(() => {
|
||||
clearTimeout(timer);
|
||||
controller.signal.removeEventListener('abort', cancel);
|
||||
if (operations.get(key) === operation) operations.delete(key);
|
||||
});
|
||||
operations.set(key, operation);
|
||||
pending.push(operation);
|
||||
pump();
|
||||
}
|
||||
operation.waiters++;
|
||||
const abort = () => {
|
||||
if (--operation.waiters === 0) operation.controller.abort();
|
||||
};
|
||||
clientSignal.addEventListener('abort', abort, { once: true });
|
||||
try {
|
||||
const value = await operation.promise;
|
||||
clientSignal.throwIfAborted();
|
||||
return value;
|
||||
} finally {
|
||||
clientSignal.removeEventListener('abort', abort);
|
||||
if (!clientSignal.aborted) operation.waiters--;
|
||||
}
|
||||
}
|
||||
async function upstream(url, signal, image = null) {
|
||||
signal.throwIfAborted();
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: { Accept: image ? 'image/png' : 'application/xml,text/xml' },
|
||||
});
|
||||
if (
|
||||
!response.ok ||
|
||||
(image &&
|
||||
!/^image\/png(?:;|$)/i.test(response.headers.get('content-type') || ''))
|
||||
) {
|
||||
await response.body?.cancel();
|
||||
throw failure('weather_upstream_unavailable');
|
||||
}
|
||||
const value = image
|
||||
? validatePng(
|
||||
await readBytesCapped(response, image.maxBytes, signal),
|
||||
image.width,
|
||||
image.height,
|
||||
)
|
||||
: await readResponseTextCapped(response, 512 * 1024, signal);
|
||||
signal.throwIfAborted();
|
||||
return value;
|
||||
}
|
||||
async function getMetadata(product, signal) {
|
||||
signal.throwIfAborted();
|
||||
const old = metadata.get(product);
|
||||
const spec = PRODUCTS[product];
|
||||
if (old && now() - old.fetchedAt < (spec.metadataTtlMs ?? 120_000))
|
||||
return { ...old, stale: false };
|
||||
try {
|
||||
if (
|
||||
now() - (attempts.get(product) ?? -Infinity) < 30_000 &&
|
||||
!operations.has(`metadata:${spec.service}`)
|
||||
)
|
||||
throw failure('weather_upstream_unavailable');
|
||||
attempts.set(product, now());
|
||||
const url = `${BASE}${spec.service}/ows?service=WMS&version=1.3.0&request=GetCapabilities`;
|
||||
const xml = await shared(
|
||||
`metadata:${spec.service}`,
|
||||
(activeSignal) => upstream(url, activeSignal),
|
||||
signal,
|
||||
);
|
||||
const value = {
|
||||
...parseWeatherCapabilities(xml, product, now()),
|
||||
fetchedAt: now(),
|
||||
};
|
||||
metadata.set(product, value);
|
||||
return { ...value, stale: false };
|
||||
} catch (error) {
|
||||
if (signal.aborted) attempts.delete(product);
|
||||
signal.throwIfAborted();
|
||||
if (error.status === 429) throw error;
|
||||
if (old && now() - old.fetchedAt <= HOUR) return { ...old, stale: true };
|
||||
throw failure('weather_metadata_unavailable');
|
||||
}
|
||||
}
|
||||
function rememberTile(key, bytes) {
|
||||
if (tiles.has(key)) tileBytes -= tiles.get(key).bytes.length;
|
||||
tiles.delete(key);
|
||||
while (tiles.size >= 128 || tileBytes + bytes.length > 16 * 1024 * 1024) {
|
||||
const oldest = tiles.keys().next().value;
|
||||
tileBytes -= tiles.get(oldest).bytes.length;
|
||||
tiles.delete(oldest);
|
||||
}
|
||||
tiles.set(key, { bytes, at: now() });
|
||||
tileBytes += bytes.length;
|
||||
}
|
||||
function describe(product, value) {
|
||||
const spec = PRODUCTS[product];
|
||||
const time = value?.times.at(-1) ?? null;
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
product,
|
||||
title: spec.title,
|
||||
coverage: spec.coverage,
|
||||
description: spec.description,
|
||||
source: 'NOAA nowCOAST',
|
||||
attribution: spec.attribution ?? 'NOAA/NWS/NESDIS nowCOAST',
|
||||
bounds: value?.bounds ?? null,
|
||||
times: value?.times ?? [],
|
||||
latest: time,
|
||||
time,
|
||||
observedAt: time,
|
||||
fetchedAt: value?.fetchedAt ?? null,
|
||||
stale: value?.stale ?? true,
|
||||
unavailable: !value,
|
||||
reason: !value
|
||||
? 'Weather imagery unavailable'
|
||||
: value.stale
|
||||
? 'Cached weather metadata; upstream unavailable'
|
||||
: null,
|
||||
tileSize: 256,
|
||||
maxLevel: 6,
|
||||
tilingScheme: 'geographic',
|
||||
tileTemplate: time
|
||||
? `/api/weather/tile?product=${product}&time=${encodeURIComponent(time)}&z={z}&x={x}&y={y}`
|
||||
: null,
|
||||
imageUrl: time
|
||||
? `/api/weather/image?product=${product}&time=${encodeURIComponent(time)}`
|
||||
: null,
|
||||
imageSize: { ...spec.image },
|
||||
};
|
||||
}
|
||||
function json(res, status, body) {
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 429 ? { 'Retry-After': '2' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
async function handler(req, res) {
|
||||
const controller = new AbortController();
|
||||
const close = () => controller.abort();
|
||||
res.once?.('close', close);
|
||||
try {
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
if (req.method !== 'GET')
|
||||
return json(res, 405, { error: 'method_not_allowed' });
|
||||
if (!['/manifest', '/tile', '/image'].includes(url.pathname))
|
||||
return json(res, 404, { error: 'not_found' });
|
||||
const allowed =
|
||||
url.pathname === '/manifest'
|
||||
? ['product']
|
||||
: url.pathname === '/image'
|
||||
? ['product', 'time', 'size', 'bbox']
|
||||
: ['product', 'time', 'z', 'x', 'y', 'size'];
|
||||
if (
|
||||
req.url.length > 512 ||
|
||||
[...url.searchParams.keys()].some(
|
||||
(key) =>
|
||||
!allowed.includes(key) || url.searchParams.getAll(key).length !== 1,
|
||||
)
|
||||
)
|
||||
return json(res, 400, { error: 'invalid_weather_query' });
|
||||
const product = url.searchParams.get('product');
|
||||
if (!Object.hasOwn(PRODUCTS, product))
|
||||
return json(res, 400, { error: 'unknown_weather_product' });
|
||||
const wholeImage = url.pathname === '/image';
|
||||
const detailBox = wholeImage
|
||||
? weatherImageBbox(url.searchParams.get('bbox'))
|
||||
: null;
|
||||
const largest = detailBox ? DETAIL_IMAGE : PRODUCTS[product].image;
|
||||
const size =
|
||||
url.searchParams.get('size') ??
|
||||
(wholeImage ? `${largest.width}x${largest.height}` : '256');
|
||||
if (
|
||||
wholeImage
|
||||
? !IMAGE_SIZES.includes(size) ||
|
||||
Number.parseInt(size, 10) > largest.width
|
||||
: !['256', '512', '1024'].includes(size)
|
||||
)
|
||||
throw failure(
|
||||
wholeImage
|
||||
? 'invalid_weather_image_size'
|
||||
: 'invalid_weather_tile_size',
|
||||
400,
|
||||
);
|
||||
if (url.pathname === '/manifest') {
|
||||
try {
|
||||
const value = await getMetadata(product, controller.signal);
|
||||
controller.signal.throwIfAborted();
|
||||
return json(res, 200, describe(product, value));
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) return;
|
||||
if (error.status === 429) throw error;
|
||||
return json(res, 200, describe(product, null));
|
||||
}
|
||||
}
|
||||
const coords = wholeImage
|
||||
? null
|
||||
: ['z', 'x', 'y'].map((key) => url.searchParams.get(key));
|
||||
if (coords?.some((value) => !/^(?:0|[1-9]\d{0,2})$/.test(value ?? '')))
|
||||
throw failure('invalid_weather_tile', 400);
|
||||
const tileBounds = coords
|
||||
? weatherTileBounds(...coords.map(Number))
|
||||
: null;
|
||||
const time = url.searchParams.get('time');
|
||||
if (!time || observationTime(time) !== time)
|
||||
throw failure('invalid_weather_time', 400);
|
||||
const value = await getMetadata(product, controller.signal);
|
||||
controller.signal.throwIfAborted();
|
||||
if (
|
||||
!value.allowedTimes.includes(time) ||
|
||||
now() - Date.parse(time) > 24 * HOUR
|
||||
)
|
||||
throw failure('unknown_weather_time', 400);
|
||||
if (
|
||||
detailBox &&
|
||||
(detailBox[0] < value.bounds.west ||
|
||||
detailBox[1] < value.bounds.south ||
|
||||
detailBox[2] > value.bounds.east ||
|
||||
detailBox[3] > value.bounds.north)
|
||||
)
|
||||
throw failure('invalid_weather_bbox', 400);
|
||||
// A single advertised extent keeps NOAA's global reflectance contrast
|
||||
// consistent across the image; independently normalized tiles create seams.
|
||||
const bbox =
|
||||
detailBox ??
|
||||
(wholeImage
|
||||
? [
|
||||
value.bounds.west,
|
||||
value.bounds.south,
|
||||
value.bounds.east,
|
||||
value.bounds.north,
|
||||
]
|
||||
: tileBounds);
|
||||
const [width, height] = wholeImage
|
||||
? size.split('x').map(Number)
|
||||
: [Number(size), Number(size)];
|
||||
const imageShape = {
|
||||
width,
|
||||
height,
|
||||
maxBytes: wholeImage
|
||||
? MAX_IMAGE_BYTES
|
||||
: Math.max(1024 * 1024, width ** 2 * 4 + 65_536),
|
||||
};
|
||||
// NOAA WMS capabilities advertise no MaxWidth/MaxHeight. One GetMap
|
||||
// returns a whole 4096×2048 extent or window, or a 1024 px tile, without
|
||||
// composition. Images are keyed by product, time, size and bbox.
|
||||
const key = `${product}:${time}:${wholeImage ? `image:${size}:${bbox.join(',')}` : `tile:${size}:${coords.join('/')}`}`;
|
||||
let cached = tiles.get(key);
|
||||
if (cached && now() - cached.at <= 24 * HOUR) {
|
||||
tiles.delete(key);
|
||||
tiles.set(key, cached);
|
||||
} else {
|
||||
// Never ask nearestValue=1 WMS to silently replace an expired frame.
|
||||
if (value.stale) throw failure('weather_metadata_stale');
|
||||
if (now() - (tileFailures.get(key) ?? -Infinity) < 30_000)
|
||||
throw failure('weather_upstream_unavailable');
|
||||
const spec = PRODUCTS[product];
|
||||
const upstreamUrl = new URL(`${BASE}${spec.service}/ows`);
|
||||
upstreamUrl.search = new URLSearchParams({
|
||||
service: 'WMS',
|
||||
version: '1.1.1',
|
||||
request: 'GetMap',
|
||||
layers: spec.layer,
|
||||
styles: spec.style,
|
||||
srs: 'EPSG:4326',
|
||||
bbox: bbox.join(','),
|
||||
width: String(imageShape.width),
|
||||
height: String(imageShape.height),
|
||||
format: 'image/png',
|
||||
transparent: 'true',
|
||||
time,
|
||||
}).toString();
|
||||
try {
|
||||
const bytes = await shared(
|
||||
`image:${key}`,
|
||||
(signal) => upstream(upstreamUrl.href, signal, imageShape),
|
||||
controller.signal,
|
||||
);
|
||||
rememberTile(key, bytes);
|
||||
cached = { bytes };
|
||||
} catch (error) {
|
||||
if (!controller.signal.aborted && error.status !== 429) {
|
||||
tileFailures.delete(key);
|
||||
tileFailures.set(key, now());
|
||||
while (tileFailures.size > 128)
|
||||
tileFailures.delete(tileFailures.keys().next().value);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'image/png',
|
||||
'Cache-Control': 'public, max-age=86400, immutable',
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
});
|
||||
res.end(cached.bytes);
|
||||
} catch (error) {
|
||||
if (!controller.signal.aborted)
|
||||
json(
|
||||
res,
|
||||
error.status === 400 || error.status === 429 ? error.status : 503,
|
||||
{
|
||||
error:
|
||||
error.status === 400 || error.status === 429
|
||||
? error.code
|
||||
: 'weather_upstream_unavailable',
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
res.removeListener?.('close', close);
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: 'weather',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/weather', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/weather', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import { fetchGfsWind } from './wind/gfs.js';
|
||||
import { fetchIfsWind } from './wind/ifs.js';
|
||||
import { decodeWindGribMessage } from './wind/decode.js';
|
||||
import { weatherScalarMetadata, weatherScalarError } from './wind/grid.js';
|
||||
|
||||
/** Serve bounded, single-flight, per-model/overlay forecast snapshots from fixed providers. */
|
||||
export function windProxy({
|
||||
fetchImpl = fetch,
|
||||
now = () => Date.now(),
|
||||
decodeImpl = decodeWindGribMessage,
|
||||
targetDx = 1,
|
||||
ttlMs = 3600_000,
|
||||
timeoutMs = 40_000,
|
||||
models = { gfs: fetchGfsWind, ifs: fetchIfsWind },
|
||||
} = {}) {
|
||||
// Request validation limits these maps to 2 models × 3 overlays.
|
||||
const caches = new Map();
|
||||
const loadings = new Map();
|
||||
const grids = new Map();
|
||||
const attempts = new Map();
|
||||
const unavailable = (model, overlay) => ({
|
||||
manifest: {
|
||||
model,
|
||||
schemaVersion: 1,
|
||||
...(overlay === 'none' ? {} : { overlay }),
|
||||
unavailable: true,
|
||||
stale: true,
|
||||
reason: 'Wind upstream unavailable',
|
||||
},
|
||||
});
|
||||
const sendJson = (res, value, status = 200) => {
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
function refresh(model, overlay, key) {
|
||||
const controller = new AbortController();
|
||||
const operation = { controller, waiters: 0, promise: null };
|
||||
loadings.set(key, operation);
|
||||
attempts.set(key, now());
|
||||
operation.promise = (async () => {
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const value = await models[model]({
|
||||
fetchImpl,
|
||||
now,
|
||||
decodeImpl,
|
||||
targetDx,
|
||||
overlay,
|
||||
signal: controller.signal,
|
||||
});
|
||||
controller.signal.throwIfAborted();
|
||||
const { grid, cycle } = value;
|
||||
const count = grid.nx * grid.ny;
|
||||
const expectedScalar = weatherScalarMetadata(overlay);
|
||||
const missingScalar =
|
||||
expectedScalar &&
|
||||
value.scalar === undefined &&
|
||||
value.scalarError === weatherScalarError(overlay);
|
||||
const scalar = missingScalar ? null : expectedScalar;
|
||||
if (
|
||||
!Number.isInteger(grid.nx) ||
|
||||
!Number.isInteger(grid.ny) ||
|
||||
grid.nx < 1 ||
|
||||
grid.ny < 1 ||
|
||||
count > 1_000_000 ||
|
||||
![grid.lo1, grid.la1, grid.dx, grid.dy].every(Number.isFinite) ||
|
||||
grid.dx <= 0 ||
|
||||
grid.dy <= 0 ||
|
||||
(scalar
|
||||
? ['kind', 'units', 'level'].some(
|
||||
(field) => value.scalar?.[field] !== scalar[field],
|
||||
)
|
||||
: value.scalar !== undefined || grid.scalar !== undefined) ||
|
||||
(!expectedScalar && value.scalarError !== undefined) ||
|
||||
(scalar && value.scalarError !== undefined)
|
||||
)
|
||||
throw new Error('Invalid wind grid');
|
||||
for (const values of [
|
||||
grid.u,
|
||||
grid.v,
|
||||
...(scalar ? [grid.scalar] : []),
|
||||
]) {
|
||||
if (
|
||||
!(values instanceof Float32Array) ||
|
||||
values.length !== count ||
|
||||
!values.every(Number.isFinite)
|
||||
)
|
||||
throw new Error('Invalid wind grid');
|
||||
}
|
||||
const suffix =
|
||||
overlay === 'none'
|
||||
? ''
|
||||
: `-${overlay}${missingScalar ? '-wind-only' : ''}`;
|
||||
const query = `model=${model}${overlay === 'none' ? '' : `&overlay=${overlay}`}`;
|
||||
const id = `${model}-${cycle.date}-${cycle.hour}-f${cycle.forecastHour || 0}-${targetDx}${suffix}`;
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
model,
|
||||
cycle,
|
||||
...(expectedScalar ? { overlay } : {}),
|
||||
...(scalar ? { scalar } : {}),
|
||||
...(missingScalar
|
||||
? { scalarError: weatherScalarError(overlay) }
|
||||
: {}),
|
||||
fetchedAt: now(),
|
||||
level: value.level,
|
||||
units: value.units,
|
||||
grid: {
|
||||
nx: grid.nx,
|
||||
ny: grid.ny,
|
||||
lo1: grid.lo1,
|
||||
la1: grid.la1,
|
||||
dx: grid.dx,
|
||||
dy: grid.dy,
|
||||
},
|
||||
stale: false,
|
||||
unavailable: false,
|
||||
reason: null,
|
||||
gridUrl: `/api/wind/grid/${id}.bin?${query}`,
|
||||
};
|
||||
const state = { id, grid, manifest, fetchedAt: now() };
|
||||
caches.set(key, state);
|
||||
// Retain the previous issued grid as well to cover a manifest/grid rollover.
|
||||
const history = grids.get(key) || new Map();
|
||||
history.set(id, state);
|
||||
while (history.size > 2) history.delete(history.keys().next().value);
|
||||
grids.set(key, history);
|
||||
return state;
|
||||
} catch {
|
||||
if (controller.signal.aborted && operation.waiters === 0) {
|
||||
attempts.delete(key);
|
||||
return unavailable(model, overlay);
|
||||
}
|
||||
const old = caches.get(key);
|
||||
if (old) {
|
||||
old.manifest = {
|
||||
...old.manifest,
|
||||
stale: true,
|
||||
reason: 'Wind upstream unavailable',
|
||||
};
|
||||
return old;
|
||||
}
|
||||
return unavailable(model, overlay);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
controller.abort(); // Cancel a sibling request if the other component failed.
|
||||
if (loadings.get(key) === operation) loadings.delete(key);
|
||||
}
|
||||
})();
|
||||
return operation;
|
||||
}
|
||||
const handler = async (req, res) => {
|
||||
const url = new URL(req.url, 'http://localhost');
|
||||
const model = url.searchParams.get('model') || 'gfs';
|
||||
const overlay = url.searchParams.get('overlay') || 'none';
|
||||
const key = `${model}:${overlay}`;
|
||||
if (req.method !== 'GET')
|
||||
return sendJson(res, { error: 'method_not_allowed' }, 405);
|
||||
if (!['gfs', 'ifs'].includes(model) || !Object.hasOwn(models, model))
|
||||
return sendJson(res, { error: 'unknown_model' }, 400);
|
||||
if (!['none', 'temperature', 'pressure'].includes(overlay))
|
||||
return sendJson(res, { error: 'unknown_overlay' }, 400);
|
||||
if (url.pathname.startsWith('/grid/')) {
|
||||
const id = url.pathname.slice(6).replace(/\.bin$/, '');
|
||||
const state =
|
||||
url.pathname === `/grid/${id}.bin` && grids.get(key)?.get(id);
|
||||
if (!state) return sendJson(res, { error: 'unknown_grid' }, 404);
|
||||
const components = [
|
||||
state.grid.u,
|
||||
state.grid.v,
|
||||
...(state.grid.scalar ? [state.grid.scalar] : []),
|
||||
];
|
||||
const bytes = Buffer.concat(
|
||||
components.map((values) =>
|
||||
Buffer.from(values.buffer, values.byteOffset, values.byteLength),
|
||||
),
|
||||
);
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'application/octet-stream',
|
||||
'Cache-Control': 'public, max-age=3600, immutable',
|
||||
});
|
||||
return res.end(bytes);
|
||||
}
|
||||
if (!['/', '/manifest', '/status'].includes(url.pathname))
|
||||
return sendJson(res, { error: 'not_found' }, 404);
|
||||
let state = caches.get(key);
|
||||
if (!state || now() - state.fetchedAt >= ttlMs) {
|
||||
let operation = loadings.get(key);
|
||||
if (!operation && now() - (attempts.get(key) ?? -Infinity) >= 60_000)
|
||||
operation = refresh(model, overlay, key);
|
||||
if (operation) {
|
||||
let disconnected = false;
|
||||
operation.waiters += 1;
|
||||
const close = () => {
|
||||
disconnected = true;
|
||||
if (--operation.waiters === 0) operation.controller.abort();
|
||||
};
|
||||
res.once?.('close', close);
|
||||
try {
|
||||
state = await operation.promise;
|
||||
} finally {
|
||||
res.removeListener?.('close', close);
|
||||
if (!disconnected) operation.waiters -= 1;
|
||||
}
|
||||
if (disconnected) return;
|
||||
}
|
||||
}
|
||||
const manifest = state?.manifest || unavailable(model, overlay).manifest;
|
||||
if (url.pathname === '/status') {
|
||||
const { gridUrl, ...status } = manifest;
|
||||
return sendJson(res, status);
|
||||
}
|
||||
return sendJson(res, manifest);
|
||||
};
|
||||
return {
|
||||
name: 'wind',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/wind', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/wind', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
export const GFS_BUCKET = 'noaa-gfs-bdp-pds';
|
||||
|
||||
/** Build the public GFS object key. */
|
||||
export function gfsObjectKey({ date, hour, forecastHour = 0 }) {
|
||||
const hh = String(hour).padStart(2, '0');
|
||||
return `gfs.${date}/${hh}/atmos/gfs.t${hh}z.pgrb2.0p25.f${String(forecastHour).padStart(3, '0')}`;
|
||||
}
|
||||
|
||||
/** Select the latest GFS cycle that should be available. */
|
||||
export function selectLatestGfsCycle(
|
||||
nowMs,
|
||||
{ availabilityLagMs = 5 * 3600_000 } = {},
|
||||
) {
|
||||
const d = new Date(nowMs - availabilityLagMs);
|
||||
const date = new Date(
|
||||
Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()),
|
||||
);
|
||||
const hour = Math.floor(d.getUTCHours() / 6) * 6;
|
||||
date.setUTCHours(hour);
|
||||
return { date: date.toISOString().slice(0, 10).replaceAll('-', ''), hour };
|
||||
}
|
||||
|
||||
/**
|
||||
* Round a time offset to the nearest published GFS forecast step so the field
|
||||
* shown is the one valid closest to now, not always the analysis (f000).
|
||||
* GFS 0.25° is hourly through f120, then every 3 h through f384.
|
||||
* @param {number} hours - Hours elapsed since the cycle run time.
|
||||
* @returns {number} A valid forecast step.
|
||||
*/
|
||||
export function nearestGfsStep(hours) {
|
||||
const value = Math.max(0, Number.isFinite(hours) ? hours : 0);
|
||||
const step = value <= 120 ? Math.round(value) : Math.round(value / 3) * 3;
|
||||
return Math.max(0, Math.min(384, step));
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/** Lazy shared ecCodes WASM instance, created only for an active wind request. */
|
||||
let eccodesPromise = null;
|
||||
function loadEccodes() {
|
||||
eccodesPromise ??= import('@meri-imperiumi/eccodes-wasm')
|
||||
.then((module) => module.createEccodes())
|
||||
.catch((error) => {
|
||||
eccodesPromise = null;
|
||||
throw error;
|
||||
});
|
||||
return eccodesPromise;
|
||||
}
|
||||
|
||||
/** Decode one bounded message in WASM memory without mounting the host filesystem. */
|
||||
export async function decodeWindGribMessage(buffer, { eccodesModule } = {}) {
|
||||
if (buffer.byteLength > 8 * 1024 * 1024)
|
||||
throw new Error('Wind GRIB exceeds byte budget');
|
||||
const eccodes = eccodesModule ?? (await loadEccodes());
|
||||
// Everything after this await is synchronous. A shared virtual file is safe
|
||||
// across U/V and model requests and its storage is released after each decode.
|
||||
const file = '/gev-wind.grib2';
|
||||
let handle = null;
|
||||
try {
|
||||
eccodes.writeFile(file, buffer);
|
||||
handle = eccodes.openGrib(file);
|
||||
const ni = handle.getLong('Ni');
|
||||
const nj = handle.getLong('Nj');
|
||||
if (
|
||||
!Number.isInteger(ni) ||
|
||||
!Number.isInteger(nj) ||
|
||||
ni < 1 ||
|
||||
nj < 1 ||
|
||||
ni * nj > 2_000_000
|
||||
)
|
||||
throw new Error('Invalid wind GRIB geometry');
|
||||
return {
|
||||
ni,
|
||||
nj,
|
||||
lo1: handle.getDouble('longitudeOfFirstGridPointInDegrees'),
|
||||
la1: handle.getDouble('latitudeOfFirstGridPointInDegrees'),
|
||||
di: handle.getDouble('iDirectionIncrementInDegrees'),
|
||||
dj: handle.getDouble('jDirectionIncrementInDegrees'),
|
||||
values: handle.getDoubleArray('values'),
|
||||
shortName: handle.getString('shortName'),
|
||||
level: handle.getLong('level'),
|
||||
units: handle.getString('units'),
|
||||
};
|
||||
} finally {
|
||||
handle?.delete();
|
||||
eccodes.writeFile(file, new Uint8Array(0));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import { readWindBody } from '../../../src/sources/windBody.js';
|
||||
/**
|
||||
* Parse a GFS `.idx` inventory into ordered message records.
|
||||
*
|
||||
* Each line is `msgNumber:byteOffset:d=YYYYMMDDHH:VAR:LEVEL:...:`. Malformed
|
||||
* lines are skipped; a payload with no valid lines is rejected so callers never
|
||||
* treat an HTML error page as an empty inventory.
|
||||
*
|
||||
* @param {string} text - Raw `.idx` body.
|
||||
* @returns {{ messages: Array<{index: number, offset: number, variable: string, level: string}> }}
|
||||
*/
|
||||
export function parseGfsIdx(text) {
|
||||
const messages = [];
|
||||
for (const line of String(text).split(/\r?\n/)) {
|
||||
if (!line) continue;
|
||||
const parts = line.split(':');
|
||||
if (
|
||||
parts.length < 6 ||
|
||||
!/^\d+$/.test(parts[0]) ||
|
||||
!/^\d+$/.test(parts[1]) ||
|
||||
!parts[3] ||
|
||||
!parts[4]
|
||||
)
|
||||
continue;
|
||||
messages.push({
|
||||
index: Number(parts[0]),
|
||||
offset: Number(parts[1]),
|
||||
variable: parts[3],
|
||||
level: parts[4],
|
||||
});
|
||||
}
|
||||
if (!messages.length) throw new Error('malformed GFS index');
|
||||
return { messages };
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte ranges for the two 10 m wind components. A message runs from its own
|
||||
* offset to one byte before the next message in file order.
|
||||
*
|
||||
* @param {{messages: Array<object>}} parsed - {@link parseGfsIdx} result.
|
||||
* @param {{level?: string}} [options]
|
||||
* @returns {{u: {start: number, end: number}, v: {start: number, end: number}}}
|
||||
*/
|
||||
export function windMessageRanges(
|
||||
parsed,
|
||||
{ level = '10 m above ground', overlay = 'none' } = {},
|
||||
) {
|
||||
weatherScalarMetadata(overlay);
|
||||
const range = (variable, fieldLevel = level) => {
|
||||
const index = parsed.messages.findIndex(
|
||||
(message) =>
|
||||
message.variable === variable && message.level === fieldLevel,
|
||||
);
|
||||
const message = parsed.messages[index];
|
||||
const next = parsed.messages[index + 1];
|
||||
if (!message || !next)
|
||||
throw new Error(`missing ${variable} at ${fieldLevel}`);
|
||||
return { start: message.offset, end: next.offset - 1 };
|
||||
};
|
||||
return {
|
||||
u: range('UGRD'),
|
||||
v: range('VGRD'),
|
||||
...(overlay === 'temperature'
|
||||
? { scalar: range('TMP', '2 m above ground') }
|
||||
: {}),
|
||||
...(overlay === 'pressure'
|
||||
? { scalar: range('PRMSL', 'mean sea level') }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Fetch a bounded text response as a Buffer. */
|
||||
export async function fetchText({ url, fetchImpl = fetch, signal }) {
|
||||
const response = await fetchImpl(url, { signal, redirect: 'error' });
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('Wind upstream unavailable');
|
||||
}
|
||||
return Buffer.from(await readWindBody(response, 2 * 1024 * 1024, signal));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch one bounded byte range. Rejects oversized ranges, non-206 responses
|
||||
* (except a whole-object 200 starting at zero), and short bodies.
|
||||
*/
|
||||
export async function fetchRange({
|
||||
url,
|
||||
start,
|
||||
end,
|
||||
fetchImpl = fetch,
|
||||
signal,
|
||||
maxBytes = 8 * 1024 * 1024,
|
||||
}) {
|
||||
if (
|
||||
!Number.isSafeInteger(start) ||
|
||||
!Number.isSafeInteger(end) ||
|
||||
start < 0 ||
|
||||
end < start ||
|
||||
end - start + 1 > maxBytes
|
||||
)
|
||||
throw new Error('range too large');
|
||||
const response = await fetchImpl(url, {
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: { Range: `bytes=${start}-${end}` },
|
||||
});
|
||||
const validStatus =
|
||||
response.status === 206 || (response.status === 200 && start === 0);
|
||||
if (!validStatus || !response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('Wind upstream unavailable');
|
||||
}
|
||||
const buffer = Buffer.from(
|
||||
await readWindBody(response, end - start + 1, signal),
|
||||
);
|
||||
if (buffer.length !== end - start + 1)
|
||||
throw new Error('invalid range length');
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/** A bounded optional request may fail without cancelling valid wind siblings. */
|
||||
export async function loadOptionalScalar({
|
||||
range,
|
||||
url,
|
||||
fetchImpl,
|
||||
decodeImpl,
|
||||
signal,
|
||||
timeoutMs = 12_000,
|
||||
}) {
|
||||
if (!range) return undefined;
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort(signal.reason);
|
||||
signal?.addEventListener('abort', abort, { once: true });
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
const bytes = await fetchRange({
|
||||
url,
|
||||
...range,
|
||||
fetchImpl,
|
||||
signal: controller.signal,
|
||||
});
|
||||
const value = await decodeImpl(bytes);
|
||||
controller.signal.throwIfAborted();
|
||||
return value;
|
||||
} catch {
|
||||
return undefined;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
controller.abort();
|
||||
signal?.removeEventListener('abort', abort);
|
||||
}
|
||||
}
|
||||
|
||||
import {
|
||||
GFS_BUCKET,
|
||||
gfsObjectKey,
|
||||
selectLatestGfsCycle,
|
||||
nearestGfsStep,
|
||||
} from './catalog.js';
|
||||
import { decodeWindGribMessage } from './decode.js';
|
||||
import { resampleWeatherSnapshot, weatherScalarMetadata } from './grid.js';
|
||||
|
||||
/** Fetch and decode the GFS 10 m wind field valid closest to now. */
|
||||
export async function fetchGfsWind({
|
||||
fetchImpl = fetch,
|
||||
now = () => Date.now(),
|
||||
targetDx = 1,
|
||||
decodeImpl = decodeWindGribMessage,
|
||||
signal,
|
||||
overlay = 'none',
|
||||
} = {}) {
|
||||
weatherScalarMetadata(overlay);
|
||||
const nowMs = now();
|
||||
const cycle = selectLatestGfsCycle(nowMs);
|
||||
const runMs = Date.UTC(
|
||||
Number(cycle.date.slice(0, 4)),
|
||||
Number(cycle.date.slice(4, 6)) - 1,
|
||||
Number(cycle.date.slice(6, 8)),
|
||||
cycle.hour,
|
||||
);
|
||||
// Pick the forecast step whose valid time is closest to now, so the layer
|
||||
// shows the freshest field the cycle offers instead of the analysis.
|
||||
const forecastHour = nearestGfsStep((nowMs - runMs) / 3600_000);
|
||||
const base = `https://${GFS_BUCKET}.s3.amazonaws.com/${gfsObjectKey({ ...cycle, forecastHour })}`;
|
||||
const index = await fetchText({ url: `${base}.idx`, fetchImpl, signal });
|
||||
const inventory = parseGfsIdx(index.toString());
|
||||
const ranges = windMessageRanges(inventory);
|
||||
let scalarRange;
|
||||
if (overlay !== 'none') {
|
||||
try {
|
||||
scalarRange = windMessageRanges(inventory, { overlay }).scalar;
|
||||
} catch {
|
||||
/* A missing optional field does not discard valid wind. */
|
||||
}
|
||||
}
|
||||
// All selected messages come from this one issue/valid-time object.
|
||||
const load = async (range) =>
|
||||
decodeImpl(await fetchRange({ url: base, ...range, fetchImpl, signal }));
|
||||
const [u, v, scalar] = await Promise.all([
|
||||
load(ranges.u),
|
||||
load(ranges.v),
|
||||
loadOptionalScalar({
|
||||
range: scalarRange,
|
||||
url: base,
|
||||
fetchImpl,
|
||||
decodeImpl,
|
||||
signal,
|
||||
}),
|
||||
]);
|
||||
signal?.throwIfAborted();
|
||||
const fields = resampleWeatherSnapshot({ u, v, scalar, overlay, targetDx });
|
||||
const runIso = new Date(runMs).toISOString();
|
||||
const validIso = new Date(runMs + forecastHour * 3600_000).toISOString();
|
||||
return {
|
||||
cycle: { ...cycle, forecastHour, runIso, validIso },
|
||||
level: '10 m above ground',
|
||||
units: 'm/s',
|
||||
...fields,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Resample a GRIB wind grid into a compact equirectangular grid.
|
||||
*
|
||||
* Source grids are north-to-south (`la1` at the top, `dj` increasing toward the
|
||||
* south) and start at `lo1`. Longitude wraps across the seam; latitude clamps.
|
||||
*
|
||||
* @param {{u: ArrayLike<number>, v: ArrayLike<number>, ni: number, nj: number,
|
||||
* lo1: number, la1: number, di: number, dj: number, dx?: number, dy?: number}} input
|
||||
* @returns {{nx: number, ny: number, lo1: number, la1: number, dx: number,
|
||||
* dy: number, u: Float32Array, v: Float32Array}}
|
||||
*/
|
||||
export function resampleWindGrid({
|
||||
u,
|
||||
v,
|
||||
scalar,
|
||||
scalarScale = 1,
|
||||
scalarOffset = 0,
|
||||
ni,
|
||||
nj,
|
||||
lo1,
|
||||
la1,
|
||||
di,
|
||||
dj,
|
||||
dx = 1,
|
||||
dy = 1,
|
||||
}) {
|
||||
if (
|
||||
!Number.isInteger(ni) ||
|
||||
!Number.isInteger(nj) ||
|
||||
ni < 1 ||
|
||||
nj < 1 ||
|
||||
ni * nj > 2_000_000 ||
|
||||
u?.length !== ni * nj ||
|
||||
v?.length !== ni * nj ||
|
||||
(scalar !== undefined && scalar?.length !== ni * nj) ||
|
||||
![lo1, la1, di, dj, dx, dy, scalarScale, scalarOffset].every(
|
||||
Number.isFinite,
|
||||
) ||
|
||||
di <= 0 ||
|
||||
dj <= 0 ||
|
||||
dx < 0.25 ||
|
||||
dy < 0.25 ||
|
||||
dx > 180 ||
|
||||
dy > 180
|
||||
)
|
||||
throw new Error('Invalid wind grid geometry');
|
||||
for (const values of [u, v, ...(scalar === undefined ? [] : [scalar])]) {
|
||||
if (!Array.prototype.every.call(values, Number.isFinite))
|
||||
throw new Error('Invalid wind grid values');
|
||||
}
|
||||
const nx = Math.round(360 / dx);
|
||||
const ny = Math.round(180 / dy) + 1;
|
||||
if (nx * ny > 1_000_000)
|
||||
throw new Error('Resampled wind grid exceeds point budget');
|
||||
const outU = new Float32Array(nx * ny);
|
||||
const outV = new Float32Array(nx * ny);
|
||||
const outScalar = scalar === undefined ? null : new Float32Array(nx * ny);
|
||||
const sample = (source, lon, lat) => {
|
||||
const x = ((lon - lo1) / di + ni) % ni;
|
||||
const y = Math.max(0, Math.min(nj - 1, (la1 - lat) / dj));
|
||||
const x0 = Math.floor(x);
|
||||
const x1 = (x0 + 1) % ni;
|
||||
const y0 = Math.floor(y);
|
||||
const y1 = Math.min(nj - 1, y0 + 1);
|
||||
const fx = x - x0;
|
||||
const fy = y - y0;
|
||||
const top = source[y0 * ni + x0] * (1 - fx) + source[y0 * ni + x1] * fx;
|
||||
const bottom = source[y1 * ni + x0] * (1 - fx) + source[y1 * ni + x1] * fx;
|
||||
return top * (1 - fy) + bottom * fy;
|
||||
};
|
||||
for (let y = 0; y < ny; y += 1) {
|
||||
for (let x = 0; x < nx; x += 1) {
|
||||
const index = y * nx + x;
|
||||
const lon = x * dx;
|
||||
const lat = 90 - y * dy;
|
||||
outU[index] = sample(u, lon, lat);
|
||||
outV[index] = sample(v, lon, lat);
|
||||
if (outScalar)
|
||||
outScalar[index] =
|
||||
sample(scalar, lon, lat) * scalarScale + scalarOffset;
|
||||
}
|
||||
}
|
||||
for (const values of [outU, outV, ...(outScalar ? [outScalar] : [])]) {
|
||||
if (!values.every(Number.isFinite))
|
||||
throw new Error('Invalid resampled wind values');
|
||||
}
|
||||
return {
|
||||
nx,
|
||||
ny,
|
||||
lo1: 0,
|
||||
la1: 90,
|
||||
dx,
|
||||
dy,
|
||||
u: outU,
|
||||
v: outV,
|
||||
...(outScalar ? { scalar: outScalar } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Metadata for the two optional same-cycle weather fields. */
|
||||
export function weatherScalarMetadata(overlay = 'none') {
|
||||
if (overlay === 'none') return null;
|
||||
if (overlay === 'temperature')
|
||||
return { kind: overlay, units: '°C', level: '2 m above ground' };
|
||||
if (overlay === 'pressure')
|
||||
return { kind: overlay, units: 'hPa', level: 'mean sea level' };
|
||||
throw new Error('Unknown weather overlay');
|
||||
}
|
||||
|
||||
/** Reject shifted/mismatched components before sharing U's grid geometry. */
|
||||
export function resampleWeatherFields({
|
||||
u,
|
||||
v,
|
||||
scalar,
|
||||
overlay = 'none',
|
||||
targetDx = 1,
|
||||
}) {
|
||||
const metadata = weatherScalarMetadata(overlay);
|
||||
const fields = [u, v, ...(metadata ? [scalar] : [])];
|
||||
if (
|
||||
!u ||
|
||||
fields.some(
|
||||
(field) =>
|
||||
!field ||
|
||||
['ni', 'nj', 'lo1', 'la1', 'di', 'dj'].some(
|
||||
(key) => field[key] !== u[key],
|
||||
),
|
||||
)
|
||||
)
|
||||
throw new Error('Mismatched weather grid geometry');
|
||||
if (metadata && scalar.units !== (overlay === 'temperature' ? 'K' : 'Pa'))
|
||||
throw new Error('Unexpected weather scalar units');
|
||||
const grid = resampleWindGrid({
|
||||
u: u.values,
|
||||
v: v.values,
|
||||
ni: u.ni,
|
||||
nj: u.nj,
|
||||
lo1: u.lo1,
|
||||
la1: u.la1,
|
||||
di: u.di,
|
||||
dj: u.dj,
|
||||
dx: targetDx,
|
||||
dy: targetDx,
|
||||
...(metadata
|
||||
? {
|
||||
scalar: scalar.values,
|
||||
scalarScale: overlay === 'pressure' ? 0.01 : 1,
|
||||
scalarOffset: overlay === 'temperature' ? -273.15 : 0,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
return { grid, ...(metadata ? { scalar: metadata } : {}) };
|
||||
}
|
||||
|
||||
/** Preserve usable same-cycle wind when only its optional companion fails. */
|
||||
export function resampleWeatherSnapshot(options) {
|
||||
try {
|
||||
return resampleWeatherFields(options);
|
||||
} catch (error) {
|
||||
if ((options.overlay ?? 'none') === 'none') throw error;
|
||||
// Revalidate wind alone: a wind defect must never become a partial success.
|
||||
const wind = resampleWeatherFields({ ...options, overlay: 'none' });
|
||||
return { ...wind, scalarError: weatherScalarError(options.overlay) };
|
||||
}
|
||||
}
|
||||
|
||||
export function weatherScalarError(overlay) {
|
||||
weatherScalarMetadata(overlay);
|
||||
return overlay === 'temperature'
|
||||
? 'Temperature field unavailable'
|
||||
: 'Mean sea level pressure field unavailable';
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import { fetchText, fetchRange, loadOptionalScalar } from './gfs.js';
|
||||
import { decodeWindGribMessage } from './decode.js';
|
||||
import { resampleWeatherSnapshot, weatherScalarMetadata } from './grid.js';
|
||||
|
||||
/** Keyless ECMWF Open Data root for real-time IFS forecasts. */
|
||||
export const IFS_BASE = 'https://data.ecmwf.int/forecasts';
|
||||
|
||||
/**
|
||||
* Select the latest IFS cycle that should be available.
|
||||
* @param {number} nowMs - Reference epoch milliseconds.
|
||||
* @param {{availabilityLagMs?: number}} [options]
|
||||
* @returns {{date: string, hour: number}}
|
||||
*/
|
||||
export function selectLatestIfsCycle(
|
||||
nowMs,
|
||||
{ availabilityLagMs = 6 * 3600_000 } = {},
|
||||
) {
|
||||
const shifted = new Date(nowMs - availabilityLagMs);
|
||||
return {
|
||||
date: shifted.toISOString().slice(0, 10).replaceAll('-', ''),
|
||||
hour: Math.floor(shifted.getUTCHours() / 6) * 6,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the IFS GRIB2 and `.index` URLs for one cycle and step.
|
||||
* @param {{date: string, hour: number, step?: number}} cycle
|
||||
* @returns {{grib: string, index: string}}
|
||||
*/
|
||||
export function ifsObjectUrls({ date, hour, step = 0 }) {
|
||||
const hh = String(hour).padStart(2, '0');
|
||||
const stem = `${IFS_BASE}/${date}/${hh}z/ifs/0p25/oper/${date}${hh}0000-${step}h-oper-fc`;
|
||||
// The inventory sits beside the GRIB2 as `<stem>.index`, NOT `<grib2>.index`.
|
||||
return { grib: `${stem}.grib2`, index: `${stem}.index` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the IFS JSON Lines inventory into `{param, offset, length}` records.
|
||||
* Malformed lines are skipped; a payload with no valid lines is rejected.
|
||||
* @param {string} text - Raw `.index` body.
|
||||
* @returns {Array<{param: string, offset: number, length: number}>}
|
||||
*/
|
||||
export function parseIfsIndex(text) {
|
||||
const entries = [];
|
||||
for (const line of String(text).split(/\r?\n/)) {
|
||||
if (!line) continue;
|
||||
let value;
|
||||
try {
|
||||
value = JSON.parse(line);
|
||||
} catch {
|
||||
continue; // non-JSON line in the index
|
||||
}
|
||||
if (
|
||||
value?.param &&
|
||||
Number.isFinite(value._offset) &&
|
||||
Number.isFinite(value._length)
|
||||
)
|
||||
entries.push({
|
||||
param: value.param,
|
||||
offset: value._offset,
|
||||
length: value._length,
|
||||
});
|
||||
}
|
||||
if (!entries.length) throw new Error('malformed IFS index');
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte ranges for the IFS 10 m wind components.
|
||||
* @param {Array<{param: string, offset: number, length: number}>} entries
|
||||
* @returns {{u: {start: number, end: number}, v: {start: number, end: number}}}
|
||||
*/
|
||||
export function ifsWindRanges(entries, { overlay = 'none' } = {}) {
|
||||
weatherScalarMetadata(overlay);
|
||||
const range = (param) => {
|
||||
const item = entries.find((entry) => entry.param === param);
|
||||
if (!item) throw new Error(`missing ${param}`);
|
||||
return { start: item.offset, end: item.offset + item.length - 1 };
|
||||
};
|
||||
return {
|
||||
u: range('10u'),
|
||||
v: range('10v'),
|
||||
...(overlay === 'temperature' ? { scalar: range('2t') } : {}),
|
||||
...(overlay === 'pressure' ? { scalar: range('msl') } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Round a time offset to the nearest published IFS forecast step. IFS 0.25°
|
||||
* oper is 3-hourly through f144, then 6-hourly to f360 for 00/12z.
|
||||
* @param {number} hours - Hours elapsed since the cycle run time.
|
||||
* @param {number} cycleHour - The cycle hour (0, 6, 12 or 18).
|
||||
* @returns {number} A valid forecast step.
|
||||
*/
|
||||
export function nearestIfsStep(hours, cycleHour) {
|
||||
const max = cycleHour === 6 || cycleHour === 18 ? 144 : 360;
|
||||
const value = Math.max(0, Number.isFinite(hours) ? hours : 0);
|
||||
const step =
|
||||
value <= 144 ? Math.round(value / 3) * 3 : Math.round(value / 6) * 6;
|
||||
return Math.max(0, Math.min(max, step));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and decode the IFS 10 m wind field valid closest to now.
|
||||
* @param {{fetchImpl?: Function, now?: Function, targetDx?: number,
|
||||
* decodeImpl?: Function}} [options]
|
||||
* @returns {Promise<{cycle: object, level: string, units: string, grid: object}>}
|
||||
*/
|
||||
export async function fetchIfsWind({
|
||||
fetchImpl = fetch,
|
||||
now = () => Date.now(),
|
||||
targetDx = 1,
|
||||
decodeImpl = decodeWindGribMessage,
|
||||
signal,
|
||||
overlay = 'none',
|
||||
} = {}) {
|
||||
weatherScalarMetadata(overlay);
|
||||
const nowMs = now();
|
||||
const cycle = selectLatestIfsCycle(nowMs);
|
||||
const runMs = Date.UTC(
|
||||
Number(cycle.date.slice(0, 4)),
|
||||
Number(cycle.date.slice(4, 6)) - 1,
|
||||
Number(cycle.date.slice(6, 8)),
|
||||
cycle.hour,
|
||||
);
|
||||
// Show the field valid closest to now rather than always the analysis.
|
||||
const forecastHour = nearestIfsStep((nowMs - runMs) / 3600_000, cycle.hour);
|
||||
const urls = ifsObjectUrls({ ...cycle, step: forecastHour });
|
||||
const index = await fetchText({ url: urls.index, fetchImpl, signal });
|
||||
const inventory = parseIfsIndex(index.toString());
|
||||
const ranges = ifsWindRanges(inventory);
|
||||
let scalarRange;
|
||||
if (overlay !== 'none') {
|
||||
try {
|
||||
scalarRange = ifsWindRanges(inventory, { overlay }).scalar;
|
||||
} catch {
|
||||
/* A missing optional field does not discard valid wind. */
|
||||
}
|
||||
}
|
||||
// All selected messages come from this one issue/valid-time object.
|
||||
const load = async (range) =>
|
||||
decodeImpl(
|
||||
await fetchRange({ url: urls.grib, ...range, fetchImpl, signal }),
|
||||
);
|
||||
const [u, v, scalar] = await Promise.all([
|
||||
load(ranges.u),
|
||||
load(ranges.v),
|
||||
loadOptionalScalar({
|
||||
range: scalarRange,
|
||||
url: urls.grib,
|
||||
fetchImpl,
|
||||
decodeImpl,
|
||||
signal,
|
||||
}),
|
||||
]);
|
||||
signal?.throwIfAborted();
|
||||
const fields = resampleWeatherSnapshot({ u, v, scalar, overlay, targetDx });
|
||||
const runIso = new Date(runMs).toISOString();
|
||||
const validIso = new Date(runMs + forecastHour * 3600_000).toISOString();
|
||||
return {
|
||||
cycle: { ...cycle, forecastHour, runIso, validIso },
|
||||
level: '10 m above ground',
|
||||
units: 'm/s',
|
||||
...fields,
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user