Merge current main into S2 GhostMaps and preserve updated import and voice behavior

This commit is contained in:
Mazeyar Moeini Feizabadi committed 2026-10-26 16:25:49 +01:00
commit 4da002bd42
515 files changed
+17681 -2202

No files matched your search

+64 -47
View File
@@ -1,4 +1,7 @@
import { readResponseTextCapped } from '../common/http.js';
import {
coalesceProxyRequest,
readResponseTextCapped,
} from '../common/http.js';
/**
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
@@ -19,6 +22,7 @@ import { readResponseTextCapped } from '../common/http.js';
export function adsbLolProxy() {
/** @type {string|null} Cached upstream JSON body. */
let _cache = null;
const inFlight = new Map();
/** @type {number} Epoch-ms when the cache was populated. */
let _cacheAt = 0;
/** @type {number} Epoch-ms until which upstream is not contacted. */
@@ -104,60 +108,73 @@ export function adsbLolProxy() {
);
return;
}
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const refresh = coalesceProxyRequest(inFlight, 'global', async () => {
const result = (status, body, cacheStatus, extra = {}) => ({
status,
body,
cacheStatus,
extra,
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
return result(200, body, 'MISS');
}
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
return result(200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
});
}
}
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
serve(res, 200, body, 'MISS');
return;
}
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
return result(
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
serve(res, 200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
});
return;
}
}
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
});
const response = await refresh.promise;
serve(
res,
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
response.status,
response.body,
response.cacheStatus,
response.extra,
);
} catch (e) {
console.error('[adsb.lol Proxy]', e.message);
+175 -132
View File
@@ -1,3 +1,5 @@
import fs from 'node:fs';
import os from 'node:os';
import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback.js';
import {
coalesceProxyRequest,
@@ -67,15 +69,24 @@ async function fetchOpenSkyStates(headers) {
try {
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
if (upstream.status === 429) {
if (upstream.status === 429 || upstream.status >= 500) {
// Start the cooldown from the headers and drop the rejected body
// unread, so a stalled or oversized 429 body cannot skip it.
_openskyCooldownUntil =
Date.now() + openSkyRateLimitCooldownMs(upstream);
Date.now() +
(upstream.status === 429
? openSkyRateLimitCooldownMs(upstream)
: 15000);
_openskyCooldownStatus = upstream.status;
void upstream.body?.cancel().catch(() => {});
return {
upstream,
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
body: JSON.stringify({
error:
upstream.status === 429
? 'OpenSky rate limited.'
: 'OpenSky upstream unavailable.',
}),
};
}
const body = await readResponseTextCapped(
@@ -109,6 +120,8 @@ async function fetchOpenSkyStates(headers) {
let _openskyTtlMs = OPENSKY_CACHE_MS;
/** @type {number} Epoch-ms before which no upstream fetch is attempted. */
let _openskyCooldownUntil = 0;
let _openskyCooldownStatus = 429;
const _openskySnapshotInFlight = new Map();
/**
* Picks the cache TTL from the remaining daily credit budget.
* Client polls every 30 s, so tiers ≤30 s cost the same 480 credits/h; the
@@ -126,6 +139,10 @@ function openskyAdaptiveTtlMs(remaining) {
}
/** @type {boolean} Guards duplicate auth-failure warnings in logs. */
let _openskyAuthWarned = false;
/** One warning per process when OPENSKY_CREDENTIALS_FILE cannot be read. */
let _openskyCredentialsFileWarned = false;
/** One warning per process when OPENSKY_AUTH_MODE=basic is requested. */
let _openskyBasicRetiredWarned = false;
/** @type {boolean} Guards duplicate invalid-auth-mode warnings. */
let _openskyAuthModeWarned = false;
/** Default auth mode when OPENSKY_AUTH_MODE env is unset. */
@@ -145,6 +162,48 @@ const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
// current instead of coasting a stale worldwide frame indefinitely.
const OPENSKY_SOURCE_STALE_MS = 120_000;
/**
* The OpenSky API client for OAuth: OPENSKY_CLIENT_ID and
* OPENSKY_CLIENT_SECRET, or else the credentials.json that OpenSky's account
* page downloads, named by OPENSKY_CREDENTIALS_FILE ("~" is the home
* directory). The file is read on each token refresh, so a replaced file is
* picked up without a restart.
*
* @param {Record<string, string|undefined>} [env]
* @param {(path: string) => string} [readFile]
* @returns {{ clientId: string, clientSecret: string } | null}
*/
export function openSkyClientCredentials(
env = process.env,
readFile = (path) => fs.readFileSync(path, 'utf8'),
) {
const clientId = String(env.OPENSKY_CLIENT_ID || '').trim();
const clientSecret = String(env.OPENSKY_CLIENT_SECRET || '').trim();
if (clientId && clientSecret) return { clientId, clientSecret };
const file = String(env.OPENSKY_CREDENTIALS_FILE || '').trim();
if (!file) return null;
const path = file.replace(/^~(?=$|[\\/])/, os.homedir());
let raw;
try {
raw = JSON.parse(readFile(path));
} catch {
if (!_openskyCredentialsFileWarned) {
console.warn(
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
);
_openskyCredentialsFileWarned = true;
}
return null;
}
const fileId = String(raw?.clientId ?? raw?.client_id ?? '').trim();
const fileSecret = String(
raw?.clientSecret ?? raw?.client_secret ?? '',
).trim();
return fileId && fileSecret
? { clientId: fileId, clientSecret: fileSecret }
: null;
}
/**
* Obtain a valid OpenSky OAuth2 bearer token, refreshing if needed.
*
@@ -163,9 +222,9 @@ export async function getOpenSkyToken() {
// return the same promise instead of issuing a duplicate token request
if (_openskyTokenPromise) return _openskyTokenPromise;
const clientId = process.env.OPENSKY_CLIENT_ID;
const clientSecret = process.env.OPENSKY_CLIENT_SECRET;
if (!clientId || !clientSecret) return null;
const credentials = openSkyClientCredentials();
if (!credentials) return null;
const { clientId, clientSecret } = credentials;
// Wrap the async token fetch in a shared promise stored in _openskyTokenPromise
_openskyTokenPromise = (async () => {
@@ -406,8 +465,10 @@ function openSkySourceIsStale(sourceEpochMs, now = Date.now()) {
*
* Supports four auth modes controlled by OPENSKY_AUTH_MODE env:
* - 'oauth' (default) — client_credentials bearer token
* - 'basic' — HTTP Basic with OPENSKY_USERNAME / OPENSKY_PASSWORD
* - 'auto' — try OAuth first, fall back to Basic, then anon
* - 'basic' — retired: OpenSky stopped accepting HTTP Basic (username and
* password) on 2026-03-18, so this requests anonymously and
* warns once
* - 'auto' — try OAuth first, then anon
* - 'anon' — no credentials
*
* Successful responses are cached for OPENSKY_CACHE_MS (~9 s). On
@@ -460,7 +521,9 @@ export function openSkyProxy() {
requestedMode: cachedMeta.requestedMode || requestedMode,
usedMode: cachedMeta.usedMode || 'unknown',
reason: isStale
? 'rate_limited_serving_stale'
? _openskyCooldownStatus === 429
? 'rate_limited_serving_stale'
: `upstream_http_${_openskyCooldownStatus}_serving_stale`
: cachedMeta.reason || 'cached',
staleSeconds: isStale
? (now - _openskyCacheTime) / 1000
@@ -487,80 +550,114 @@ export function openSkyProxy() {
)
return;
res.writeHead(
429,
_openskyCooldownStatus,
buildOpenSkyHeaders({
cacheStatus: 'COOLDOWN',
requestedMode,
usedMode: 'none',
reason: 'rate_limited',
reason:
_openskyCooldownStatus === 429
? 'rate_limited'
: `upstream_http_${_openskyCooldownStatus}`,
retryAfterSeconds: (_openskyCooldownUntil - now) / 1000,
}),
);
res.end(
JSON.stringify({
error: 'OpenSky rate limited; proxy cooling down.',
error: 'OpenSky upstream unavailable; proxy cooling down.',
}),
);
return;
}
const basicUser = process.env.OPENSKY_USERNAME || '';
const basicPass = process.env.OPENSKY_PASSWORD || '';
const hasBasicCreds = Boolean(basicUser && basicPass);
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
const refresh = coalesceProxyRequest(
_openskySnapshotInFlight,
'global',
async () => {
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
if (requestedMode === 'basic') {
if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'basic_credentials';
} else {
reason = 'missing_basic_creds';
}
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'oauth_unavailable_fallback_basic';
} else {
reason = 'missing_oauth_and_basic_creds';
}
}
if (requestedMode === 'basic') {
// OpenSky no longer accepts a Basic header, so sending
// OPENSKY_USERNAME / OPENSKY_PASSWORD would only fail; ask
// anonymously and say once what to configure instead.
if (!_openskyBasicRetiredWarned) {
console.warn(
'[OpenSky] OPENSKY_AUTH_MODE=basic: OpenSky retired username/password sign-in on 2026-03-18, so flights load anonymously (rate-limited). Use OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
);
_openskyBasicRetiredWarned = true;
}
reason = 'basic_retired';
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'missing_oauth_creds';
}
}
let { upstream, body } = await fetchOpenSkyStates(headers);
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
if (
(upstream.status === 401 || upstream.status === 403) &&
requestedMode === 'auto' &&
usedMode === 'oauth' &&
hasBasicCreds
) {
const retryHeaders = {
Accept: 'application/json',
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
};
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
usedMode = 'basic';
reason = 'oauth_rejected_fallback_basic';
}
const { upstream, body } = await fetchOpenSkyStates(headers);
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
const sourceEpochMs = upstream.ok
? openSkySourceEpochMs(body)
: null;
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
return {
upstream,
body,
usedMode,
reason,
sourceEpochMs,
};
},
);
let { upstream, body, usedMode, reason, sourceEpochMs } =
await refresh.promise;
if (
upstream.ok &&
openSkySourceIsStale(sourceEpochMs, now) &&
@@ -571,20 +668,16 @@ export function openSkyProxy() {
'opensky_snapshot_stale_regional_fallback',
))
) {
// Keep the last global snapshot available as a fail-soft cache,
// but do not label or render it as a fresh live result.
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = { requestedMode, usedMode, reason };
return;
}
if (upstream.status === 429) {
reason = 'rate_limited';
if (upstream.status === 429 || upstream.status >= 500) {
reason =
upstream.status === 429
? 'rate_limited'
: `upstream_http_${upstream.status}`;
// Credit governor: fetchOpenSkyStates already started the cooldown
// from OpenSky's retry-after — no upstream attempts until then.
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
const cooldownMs = Math.max(0, _openskyCooldownUntil - Date.now());
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
@@ -594,7 +687,7 @@ export function openSkyProxy() {
cacheStatus: 'STALE',
requestedMode,
usedMode,
reason: 'rate_limited_serving_stale',
reason: `${reason}_serving_stale`,
staleSeconds: (now - _openskyCacheTime) / 1000,
retryAfterSeconds: cooldownMs / 1000,
}),
@@ -615,35 +708,24 @@ export function openSkyProxy() {
}
if (upstream.status === 401 || upstream.status === 403) {
if (requestedMode === 'basic' && !hasBasicCreds) {
if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth missing. Basic mode requires OPENSKY_USERNAME and OPENSKY_PASSWORD.',
});
reason = 'missing_basic_creds';
} else if (requestedMode === 'oauth' && usedMode !== 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET.',
'OpenSky auth invalid. OAuth mode requires valid OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'oauth_invalid_or_missing';
} else if (usedMode === 'basic') {
body = JSON.stringify({
error: 'OpenSky auth invalid. Username/password were rejected.',
});
reason = 'basic_invalid_credentials';
} else if (usedMode === 'oauth') {
body = JSON.stringify({
error:
'OpenSky auth invalid. OAuth client credentials were rejected.',
});
reason = 'oauth_invalid_credentials';
} else if (requestedMode === 'auto' && !hasBasicCreds) {
} else if (requestedMode === 'auto') {
body = JSON.stringify({
error:
'OpenSky auth missing. Provide basic credentials or valid OAuth client credentials.',
'OpenSky auth missing. Provide OAuth client credentials: OPENSKY_CLIENT_ID and OPENSKY_CLIENT_SECRET, or OPENSKY_CREDENTIALS_FILE.',
});
reason = 'missing_oauth_and_basic_creds';
reason = 'missing_oauth_creds';
} else {
body = JSON.stringify({
error: 'OpenSky auth required.',
@@ -652,45 +734,6 @@ export function openSkyProxy() {
}
}
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'basic' &&
reason === 'basic_credentials'
) {
reason = 'basic_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
res.writeHead(
upstream.status,
buildOpenSkyHeaders({
@@ -0,0 +1,81 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import os from 'node:os';
import path from 'node:path';
import { openSkyClientCredentials } from './opensky.js';
const file = (contents) => (filePath) => {
file.read = filePath;
if (contents instanceof Error) throw contents;
return contents;
};
test('OpenSky client env vars come first', () => {
const read = () => assert.fail('the file is not read when env vars are set');
assert.deepEqual(
openSkyClientCredentials(
{
OPENSKY_CLIENT_ID: ' id ',
OPENSKY_CLIENT_SECRET: 'secret',
OPENSKY_CREDENTIALS_FILE: 'credentials.json',
},
read,
),
{ clientId: 'id', clientSecret: 'secret' },
);
});
test('OPENSKY_CREDENTIALS_FILE supplies the client when env vars are empty', () => {
const env = {
OPENSKY_CLIENT_ID: '',
OPENSKY_CREDENTIALS_FILE: '~/Downloads/credentials.json',
};
// The file OpenSky's account page downloads uses camelCase keys.
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"clientId":"cid","clientSecret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
assert.equal(
path.normalize(file.read),
path.normalize(path.join(os.homedir(), 'Downloads/credentials.json')),
);
assert.deepEqual(
openSkyClientCredentials(
env,
file('{"client_id":"cid","client_secret":"csecret"}'),
),
{ clientId: 'cid', clientSecret: 'csecret' },
);
});
test('a missing, unreadable or incomplete credentials file means no client', () => {
assert.equal(openSkyClientCredentials({}, file('{}')), null);
const env = { OPENSKY_CREDENTIALS_FILE: 'credentials.json' };
const missing = Object.assign(new Error('nope'), { code: 'ENOENT' });
assert.equal(openSkyClientCredentials(env, file(missing)), null);
assert.equal(openSkyClientCredentials(env, file('not json')), null);
assert.equal(openSkyClientCredentials(env, file('{"clientId":"cid"}')), null);
});
test('credential file warnings never expose file contents or read errors', async (t) => {
const warnings = [];
t.mock.method(console, 'warn', (message) => warnings.push(message));
for (const contents of [
'{"clientId":"id","clientSecret":FAKE_SECRET_123}',
new Error('could not read FAKE_SECRET_123'),
]) {
const { openSkyClientCredentials: resolve } = await import(
`./opensky.js?warning=${warnings.length}`
);
const env = { OPENSKY_CREDENTIALS_FILE: '/synthetic/credentials.json' };
assert.equal(resolve(env, file(contents)), null);
assert.equal(resolve(env, file(contents)), null);
}
assert.deepEqual(warnings, [
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
'[OpenSky] OPENSKY_CREDENTIALS_FILE could not be read or parsed as JSON.',
]);
});
+13 -8
View File
@@ -7,10 +7,9 @@ import { readCappedResponseText } from '../common/http.js';
* own credit bucket, 4 credits per call on the free tier). OAuth via the
* shared coalesced token. 60s per-icao cache; 404/429 forwarded so the
* client can fall back to its accumulated trail silently.
* /api/military/track?hex=<hex> — adsb.lol tar1090 readsb trace
* (undocumented but live; no browser CORS, hence this proxy). Up to ~24h
* of real history per aircraft. Treat as best-effort; data is ODbL —
* credit "adsb.lol (ODbL)" in the UI.
* /api/military/track?hex=<hex> — compatibility refusal for the retired,
* undocumented ADSB.lol trace route. The standalone military source uses
* locally accumulated trails; provider-supplied history remains optional.
*/
export function trackBackfillProxies() {
const TRACK_CACHE_MS = 60000;
@@ -107,10 +106,16 @@ export function trackBackfillProxies() {
);
return;
}
await proxyJson(
res,
`lol:${hex}`,
`https://adsb.lol/data/traces/${hex.slice(-2)}/trace_full_${hex}.json`,
res.statusCode = 503;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Cache-Control', 'no-store');
res.setHeader('Retry-After', '1800');
res.end(
JSON.stringify({
error:
'Military history unavailable; locally accumulated trails remain available',
code: 'history_unavailable',
}),
);
} catch (error) {
res.statusCode = 502;
+1 -5
View File
@@ -1,9 +1,5 @@
import { createCctvCatalog } from './cctv/catalog.js';
import {
normalizeFeedType,
isVideoFeedType,
toFiniteNumber,
} from './cctv/normalize.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
cctvLiveMediaType,
+1 -1
View File
@@ -18,7 +18,7 @@ export const CCTV_MAX_SOURCES_CEILING = 5000;
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
const TXDOT_ORIGIN = 'https://its.txdot.gov';
export const TXDOT_CCTV_STATUS_URL = (district) =>
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
+1 -1
View File
@@ -3,7 +3,7 @@ import path from 'node:path';
import { poseHash } from '../../../src/data/cctvFootprint.js';
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
export const DEFAULT_GROUND_HEIGHTS_FILE =
const DEFAULT_GROUND_HEIGHTS_FILE =
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
+2 -2
View File
@@ -92,7 +92,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
* @param {ReadableStream|NodeJS.ReadableStream|null} body
* @returns {import('stream').Readable|null}
*/
export function toReadable(body) {
function toReadable(body) {
if (!body) return null;
if (typeof body.pipe === 'function') return body;
if (typeof body.getReader === 'function') {
@@ -108,7 +108,7 @@ export function toReadable(body) {
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
* value is missing or is not a `type/subtype` media type.
*/
export function parseMediaType(value) {
function parseMediaType(value) {
const [head, ...rest] = String(value ?? '').split(';');
const essence = head.trim().toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
+3 -3
View File
@@ -64,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
* @param {string} text
* @returns {string}
*/
export function normalizeKey(text) {
function normalizeKey(text) {
return String(text || '')
.trim()
.toLowerCase()
@@ -80,7 +80,7 @@ export function normalizeKey(text) {
* @param {string} value
* @returns {{lat:number, lon:number}}
*/
export function parsePointString(value) {
function parsePointString(value) {
const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
@@ -100,7 +100,7 @@ export function parsePointString(value) {
* @param {string|object|null} value
* @returns {{lat:number, lon:number}}
*/
export function coerceLatLon(value) {
function coerceLatLon(value) {
if (!value) return { lat: NaN, lon: NaN };
if (typeof value === 'string') {
+1 -2
View File
@@ -133,7 +133,6 @@ import {
fallbackHeadingFromId,
isLikelyFinlandCoordinate,
fintrafficCameraName,
hashSeed,
isPlausibleLatLon,
isLikelyBcCoordinate,
isLikelyTexasCoordinate,
@@ -1869,7 +1868,7 @@ export function normalizeCataloniaImageUrl(rawUrl) {
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
* @returns {string}
*/
export function cataloniaCameraId(imageUrl) {
function cataloniaCameraId(imageUrl) {
const parsed = new URL(imageUrl);
const slug = (text) =>
text
+1 -1
View File
@@ -70,7 +70,7 @@ export async function fetchHlsBytes(
}
}
export function sameOriginHlsUrl(value, base) {
function sameOriginHlsUrl(value, base) {
const url = new URL(value, base);
if (
!['https:', 'http:'].includes(url.protocol) ||
+3 -3
View File
@@ -12,14 +12,14 @@ export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
export const DEFAULT_TAIWAN_MAX_SOURCES = 600;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
/** After a failed download, wait this long before asking again. The catalog
* host does not answer from some networks outside Taiwan, and every attempt
* there holds the shared catalog refresh until the connect timeout. */
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
/** The table is republished daily; a good copy stands in for a failed
* download for up to this long. */
export const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// The table publishes each camera's MJPEG stream. The same camera servers
// answer a single JPEG for the same camera ID on a sibling path, which is what
@@ -50,7 +50,7 @@ const ANCHORS = [
/** Taiwan main island with slack for the coast; the freeway network has no
* offshore-island cameras. */
export function isLikelyTaiwanCoordinate(lat, lon) {
function isLikelyTaiwanCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
+6 -1
View File
@@ -43,6 +43,11 @@ function number(value, min, max) {
const n = Number(value);
return Number.isFinite(n) && n >= min && n <= max ? n : null;
}
// CurrentStorms.json reports movementSpeed in miles per hour (intensity is in
// knots); convert so `speedKt` means what it says.
function knotsFromMph(mph) {
return mph === null ? null : Math.round(mph * 0.868976);
}
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
// up to about half an hour before; allow that, but not a clock far ahead.
const EARLY_ADVISORY_MS = 90 * 60_000;
@@ -120,7 +125,7 @@ export function parseCycloneStatus(payload, now = Date.now()) {
pressureHpa: number(raw.pressure, 800, 1100),
movement: {
directionDegrees: number(raw.movementDir, 0, 360),
speedKt: number(raw.movementSpeed, 0, 200),
speedKt: knotsFromMph(number(raw.movementSpeed, 0, 200)),
},
advisoryUrl: officialLink(forecast.url),
outlookUrl: raw.id.startsWith('al')
+9 -9
View File
@@ -27,13 +27,13 @@ import { normalizeDump1090Aircraft } from '../../src/sources/adsbRecords.js';
*/
export const LOCAL_RECEIVERS_ROUTE = '/api/local-receivers/aircraft';
export const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
export const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
export const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
export const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
export const LOCAL_RECEIVER_CACHE_MS = 1_000;
const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
const LOCAL_RECEIVER_CACHE_MS = 1_000;
/** A feed whose own `now` is older than this is reported stale. */
export const LOCAL_RECEIVER_STALE_MS = 10_000;
const LOCAL_RECEIVER_STALE_MS = 10_000;
const MAX_FEEDS = 8;
const LOG_PREFIX = '[local-receivers]';
@@ -139,7 +139,7 @@ class FeedError extends Error {
* @param {string} address Resolved IP address.
* @returns {boolean}
*/
export function isLocalReceiverAddress(address) {
function isLocalReceiverAddress(address) {
const value = String(address || '').toLowerCase();
if (value === '::1') return true;
const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(value);
@@ -152,7 +152,7 @@ export function isLocalReceiverAddress(address) {
* @param {typeof dnsLookup} lookupImpl `dns.promises.lookup`-compatible.
* @returns {Promise<Array<{address:string, family:number}>>}
*/
export async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
let resolved;
try {
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
@@ -274,7 +274,7 @@ function fetchLocalReceiverFeed(url, options) {
* @param {object} options
* @returns {Promise<object>} Parsed JSON document.
*/
export async function fetchLocalReceiverDocument(
async function fetchLocalReceiverDocument(
url,
{
fetchImpl = fetchLocalReceiverFeed,
+1 -1
View File
@@ -42,7 +42,7 @@ export class TileUpstreamError extends Error {
}
/** Seconds a Retry-After header asks for (an integer or an HTTP date), else null. */
export function retryAfterSeconds(header, now = Date.now()) {
function retryAfterSeconds(header, now = Date.now()) {
const text = String(header ?? '').trim();
if (!text) return null;
const seconds = /^\d+$/.test(text)
+1 -1
View File
@@ -145,7 +145,7 @@ const CODEX_LOGIN_ENV =
/^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
export function codexLoginEnvironment(environment, codexHome) {
function codexLoginEnvironment(environment, codexHome) {
const env = {};
for (const [name, value] of Object.entries(environment || {}))
if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
+1 -1
View File
@@ -157,4 +157,4 @@ function createDebugLogHandler({
};
}
export { createDebugLogHandler, omitVoiceContent };
export { createDebugLogHandler };
+1
View File
@@ -16,6 +16,7 @@ function realtimeInstructions(
'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.',
'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.',
'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
+3 -3
View File
@@ -511,18 +511,18 @@ export const ACTION_DESCRIPTIONS = {
},
move_camera: {
description:
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").',
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.',
$position: 1,
parameters: {
properties: {
direction: {
description:
'Required except for orbit (defaults right/clockwise) and stop.',
'Required for pan/tilt/rotate; orbit/rotate_globe default right.',
$position: 2,
},
mode: {
description:
'once = bounded eased nudge (default); continuous = until stop/manual input.',
'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.',
$position: 2,
},
},
+32 -59
View File
@@ -164,6 +164,30 @@ function overpassProxy({ routing = {} } = {}) {
readDisk: () =>
readOverpassDisk(cacheKey, overpassDiskTtlMs(cacheKey)),
allowUpstream: () => _overpassRateLimiter(clientKey(req)),
loadUpstream: async () => {
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT)
return {
status: 503,
contentType: 'application/json',
retryAfterMs: 2000,
body: JSON.stringify({
error: 'Overpass proxy busy — try again shortly',
}),
};
_overpassConcurrent++;
try {
const payload = await fetchOverpassPayload(safeBody);
if (overpassPayloadIsData(payload)) {
const entry = { ...payload, cachedAt: Date.now() };
_overpassCache.set(cacheKey, entry);
trimOverpassCache();
writeOverpassDisk(cacheKey, entry);
}
return payload;
} finally {
_overpassConcurrent--;
}
},
});
if (preflight.source === 'RATE_LIMITED') {
res.writeHead(429, {
@@ -173,71 +197,20 @@ function overpassProxy({ routing = {} } = {}) {
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
if (preflight.source !== 'UPSTREAM') {
// A coalesced caller sees the same failure as the original request
// and must get the same last-good fallback, not the raw refusal.
if (!overpassPayloadIsData(preflight.payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
if (preflight.source === 'DISK') {
_overpassCache.set(cacheKey, preflight.payload);
trimOverpassCache();
}
sendOverpassResponse(res, preflight.payload, preflight.source);
return;
}
// From here onward the request is genuinely upstream-bound and has
// consumed one local limiter slot. Cache and dedupe hits above do not.
if (_overpassConcurrent >= OVERPASS_MAX_CONCURRENT) {
res.writeHead(503, {
'Content-Type': 'application/json',
'Retry-After': '2',
});
res.end(
JSON.stringify({
error: 'Overpass proxy busy — try again shortly',
}),
);
return;
}
_overpassConcurrent += 1;
const requestPromise = fetchOverpassPayload(safeBody)
.then((payload) => {
// Only a 2xx is data. `< 500` cached every 4xx, so one mirror's
// refusal was written to memory AND disk — and boundary-class
// queries hold a month-long TTL, so a single 406 outlived the
// outage that caused it.
if (overpassPayloadIsData(payload)) {
const entry = { ...payload, cachedAt: Date.now() };
_overpassCache.set(cacheKey, entry);
trimOverpassCache();
writeOverpassDisk(cacheKey, entry);
}
return payload;
})
.finally(() => {
_overpassConcurrent -= 1;
_overpassInFlight.delete(cacheKey);
});
_overpassInFlight.set(cacheKey, requestPromise);
const payload = await requestPromise;
// Degraded upstream (rate-limited on every mirror / 5xx / runtime
// error): last-good roads beat an empty layer — serve stale from
// memory or disk at ANY age before surfacing the failure.
if (!overpassPayloadIsData(payload)) {
if (!overpassPayloadIsData(preflight.payload)) {
const stale = await readStaleOverpass(cacheKey);
if (stale) {
sendOverpassResponse(res, stale, 'STALE');
return;
}
}
sendOverpassResponse(res, payload, 'MISS');
// Preflight already published the disk hit to memory; only trim.
if (preflight.source === 'DISK') trimOverpassCache();
sendOverpassResponse(
res,
preflight.payload,
preflight.source === 'UPSTREAM' ? 'MISS' : preflight.source,
);
} catch (e) {
// Every mirror threw (network-level). Same serve-stale rule.
const stale = cacheKey ? await readStaleOverpass(cacheKey) : null;
+29 -13
View File
@@ -1,3 +1,4 @@
import { coalesceProxyRequest } from '../common/http.js';
import { isOverpassBoundaryQuery } from './query.js';
import {
OVERPASS_BOUNDARY_DISK_TTL_MS,
@@ -78,7 +79,9 @@ function writeOverpassDisk(cacheKey, payload) {
/**
* Resolve every cache/coalescing layer before admitting a request to the local
* upstream rate limiter. The injected limiter callback is invoked exactly once
* for a complete cache miss and never for memory, in-flight, or disk hits.
* for a complete cache miss and never for memory, in-flight, or disk hits. A
* caller that joined a reservation rejected by another client's limiter
* retries with its own limiter instead of inheriting that 429.
* Exported so the admission ordering can be tested without a Vite server.
*
* @param {object} options
@@ -87,6 +90,7 @@ function writeOverpassDisk(cacheKey, payload) {
* @param {Map<string, Promise<object>>} options.inFlight
* @param {()=>Promise<object|null>} options.readDisk
* @param {()=>boolean} options.allowUpstream
* @param {()=>Promise<object>} options.loadUpstream Fetch and publish the admitted result.
* @param {number} [options.now]
* @param {number} [options.cacheMs]
* @returns {Promise<{source:'HIT'|'INFLIGHT'|'DISK'|'UPSTREAM'|'RATE_LIMITED', payload:object|null}>}
@@ -97,22 +101,34 @@ async function resolveOverpassPreflight({
inFlight,
readDisk,
allowUpstream,
loadUpstream = async () => null,
now = Date.now(),
cacheMs = OVERPASS_CACHE_MS,
}) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
return { source: 'HIT', payload: cached };
for (;;) {
const cached = memoryCache.get(cacheKey);
if (overpassPayloadIsData(cached) && now - cached.cachedAt <= cacheMs)
return { source: 'HIT', payload: cached };
const pending = inFlight.get(cacheKey);
if (pending) return { source: 'INFLIGHT', payload: await pending };
const disk = await readDisk();
if (overpassPayloadIsData(disk)) return { source: 'DISK', payload: disk };
return allowUpstream()
? { source: 'UPSTREAM', payload: null }
: { source: 'RATE_LIMITED', payload: null };
// Reserve synchronously before disk I/O. The owner retains the reservation
// through its own upstream admission, fetch and cache publication.
const request = coalesceProxyRequest(inFlight, cacheKey, async () => {
const disk = await readDisk();
if (overpassPayloadIsData(disk)) {
memoryCache.set(cacheKey, disk);
return { source: 'DISK', payload: disk };
}
if (!allowUpstream()) return { source: 'RATE_LIMITED', payload: null };
return { source: 'UPSTREAM', payload: await loadUpstream() };
});
const result = await request.promise;
if (!request.shared) return result;
// The owner's quota rejection belongs to the owner's client. The rejected
// reservation is released before joiners resume, so retry and let this
// caller own the next reservation (or join an admitted one) on its quota.
if (result.source === 'RATE_LIMITED') continue;
return { ...result, source: 'INFLIGHT' };
}
}
/** Return only last-good Overpass data, regardless of its age. */
+30 -24
View File
@@ -5,7 +5,7 @@ import { readResponseTextCapped } from '../common/http.js';
import {
normalizeRadioBrowserStation,
publicRadioStation,
cleanRadioText,
normalizeRadioTag,
} from './stations.js';
import {
radioMirrorOrigin,
@@ -25,8 +25,9 @@ import {
RADIO_USER_AGENT,
RADIO_UUID_RE,
RADIO_FALLBACK_MIRRORS,
RADIO_REJECTED_STATION_COUNT_MAX,
} from './constants.js';
export async function mapRadioConcurrent(values, concurrency, mapper) {
async function mapRadioConcurrent(values, concurrency, mapper) {
const results = new Array(values.length);
let cursor = 0;
const workers = Array.from(
@@ -177,29 +178,25 @@ export function createRadioProxyMiddleware({
const rows = await fetchPath(`/json/stations/search?${params}`);
if (!Array.isArray(rows))
throw new Error('Radio Browser catalog payload was not an array');
if (
!rows.every(
(row) =>
row &&
typeof row === 'object' &&
!Array.isArray(row) &&
typeof row.stationuuid === 'string' &&
typeof row.name === 'string' &&
(typeof row.url_resolved === 'string' ||
typeof row.url === 'string'),
)
)
throw new Error(
'Radio Browser catalog contained a malformed station row',
);
const stations = rows
const structuralRows = rows.filter(
(row) =>
row &&
typeof row === 'object' &&
!Array.isArray(row) &&
typeof row.stationuuid === 'string' &&
typeof row.name === 'string' &&
(typeof row.url_resolved === 'string' ||
typeof row.url === 'string'),
);
// Malformed rows are skipped individually, but counted so the
// catalog keeps a diagnostic for upstream schema drift. Policy
// rejections in normalization (offline, non-HTTPS, codec) are normal
// filtering and are not counted.
const rejectedRows = rows.length - structuralRows.length;
const stations = structuralRows
.map(normalizeRadioBrowserStation)
.filter(Boolean);
const requestedTag = cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim();
const requestedTag = normalizeRadioTag(tag);
const requestedTagCovered =
!requestedTag ||
stations.some((station) =>
@@ -215,9 +212,10 @@ export function createRadioProxyMiddleware({
// also contain an accepted station tagged for the requested category.
succeeded: stations.length > 0 && requestedTagCovered,
stations,
rejectedRows,
};
} catch {
return { succeeded: false, stations: [] };
return { succeeded: false, stations: [], rejectedRows: 0 };
}
},
);
@@ -257,6 +255,10 @@ export function createRadioProxyMiddleware({
if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS)
healthReasons.push('station-coverage-below-policy');
const degraded = healthReasons.length > 0;
const rejectedStationCount = Math.min(
RADIO_REJECTED_STATION_COUNT_MAX,
outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0),
);
const coverage = {
successfulQueries,
totalQueries: queries.length,
@@ -271,6 +273,9 @@ export function createRadioProxyMiddleware({
degraded,
degradedReason: degraded ? healthReasons.join(',') : null,
coverage,
// Diagnostic only: it never fails health policy, so a usable generation
// with a few skipped rows is still admitted. The client reports it.
rejectedStationCount,
};
if (degraded && catalogCache) {
const error = new Error(
@@ -360,6 +365,7 @@ export function createRadioProxyMiddleware({
coverage: catalog.coverage || null,
acceptedGeneration: catalog.acceptedGeneration ?? null,
catalogInstance,
rejectedStationCount: catalog.rejectedStationCount ?? 0,
});
} catch (error) {
sendJson(res, 503, {
+3
View File
@@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
export const RADIO_USER_AGENT =
'GodsEyeView/1.0 (Radio Browser directory client)';
export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js';
// Upper bound for the structurally malformed upstream rows reported in the
// /stations envelope. The client layer validates against the same bound.
export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000;
export const RADIO_FALLBACK_MIRRORS = Object.freeze([
'https://de1.api.radio-browser.info',
'https://de2.api.radio-browser.info',
+1
View File
@@ -1,5 +1,6 @@
export {
cleanRadioText,
normalizeRadioTag,
isNonGlobalIpv4,
publicRadioHttpsUrl,
normalizeRadioBrowserStation,
+59 -6
View File
@@ -1,6 +1,8 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import { providerCooldownMs } from './cooldown.js';
import { admitSameSiteKeyless } from '../common/same-site.js';
import { readResponseTextCapped } from '../common/http.js';
/**
@@ -24,6 +26,18 @@ import { readResponseTextCapped } from '../common/http.js';
export function celestrakProxy() {
const TLE_TTL_MS = 6 * 3600_000;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const groups = new Set([
'stations',
'visual',
'gps-ops',
'glo-ops',
'galileo',
'geo',
'starlink',
// Space Missions (`getActiveTle`) reads the full active catalog.
'active',
]);
let cooldownUntil = 0;
const mem = new Map(); // group -> { at: epochMs, body: string }
const inflight = new Map(); // group -> Promise<{at, body}|null>
@@ -64,7 +78,16 @@ export function celestrakProxy() {
'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
},
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
if (!res.ok) {
// Group refreshes run in parallel: a later failure without Retry-After
// must never shorten a longer deadline an earlier one already set.
cooldownUntil = Math.max(
cooldownUntil,
Date.now() + providerCooldownMs(res),
);
void res.body?.cancel().catch(() => {});
throw new Error(`HTTP ${res.status}`);
}
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
@@ -80,12 +103,20 @@ export function celestrakProxy() {
const group = String(req.url || '')
.replace(/^\//, '')
.split('?')[0];
if (!/^[a-z0-9-]+$/i.test(group)) {
// Keyless and needed to render the satellites layer: refuse cross-site
// browsers but keep the documented reverse-proxy deployment working.
if (admitSameSiteKeyless(req, res)) return;
if (req.method !== 'GET') {
res.writeHead(405);
res.end();
return;
}
if (!groups.has(group)) {
res.writeHead(400, { 'Content-Type': 'text/plain' });
res.end('invalid group');
return;
}
const send = (status, body, cacheStatus) => {
const send = (status, body, cacheStatus, entry) => {
// Guard against a double-send (e.g. a throw AFTER a response already
// went out routing into the catch's send): writeHead after headersSent
// throws "Cannot set headers after they are sent".
@@ -93,6 +124,17 @@ export function celestrakProxy() {
res.writeHead(status, {
'Content-Type': 'text/plain',
'x-tle-cache': cacheStatus,
'Cache-Control': 'no-store',
...(entry
? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
: {}),
...(cooldownUntil > 0 && Date.now() < cooldownUntil
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
),
}
: {}),
});
res.end(body);
};
@@ -104,7 +146,16 @@ export function celestrakProxy() {
if (entry) mem.set(group, entry);
}
if (entry && now - entry.at < TLE_TTL_MS) {
send(200, entry.body, 'HIT');
send(200, entry.body, 'HIT', entry);
return;
}
if (Date.now() < cooldownUntil) {
send(
entry ? 200 : 502,
entry?.body || 'celestrak upstream cooling down',
entry ? 'STALE-ERROR' : 'NONE',
entry,
);
return;
}
// Stale or missing → refresh, single-flight per group.
@@ -118,6 +169,8 @@ export function celestrakProxy() {
return fresh;
})
.catch((err) => {
if (Date.now() >= cooldownUntil)
cooldownUntil = Date.now() + 60000;
console.warn(
'[celestrak-proxy] refresh failed — serving cache if any',
);
@@ -128,9 +181,9 @@ export function celestrakProxy() {
}
const fresh = await inflight.get(group);
if (fresh) {
send(200, fresh.body, 'MISS');
send(200, fresh.body, 'MISS', fresh);
} else if (entry) {
send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty
send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty
} else {
send(502, 'celestrak fetch failed and no cache available', 'NONE');
}
+12
View File
@@ -0,0 +1,12 @@
/** Bound an upstream Retry-After delay (seconds or HTTP date). */
export function providerCooldownMs(response, now = Date.now()) {
const raw = response?.headers?.get('retry-after')?.trim();
const seconds = raw && /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) : NaN;
const deadline = raw && !Number.isFinite(seconds) ? Date.parse(raw) : NaN;
const delay = Number.isFinite(seconds)
? seconds * 1000
: Number.isFinite(deadline)
? deadline - now
: 60000;
return Math.min(30 * 60000, Math.max(5000, delay));
}
+38 -2
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { providerCooldownMs } from './cooldown.js';
import { promises as fsp } from 'node:fs';
import {
readResponseTextCapped,
@@ -30,6 +31,8 @@ export function rocketLaunchesProxy() {
'launch-library-2-v2.3.json',
);
let cache = null;
let cooldownUntil = 0;
let cooldownStatus = 502;
let diskLoaded = false;
const inFlight = new Map();
@@ -62,7 +65,18 @@ export function rocketLaunchesProxy() {
function send(res, status, body, cacheState) {
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': status === 200 ? 'public, max-age=900' : 'no-store',
'Cache-Control':
status === 200 && cacheState !== 'STALE-ERROR'
? 'public, max-age=900'
: 'no-store',
...(cache ? { 'X-GEV-Cached-At': new Date(cache.at).toISOString() } : {}),
...(Date.now() < cooldownUntil
? {
'Retry-After': String(
Math.ceil((cooldownUntil - Date.now()) / 1000),
),
}
: {}),
'X-GEV-Cache': cacheState,
});
res.end(body);
@@ -75,17 +89,25 @@ export function rocketLaunchesProxy() {
signal: AbortSignal.timeout(20000),
headers: launchLibraryRequestHeaders(),
});
const body = await readResponseTextCapped(upstream, maxResponseBytes);
if (!upstream.ok) {
const deadline = Date.now() + providerCooldownMs(upstream);
// Never shorten a longer cooldown that is already in force.
if (deadline >= cooldownUntil) {
cooldownUntil = deadline;
cooldownStatus = upstream.status;
}
void upstream.body?.cancel().catch(() => {});
const error = new Error(`upstream HTTP ${upstream.status}`);
error.upstreamStatus = upstream.status;
throw error;
}
const body = await readResponseTextCapped(upstream, maxResponseBytes);
const parsed = JSON.parse(body);
if (!Array.isArray(parsed?.results))
throw new Error('malformed upstream response');
const fresh = { at: Date.now(), body };
cache = fresh;
cooldownUntil = 0;
void saveDiskCache(fresh);
return fresh;
}
@@ -103,6 +125,16 @@ export function rocketLaunchesProxy() {
return;
}
const stale = cache;
if (now < cooldownUntil) {
send(
res,
stale ? 200 : cooldownStatus,
stale?.body ||
JSON.stringify({ error: 'Launch Library 2 unavailable' }),
stale ? 'STALE-ERROR' : 'NONE',
);
return;
}
const request = coalesceProxyRequest(
inFlight,
'recent-launches',
@@ -112,6 +144,10 @@ export function rocketLaunchesProxy() {
const fresh = await request.promise;
send(res, 200, fresh.body, request.shared ? 'INFLIGHT' : 'MISS');
} catch (error) {
if (Date.now() >= cooldownUntil) {
cooldownUntil = Date.now() + 60000;
cooldownStatus = 502;
}
// Log only a bounded status, never upstream bodies, URLs, or credentials.
const status = Number.isInteger(error?.upstreamStatus)
? error.upstreamStatus
+20 -11
View File
@@ -20,8 +20,8 @@ import { admitSameSiteKeyless } from './common/same-site.js';
* cache and serve-stale shape. Cache entries and stale fallback are keyed per
* 5dp point, so reordered and partially overlapping batches reuse prior work.
* Only missing/stale points go upstream; the response is rebuilt in exact
* request order. Larger requests are chunked sequentially, and one failing
* chunk does not discard the chunks that resolved.
* request order. Each request is bounded to one 64-point upstream chunk;
* larger client workloads are split by the browser terrain service.
*
* Bounds, because every distinct 5dp point is its own key: coordinates must be
* in WGS-84 range (rejected at parse), the point cache has an entry ceiling
@@ -55,7 +55,7 @@ export function terrainHeightsProxy(options = {}) {
// 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
// that range.
const UPSTREAM_CHUNK = 64;
const MAX_POINTS = 2000;
const MAX_POINTS = UPSTREAM_CHUNK;
/**
* Entry ceiling for the point cache, and with it the file written from it.
*
@@ -90,15 +90,23 @@ export function terrainHeightsProxy(options = {}) {
/**
* Drop the oldest entries until the map is back inside its ceiling.
* Oldest-first by fetch time, the same shape `trackBackfillProxies` uses.
* Terrain does not move, so evicting an old point costs one refetch, never
* a wrong answer.
* The map is kept in fetch-time order (sorted once on load, and the resolver
* re-inserts every point it refetches), so the oldest are at the front and
* no write sorts the map. Terrain does not move, so evicting an old point
* costs one refetch, never a wrong answer.
*/
function evictOverflow() {
if (mem.size <= CACHE_MAX_ENTRIES) return;
for (const key of mem.keys()) {
if (mem.size <= CACHE_MAX_ENTRIES) return;
mem.delete(key);
}
}
/** Put the map in fetch-time order; a file is not guaranteed to be. */
function orderByAge() {
const byAge = [...mem.entries()].sort((a, b) => a[1].at - b[1].at);
for (let i = 0; i < byAge.length && mem.size > CACHE_MAX_ENTRIES; i += 1)
mem.delete(byAge[i][0]);
mem.clear();
for (const [key, entry] of byAge) mem.set(key, entry);
}
/** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
const inflight = new Map();
@@ -165,6 +173,7 @@ export function terrainHeightsProxy(options = {}) {
}
diskDirty = mem.size > 0;
}
orderByAge();
evictOverflow();
} catch {
/* no disk cache yet */
@@ -255,7 +264,6 @@ export function terrainHeightsProxy(options = {}) {
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
await loadDiskOnce();
const parsedUrl = new URL(req.url || '', 'http://internal');
const rawPoints = parsedUrl.searchParams.get('points');
const points = parseTerrainPoints(rawPoints);
@@ -267,12 +275,13 @@ export function terrainHeightsProxy(options = {}) {
return;
}
if (points.length > MAX_POINTS) {
send(500, {
send(413, {
error: `too many points (${points.length}); max ${MAX_POINTS} per request`,
});
return;
}
await loadDiskOnce();
const outcome = await resolveTerrainHeightRequest({
points,
cache: mem,
+9 -6
View File
@@ -182,14 +182,17 @@ export function tomtomProxy() {
});
res.end(JSON.stringify(obj));
};
const sendTile = (buf, cacheStatus) => {
// x-tomtom-fetched-at is when the proxy fetched the tile (ms since the
// epoch), so a client handed a STALE-* copy can say how old it is.
const sendTile = (entry, cacheStatus) => {
if (res.headersSent) return;
res.writeHead(200, {
'Content-Type': 'application/x-protobuf',
'Cache-Control': 'no-store',
'x-tomtom-cache': cacheStatus,
'x-tomtom-fetched-at': String(Math.round(entry.at)),
});
res.end(buf);
res.end(entry.buf);
};
try {
@@ -235,14 +238,14 @@ export function tomtomProxy() {
}
// Fresh cache hit — never counts against the budget.
if (entry && now - entry.at < TILE_TTL_MS) {
sendTile(entry.buf, 'HIT');
sendTile(entry, 'HIT');
return;
}
// Budget governor: over the soft cap, last-good data beats a dead layer.
if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) {
if (entry) {
sendTile(entry.buf, 'STALE-BUDGET');
sendTile(entry, 'STALE-BUDGET');
} else {
sendJson(429, { error: 'budget' });
}
@@ -271,9 +274,9 @@ export function tomtomProxy() {
}
const fresh = await inflight.get(key);
if (fresh) {
sendTile(fresh.buf, 'MISS');
sendTile(fresh, 'MISS');
} else if (entry) {
sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty
sendTile(entry, 'STALE-ERROR'); // upstream down — stale beats empty
} else {
sendJson(502, { error: 'upstream' });
}
-1
View File
@@ -1,5 +1,4 @@
export {
createTransitHistory,
TRANSIT_HISTORY_LIMITS,
validTransitIdentifier,
} from '../../src/sources/transitHistoryStore.js';
+55 -10
View File
@@ -43,14 +43,55 @@ export function ingestAisStreamEnvelope(envelope) {
if (!mmsi) return false;
if (messageType === 'ShipStaticData' || messageType === 'StaticDataReport') {
const previous = _aisStreamStatic.get(mmsi) || {};
const live = _aisStreamVessels.get(mmsi);
const liveName = live?.name === `MMSI ${mmsi}` ? '' : live?.name;
const fields =
messageType === 'StaticDataReport'
? {
Name:
message.ReportA?.Valid === false
? undefined
: message.ReportA?.Name,
Type:
message.ReportB?.Valid === false
? undefined
: message.ReportB?.ShipType,
}
: message;
// Static reports may be partial (including Class B's separate A/B parts).
// Keep known values in the cache as well as in the current live row.
const staticData = {
name: vesselNameFromAis(metadata, message, _aisStreamStatic.get(mmsi)),
type: vesselTypeFromAis(message, _aisStreamStatic.get(mmsi)),
destination: stringValue(message.Destination),
imo: stringValue(message.ImoNumber ?? message.IMO),
// The envelope repeats a cached name even on Class B's type-only part.
// Only an actual static name report replaces a known live name.
name:
stringValue(fields.Name ?? fields.ShipName) ||
liveName ||
previous.name ||
stringValue(metadata.ShipName),
type:
staticCode(fields.Type ?? fields.ShipType) ||
staticCode(live?.type) ||
staticCode(previous.type) ||
'',
destination:
stringValue(fields.Destination) ||
live?.destination ||
previous.destination ||
'',
imo:
staticCode(fields.ImoNumber ?? fields.IMO) ||
staticCode(live?.imo) ||
staticCode(previous.imo) ||
'',
};
_aisStreamStatic.set(mmsi, staticData);
mergeAisStaticIntoLiveVessel(mmsi, staticData);
// AISStream repeats the last known position in a static message's
// MetaData, but the message has no speed, course or heading and is not a
// new fix. Rebuilding a tracked vessel from it would blank its motion and
// restamp its position time, so only merge the static fields.
if (_aisStreamVessels.has(mmsi)) return true;
}
const lat = numberValue(
@@ -191,12 +232,10 @@ function approxMetersBetween(lat1, lon1, lat2, lon2) {
function mergeAisStaticIntoLiveVessel(mmsi, staticData) {
const existing = _aisStreamVessels.get(mmsi);
if (!existing) return;
if (staticData.name && (!existing.name || existing.name === `MMSI ${mmsi}`))
existing.name = staticData.name;
if (staticData.type && !existing.type) existing.type = staticData.type;
if (staticData.destination && !existing.destination)
existing.destination = staticData.destination;
if (staticData.imo && !existing.imo) existing.imo = staticData.imo;
if (staticData.name) existing.name = staticData.name;
if (staticData.type) existing.type = staticData.type;
if (staticData.destination) existing.destination = staticData.destination;
if (staticData.imo) existing.imo = staticData.imo;
}
function vesselNameFromAis(metadata, message, staticData = {}) {
@@ -266,6 +305,12 @@ function stringValue(value) {
return String(value).trim();
}
// AIS uses zero for unavailable ship type and IMO, not as a replacement value.
function staticCode(value) {
const number = numberValue(value);
return Number.isInteger(number) && number > 0 ? String(number) : '';
}
function numberValue(value) {
if (value === null || value === undefined || value === '') return null;
const number = Number(value);
+1 -1
View File
@@ -3,7 +3,7 @@ import { decodeWindGribMessage } from './decode.js';
import { resampleWeatherSnapshot, weatherScalarMetadata } from './grid.js';
/** Keyless ECMWF Open Data root for real-time IFS forecasts. */
export const IFS_BASE = 'https://data.ecmwf.int/forecasts';
const IFS_BASE = 'https://data.ecmwf.int/forecasts';
/**
* Select the latest IFS cycle that should be available.