mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 02:14:01 +00:00
Merge branch 'main' into codex/fix-terrain-request-bound
This commit is contained in:
130 files changed
+18671
-610
No files matched your search
@@ -1,4 +1,7 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
|
||||
@@ -19,6 +22,7 @@ import { readResponseTextCapped } from '../common/http.js';
|
||||
export function adsbLolProxy() {
|
||||
/** @type {string|null} Cached upstream JSON body. */
|
||||
let _cache = null;
|
||||
const inFlight = new Map();
|
||||
/** @type {number} Epoch-ms when the cache was populated. */
|
||||
let _cacheAt = 0;
|
||||
/** @type {number} Epoch-ms until which upstream is not contacted. */
|
||||
@@ -104,60 +108,73 @@ export function adsbLolProxy() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const refresh = coalesceProxyRequest(inFlight, 'global', async () => {
|
||||
const result = (status, body, cacheStatus, extra = {}) => ({
|
||||
status,
|
||||
body,
|
||||
cacheStatus,
|
||||
extra,
|
||||
});
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
_cooldownStatus = 0;
|
||||
return result(200, body, 'MISS');
|
||||
}
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
const failedAt = Date.now();
|
||||
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
|
||||
_cooldownStatus = upstream.status;
|
||||
console.warn(
|
||||
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
|
||||
);
|
||||
if (_cache) {
|
||||
// Do not wait for a failing response body before serving usable data.
|
||||
upstream.body?.cancel().catch(() => {});
|
||||
return result(200, _cache, 'STALE', {
|
||||
'X-ADS-B-Upstream-Status': String(upstream.status),
|
||||
'X-ADS-B-Retry-After-Seconds': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
_cooldownStatus = 0;
|
||||
serve(res, 200, body, 'MISS');
|
||||
return;
|
||||
}
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
const failedAt = Date.now();
|
||||
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
|
||||
_cooldownStatus = upstream.status;
|
||||
console.warn(
|
||||
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
|
||||
return result(
|
||||
upstream.status,
|
||||
body,
|
||||
'MISS',
|
||||
_cooldownUntil > Date.now()
|
||||
? {
|
||||
'Retry-After': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
}
|
||||
: {},
|
||||
);
|
||||
if (_cache) {
|
||||
// Do not wait for a failing response body before serving usable data.
|
||||
upstream.body?.cancel().catch(() => {});
|
||||
serve(res, 200, _cache, 'STALE', {
|
||||
'X-ADS-B-Upstream-Status': String(upstream.status),
|
||||
'X-ADS-B-Retry-After-Seconds': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
});
|
||||
const response = await refresh.promise;
|
||||
serve(
|
||||
res,
|
||||
upstream.status,
|
||||
body,
|
||||
'MISS',
|
||||
_cooldownUntil > Date.now()
|
||||
? {
|
||||
'Retry-After': String(
|
||||
Math.ceil((_cooldownUntil - Date.now()) / 1000),
|
||||
),
|
||||
}
|
||||
: {},
|
||||
response.status,
|
||||
response.body,
|
||||
response.cacheStatus,
|
||||
response.extra,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error('[adsb.lol Proxy]', e.message);
|
||||
|
||||
@@ -67,15 +67,24 @@ async function fetchOpenSkyStates(headers) {
|
||||
try {
|
||||
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
|
||||
if (upstream.status === 429) {
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
// Start the cooldown from the headers and drop the rejected body
|
||||
// unread, so a stalled or oversized 429 body cannot skip it.
|
||||
_openskyCooldownUntil =
|
||||
Date.now() + openSkyRateLimitCooldownMs(upstream);
|
||||
Date.now() +
|
||||
(upstream.status === 429
|
||||
? openSkyRateLimitCooldownMs(upstream)
|
||||
: 15000);
|
||||
_openskyCooldownStatus = upstream.status;
|
||||
void upstream.body?.cancel().catch(() => {});
|
||||
return {
|
||||
upstream,
|
||||
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
|
||||
body: JSON.stringify({
|
||||
error:
|
||||
upstream.status === 429
|
||||
? 'OpenSky rate limited.'
|
||||
: 'OpenSky upstream unavailable.',
|
||||
}),
|
||||
};
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
@@ -109,6 +118,8 @@ async function fetchOpenSkyStates(headers) {
|
||||
let _openskyTtlMs = OPENSKY_CACHE_MS;
|
||||
/** @type {number} Epoch-ms before which no upstream fetch is attempted. */
|
||||
let _openskyCooldownUntil = 0;
|
||||
let _openskyCooldownStatus = 429;
|
||||
const _openskySnapshotInFlight = new Map();
|
||||
/**
|
||||
* Picks the cache TTL from the remaining daily credit budget.
|
||||
* Client polls every 30 s, so tiers ≤30 s cost the same 480 credits/h; the
|
||||
@@ -460,7 +471,9 @@ export function openSkyProxy() {
|
||||
requestedMode: cachedMeta.requestedMode || requestedMode,
|
||||
usedMode: cachedMeta.usedMode || 'unknown',
|
||||
reason: isStale
|
||||
? 'rate_limited_serving_stale'
|
||||
? _openskyCooldownStatus === 429
|
||||
? 'rate_limited_serving_stale'
|
||||
: `upstream_http_${_openskyCooldownStatus}_serving_stale`
|
||||
: cachedMeta.reason || 'cached',
|
||||
staleSeconds: isStale
|
||||
? (now - _openskyCacheTime) / 1000
|
||||
@@ -487,80 +500,140 @@ export function openSkyProxy() {
|
||||
)
|
||||
return;
|
||||
res.writeHead(
|
||||
429,
|
||||
_openskyCooldownStatus,
|
||||
buildOpenSkyHeaders({
|
||||
cacheStatus: 'COOLDOWN',
|
||||
requestedMode,
|
||||
usedMode: 'none',
|
||||
reason: 'rate_limited',
|
||||
reason:
|
||||
_openskyCooldownStatus === 429
|
||||
? 'rate_limited'
|
||||
: `upstream_http_${_openskyCooldownStatus}`,
|
||||
retryAfterSeconds: (_openskyCooldownUntil - now) / 1000,
|
||||
}),
|
||||
);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'OpenSky rate limited; proxy cooling down.',
|
||||
error: 'OpenSky upstream unavailable; proxy cooling down.',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const basicUser = process.env.OPENSKY_USERNAME || '';
|
||||
const basicPass = process.env.OPENSKY_PASSWORD || '';
|
||||
const hasBasicCreds = Boolean(basicUser && basicPass);
|
||||
const headers = { Accept: 'application/json' };
|
||||
let usedMode = 'anon';
|
||||
let reason = 'forced_anonymous';
|
||||
const refresh = coalesceProxyRequest(
|
||||
_openskySnapshotInFlight,
|
||||
'global',
|
||||
async () => {
|
||||
const basicUser = process.env.OPENSKY_USERNAME || '';
|
||||
const basicPass = process.env.OPENSKY_PASSWORD || '';
|
||||
const hasBasicCreds = Boolean(basicUser && basicPass);
|
||||
const headers = { Accept: 'application/json' };
|
||||
let usedMode = 'anon';
|
||||
let reason = 'forced_anonymous';
|
||||
|
||||
if (requestedMode === 'basic') {
|
||||
if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'basic_credentials';
|
||||
} else {
|
||||
reason = 'missing_basic_creds';
|
||||
}
|
||||
} else if (requestedMode === 'oauth') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else {
|
||||
reason = 'oauth_invalid_or_missing';
|
||||
}
|
||||
} else if (requestedMode === 'auto') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_unavailable_fallback_basic';
|
||||
} else {
|
||||
reason = 'missing_oauth_and_basic_creds';
|
||||
}
|
||||
}
|
||||
if (requestedMode === 'basic') {
|
||||
if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'basic_credentials';
|
||||
} else {
|
||||
reason = 'missing_basic_creds';
|
||||
}
|
||||
} else if (requestedMode === 'oauth') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else {
|
||||
reason = 'oauth_invalid_or_missing';
|
||||
}
|
||||
} else if (requestedMode === 'auto') {
|
||||
const token = await getOpenSkyToken();
|
||||
if (token) {
|
||||
headers.Authorization = `Bearer ${token}`;
|
||||
usedMode = 'oauth';
|
||||
reason = 'oauth_token';
|
||||
} else if (hasBasicCreds) {
|
||||
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_unavailable_fallback_basic';
|
||||
} else {
|
||||
reason = 'missing_oauth_and_basic_creds';
|
||||
}
|
||||
}
|
||||
|
||||
let { upstream, body } = await fetchOpenSkyStates(headers);
|
||||
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
|
||||
if (
|
||||
(upstream.status === 401 || upstream.status === 403) &&
|
||||
requestedMode === 'auto' &&
|
||||
usedMode === 'oauth' &&
|
||||
hasBasicCreds
|
||||
) {
|
||||
const retryHeaders = {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
|
||||
};
|
||||
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_rejected_fallback_basic';
|
||||
}
|
||||
let { upstream, body } = await fetchOpenSkyStates(headers);
|
||||
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
|
||||
if (
|
||||
(upstream.status === 401 || upstream.status === 403) &&
|
||||
requestedMode === 'auto' &&
|
||||
usedMode === 'oauth' &&
|
||||
hasBasicCreds
|
||||
) {
|
||||
const retryHeaders = {
|
||||
Accept: 'application/json',
|
||||
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
|
||||
};
|
||||
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
|
||||
usedMode = 'basic';
|
||||
reason = 'oauth_rejected_fallback_basic';
|
||||
}
|
||||
|
||||
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
|
||||
const sourceEpochMs = upstream.ok
|
||||
? openSkySourceEpochMs(body)
|
||||
: null;
|
||||
// Refine the reason string to reflect the actual outcome
|
||||
if (upstream.ok && reason === 'forced_anonymous') {
|
||||
reason = 'anonymous_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'basic' &&
|
||||
reason === 'basic_credentials'
|
||||
) {
|
||||
reason = 'basic_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'oauth' &&
|
||||
reason === 'oauth_token'
|
||||
) {
|
||||
reason = 'oauth_ok';
|
||||
}
|
||||
|
||||
// Only cache successful responses — error responses (401/403/429/5xx)
|
||||
// should not be served from cache on subsequent requests
|
||||
if (upstream.ok) {
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = {
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason,
|
||||
};
|
||||
// Credit governor: adapt the cache TTL to the remaining daily
|
||||
// budget so a continuously-open app stretches its polls instead of
|
||||
// exhausting the quota mid-day. Success also clears any cooldown.
|
||||
const remaining = Number(
|
||||
upstream.headers.get('x-rate-limit-remaining'),
|
||||
);
|
||||
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
|
||||
_openskyCooldownUntil = 0;
|
||||
}
|
||||
|
||||
return {
|
||||
upstream,
|
||||
body,
|
||||
usedMode,
|
||||
reason,
|
||||
hasBasicCreds,
|
||||
sourceEpochMs,
|
||||
};
|
||||
},
|
||||
);
|
||||
let { upstream, body, usedMode, reason, hasBasicCreds, sourceEpochMs } =
|
||||
await refresh.promise;
|
||||
if (
|
||||
upstream.ok &&
|
||||
openSkySourceIsStale(sourceEpochMs, now) &&
|
||||
@@ -571,20 +644,16 @@ export function openSkyProxy() {
|
||||
'opensky_snapshot_stale_regional_fallback',
|
||||
))
|
||||
) {
|
||||
// Keep the last global snapshot available as a fail-soft cache,
|
||||
// but do not label or render it as a fresh live result.
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = { requestedMode, usedMode, reason };
|
||||
return;
|
||||
}
|
||||
if (upstream.status === 429) {
|
||||
reason = 'rate_limited';
|
||||
if (upstream.status === 429 || upstream.status >= 500) {
|
||||
reason =
|
||||
upstream.status === 429
|
||||
? 'rate_limited'
|
||||
: `upstream_http_${upstream.status}`;
|
||||
// Credit governor: fetchOpenSkyStates already started the cooldown
|
||||
// from OpenSky's retry-after — no upstream attempts until then.
|
||||
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
|
||||
const cooldownMs = Math.max(0, _openskyCooldownUntil - Date.now());
|
||||
// Serve the last-good body instead of the 429 when we have one —
|
||||
// the layer keeps rendering (STALE-cued) instead of dying.
|
||||
if (_openskyCacheBody && _openskyCacheStatus === 200) {
|
||||
@@ -594,7 +663,7 @@ export function openSkyProxy() {
|
||||
cacheStatus: 'STALE',
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason: 'rate_limited_serving_stale',
|
||||
reason: `${reason}_serving_stale`,
|
||||
staleSeconds: (now - _openskyCacheTime) / 1000,
|
||||
retryAfterSeconds: cooldownMs / 1000,
|
||||
}),
|
||||
@@ -652,45 +721,6 @@ export function openSkyProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
// Refine the reason string to reflect the actual outcome
|
||||
if (upstream.ok && reason === 'forced_anonymous') {
|
||||
reason = 'anonymous_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'basic' &&
|
||||
reason === 'basic_credentials'
|
||||
) {
|
||||
reason = 'basic_ok';
|
||||
} else if (
|
||||
upstream.ok &&
|
||||
usedMode === 'oauth' &&
|
||||
reason === 'oauth_token'
|
||||
) {
|
||||
reason = 'oauth_ok';
|
||||
}
|
||||
|
||||
// Only cache successful responses — error responses (401/403/429/5xx)
|
||||
// should not be served from cache on subsequent requests
|
||||
if (upstream.ok) {
|
||||
_openskyCacheBody = body;
|
||||
_openskyCacheStatus = upstream.status;
|
||||
_openskyCacheTime = now;
|
||||
_openskyCacheSourceEpochMs = sourceEpochMs;
|
||||
_openskyCacheMeta = {
|
||||
requestedMode,
|
||||
usedMode,
|
||||
reason,
|
||||
};
|
||||
// Credit governor: adapt the cache TTL to the remaining daily
|
||||
// budget so a continuously-open app stretches its polls instead of
|
||||
// exhausting the quota mid-day. Success also clears any cooldown.
|
||||
const remaining = Number(
|
||||
upstream.headers.get('x-rate-limit-remaining'),
|
||||
);
|
||||
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
|
||||
_openskyCooldownUntil = 0;
|
||||
}
|
||||
|
||||
res.writeHead(
|
||||
upstream.status,
|
||||
buildOpenSkyHeaders({
|
||||
|
||||
+63
-29
@@ -6,6 +6,8 @@ import {
|
||||
} from './cctv/normalize.js';
|
||||
import {
|
||||
buildSyntheticCctvSvg,
|
||||
cctvLiveMediaType,
|
||||
cctvRasterImageType,
|
||||
proxyMediaResponse,
|
||||
fetchCctvImageFromUpstream,
|
||||
fetchTxdotSnapshot,
|
||||
@@ -17,6 +19,8 @@ import {
|
||||
CCTV_MAX_SOURCES_CEILING,
|
||||
} from './cctv/constants.js';
|
||||
import { sanitizeCctvRangeHeader } from './cctv/range.js';
|
||||
import { admitSameSiteKeyless } from './common/same-site.js';
|
||||
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
|
||||
import { createHlsPuller } from './cctv/stream.js';
|
||||
import { googleServerApiKey } from './places/google-key.js';
|
||||
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
|
||||
@@ -116,8 +120,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' },
|
||||
signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const svType = svResp.headers.get('content-type') || '';
|
||||
if (!svResp.ok || !svType.startsWith('image/')) return null;
|
||||
const svType = cctvRasterImageType(svResp.headers.get('content-type'));
|
||||
if (!svResp.ok || !svType) {
|
||||
try {
|
||||
await svResp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
@@ -135,11 +146,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
});
|
||||
server.middlewares.use('/api/cctv', async (req, res) => {
|
||||
try {
|
||||
const url = new URL(req.url || '/', 'http://localhost');
|
||||
// Refuse cross-site requests before any catalog or upstream work. The
|
||||
// keyless gate keeps the routes reachable behind a reverse proxy; the
|
||||
// key-bearing Street View fallback below has its own check.
|
||||
if (admitSameSiteKeyless(req, res)) return;
|
||||
const sources = await getCctvSources();
|
||||
const sourceById = new Map(
|
||||
sources.map((source) => [source.id, source]),
|
||||
);
|
||||
const url = new URL(req.url || '/', 'http://localhost');
|
||||
|
||||
if (url.pathname === '/sources') {
|
||||
const body = {
|
||||
@@ -217,8 +232,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
const mediaUrl = source?.url || '';
|
||||
const feedType = normalizeFeedType(source?.feedType || 'image');
|
||||
// Live feeds only. A still camera is served by /frame, whose fetch
|
||||
// keeps redirects on the registered origin, accepts only image/* and
|
||||
// caps the body; relaying it here would bypass all three.
|
||||
// keeps redirects on the registered origin, accepts only raster
|
||||
// images and caps the body; relaying it here would bypass all three.
|
||||
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
|
||||
res.writeHead(404, {
|
||||
'Content-Type': 'application/json',
|
||||
@@ -382,7 +397,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
}
|
||||
return;
|
||||
}
|
||||
const contentType = upstream.headers.get('content-type') || '';
|
||||
if (!upstream.ok) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
@@ -407,31 +421,42 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
isVideoFeedType(feedType) &&
|
||||
!(
|
||||
contentType.startsWith('video/') ||
|
||||
contentType.includes('mpegurl')
|
||||
)
|
||||
) {
|
||||
// The relayed type is the feed kind's own, checked before the
|
||||
// camera counts as healthy and before any byte is relayed.
|
||||
const contentType = cctvLiveMediaType(
|
||||
feedType,
|
||||
upstream.headers.get('content-type'),
|
||||
);
|
||||
if (!contentType) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* already closed */
|
||||
}
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
sourceKind: 'upstream',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: `Unexpected media type ${contentType || 'unknown'}`,
|
||||
message: 'Unexpected media type',
|
||||
});
|
||||
} else {
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: isVideoFeedType(feedType)
|
||||
? 'Live stream connected'
|
||||
: 'Snapshot feed connected',
|
||||
res.writeHead(502, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Media proxy failed' }));
|
||||
return;
|
||||
}
|
||||
setHealth(cameraId, {
|
||||
status: 'ok',
|
||||
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
|
||||
label: source?.provider || 'Configured source',
|
||||
message: isVideoFeedType(feedType)
|
||||
? 'Live stream connected'
|
||||
: 'Snapshot feed connected',
|
||||
});
|
||||
|
||||
await proxyMediaResponse(res, upstream, {
|
||||
contentType,
|
||||
sourceHeader: isVideoFeedType(feedType)
|
||||
? 'live-media'
|
||||
: 'upstream-image',
|
||||
@@ -524,13 +549,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sv = await streetViewFallback({
|
||||
lat,
|
||||
lon,
|
||||
heading,
|
||||
fov,
|
||||
pitch,
|
||||
});
|
||||
// The Street View fallback spends the Google key, so a forwarded
|
||||
// request (whose client this server cannot see) gets the placeholder,
|
||||
// as if no key were configured. Any forwarding header counts, even an
|
||||
// empty one.
|
||||
const forwarded = PROXY_SIGNALS.some(
|
||||
(name) => req.headers?.[name] !== undefined,
|
||||
);
|
||||
const sv = forwarded
|
||||
? null
|
||||
: await streetViewFallback({
|
||||
lat,
|
||||
lon,
|
||||
heading,
|
||||
fov,
|
||||
pitch,
|
||||
});
|
||||
if (sv?.ok) {
|
||||
setHealth(cameraId, {
|
||||
status: 'degraded',
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
import { Readable } from 'node:stream';
|
||||
import { hashSeed, escapeXml } from './normalize.js';
|
||||
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
|
||||
import {
|
||||
CALGARY_IMAGE_ORIGIN,
|
||||
CCTV_FRAME_FETCH_TIMEOUT_MS,
|
||||
CCTV_FRAME_MAX_BODY_BYTES,
|
||||
CCTV_MEDIA_FETCH_TIMEOUT_MS,
|
||||
CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
CCTV_MEDIA_MAX_BODY_BYTES,
|
||||
HK_IMAGE_ORIGIN,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
NSW_IMAGE_ORIGIN,
|
||||
NSW_IMAGE_USER_AGENT,
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
TARKTEE_IMAGE_ORIGIN,
|
||||
TFL_IMAGE_ORIGIN,
|
||||
WARENDORF_IMAGE_ORIGINS,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
} from './constants.js';
|
||||
/**
|
||||
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
|
||||
@@ -93,6 +101,73 @@ export function toReadable(body) {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a Content-Type value into its lower-case essence and parameters.
|
||||
*
|
||||
* @param {string|null|undefined} value
|
||||
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
|
||||
* value is missing or is not a `type/subtype` media type.
|
||||
*/
|
||||
export function parseMediaType(value) {
|
||||
const [head, ...rest] = String(value ?? '').split(';');
|
||||
const essence = head.trim().toLowerCase();
|
||||
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
|
||||
const params = new Map();
|
||||
for (const part of rest) {
|
||||
const eq = part.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const name = part.slice(0, eq).trim().toLowerCase();
|
||||
let paramValue = part.slice(eq + 1).trim();
|
||||
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
|
||||
if (name && !params.has(name)) params.set(name, paramValue);
|
||||
}
|
||||
return { essence, params };
|
||||
}
|
||||
|
||||
/** Still-image types a camera frame may be served as. */
|
||||
const CCTV_RASTER_IMAGE_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/gif',
|
||||
'image/webp',
|
||||
'image/avif',
|
||||
]);
|
||||
|
||||
/**
|
||||
* The canonical type of a raster camera frame, or null for any other type.
|
||||
*
|
||||
* @param {string|null|undefined} contentType
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function cctvRasterImageType(contentType) {
|
||||
const essence = parseMediaType(contentType)?.essence;
|
||||
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The type to relay a live feed as, or null when the upstream type does not
|
||||
* match the feed kind. MJPEG keeps its part boundary, without which the body
|
||||
* cannot be split into frames.
|
||||
*
|
||||
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
|
||||
* @param {string|null|undefined} contentType
|
||||
* @returns {string|null}
|
||||
*/
|
||||
export function cctvLiveMediaType(feedType, contentType) {
|
||||
const parsed = parseMediaType(contentType);
|
||||
if (!parsed) return null;
|
||||
if (feedType === 'mp4' || feedType === 'webm')
|
||||
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
|
||||
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
|
||||
return null;
|
||||
const boundary = parsed.params.get('boundary') || '';
|
||||
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
|
||||
return null;
|
||||
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
|
||||
? `multipart/x-mixed-replace; boundary=${boundary}`
|
||||
: `multipart/x-mixed-replace; boundary="${boundary}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
|
||||
*
|
||||
@@ -104,6 +179,8 @@ export function toReadable(body) {
|
||||
* @param {Response} upstream - fetch() Response object.
|
||||
* @param {object} [opts]
|
||||
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
|
||||
* @param {string} [opts.contentType] - Validated type to send instead of the
|
||||
* upstream's own.
|
||||
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
|
||||
* allowed between upstream chunks before the stream is released. Injectable
|
||||
* only to keep the deadline unit-testable.
|
||||
@@ -114,10 +191,10 @@ export async function proxyMediaResponse(
|
||||
{
|
||||
sourceHeader = 'upstream',
|
||||
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
|
||||
contentType = upstream.headers.get('content-type') ||
|
||||
'application/octet-stream',
|
||||
} = {},
|
||||
) {
|
||||
const contentType =
|
||||
upstream.headers.get('content-type') || 'application/octet-stream';
|
||||
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
|
||||
const contentLength = upstream.headers.get('content-length');
|
||||
const contentRange = upstream.headers.get('content-range');
|
||||
@@ -340,7 +417,14 @@ export async function fetchCctvMediaUpstream(
|
||||
if (downstream?.aborted) controller.abort();
|
||||
else downstream?.addEventListener?.('abort', onDownstreamAbort);
|
||||
try {
|
||||
return await fetchImpl(url, { headers, signal: controller.signal });
|
||||
// Redirects stay on the registered origin, as for frames.
|
||||
const upstream = await fetchWithinHost(
|
||||
url,
|
||||
{ headers, signal: controller.signal },
|
||||
fetchImpl,
|
||||
);
|
||||
if (!upstream) throw new Error('Media redirect refused');
|
||||
return upstream;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
downstream?.removeEventListener?.('abort', onDownstreamAbort);
|
||||
@@ -428,9 +512,40 @@ export async function fetchTxdotSnapshot(
|
||||
/** Redirect hops the frame path will follow, and only within the same host. */
|
||||
const MAX_SAME_HOST_REDIRECTS = 2;
|
||||
|
||||
/**
|
||||
* Directories camera catalogs are pinned to, by host. A redirect on one of
|
||||
* these hosts must stay inside the pinned directory too, so a same-origin
|
||||
* redirect cannot leave it (another bucket on a shared storage host).
|
||||
*/
|
||||
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
|
||||
for (const prefix of [
|
||||
TFL_IMAGE_ORIGIN,
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
TARKTEE_IMAGE_ORIGIN,
|
||||
...WARENDORF_IMAGE_ORIGINS,
|
||||
NSW_IMAGE_ORIGIN,
|
||||
CALGARY_IMAGE_ORIGIN,
|
||||
HK_IMAGE_ORIGIN,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
]) {
|
||||
const host = new URL(prefix).hostname;
|
||||
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
|
||||
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
|
||||
prefix,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Whether a redirect target stays inside its host's pinned directory. */
|
||||
function withinPinnedPrefix(url) {
|
||||
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
|
||||
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a registered frame URL following redirects ONLY within the original
|
||||
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
|
||||
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
|
||||
* on a pinned camera host, within its pinned directory.
|
||||
* Default redirect-following would let an upstream steer a host-pinned
|
||||
* request, and its host-specific headers, to any origin, another port, or a
|
||||
* plaintext downgrade.
|
||||
@@ -471,7 +586,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (next.origin !== origin) return null;
|
||||
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
|
||||
current = next;
|
||||
}
|
||||
return null;
|
||||
@@ -543,9 +658,15 @@ export async function fetchCctvImageFromUpstream(
|
||||
fetchImpl,
|
||||
);
|
||||
if (!upstream) return null;
|
||||
const contentType = upstream.headers.get('content-type') || '';
|
||||
if (!upstream.ok || !contentType.startsWith('image/')) {
|
||||
controller.abort();
|
||||
const contentType = cctvRasterImageType(
|
||||
upstream.headers.get('content-type'),
|
||||
);
|
||||
if (!upstream.ok || !contentType) {
|
||||
try {
|
||||
await upstream.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const body = await readCappedResponseBytes(upstream, maxBytes);
|
||||
|
||||
@@ -659,3 +659,38 @@ export function isLikelyTorontoCoordinate(lat, lon) {
|
||||
lon <= -79.1
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A camera URL under a pinned directory, by parsed URL rather than string
|
||||
* prefix: the same scheme and exact host as `prefix`, no credentials, the
|
||||
* default port, and a normalized path inside the prefix's directory. Dot
|
||||
* segments (including encoded ones) and backslashes are resolved before the
|
||||
* path is compared, and an encoded slash or backslash is refused.
|
||||
*
|
||||
* @param {string} raw - Camera URL from a catalog or redirect.
|
||||
* @param {string} prefix - Pinned origin and directory, ending in `/`.
|
||||
* @returns {string|null} The normalized URL, or null when it is not pinned.
|
||||
*/
|
||||
export function pinnedCameraUrl(raw, prefix) {
|
||||
let url;
|
||||
let pin;
|
||||
try {
|
||||
url = new URL(String(raw ?? '').trim());
|
||||
pin = new URL(prefix);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
url.protocol !== pin.protocol ||
|
||||
url.hostname !== pin.hostname ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.port ||
|
||||
!url.pathname.startsWith(
|
||||
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
|
||||
) ||
|
||||
/%2f|%5c/i.test(url.pathname)
|
||||
)
|
||||
return null;
|
||||
return url.href;
|
||||
}
|
||||
@@ -149,6 +149,7 @@ import {
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
prioritizeSources,
|
||||
pinnedCameraUrl,
|
||||
} from './normalize.js';
|
||||
import { directionToHeading } from '../../../src/data/directionText.js';
|
||||
import {
|
||||
@@ -310,9 +311,12 @@ export async function loadCaltransSourcesFromOpenData() {
|
||||
const lon = toFiniteNumber(loc.longitude);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
|
||||
const imageUrl = String(cctv.imageData?.static?.currentImageURL || '');
|
||||
// Official-host pin (see JSDoc). Also drops records with no still image.
|
||||
if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue;
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
cctv.imageData?.static?.currentImageURL,
|
||||
'https://cwwp2.dot.ca.gov/',
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
|
||||
const locationName = String(loc.locationName || '').trim();
|
||||
// Leading token of locationName is the stable camera code ("TV102 -- I-580 : …").
|
||||
@@ -417,8 +421,8 @@ export async function loadTflSourcesFromOpenData() {
|
||||
const lat = toFiniteNumber(place?.lat);
|
||||
const lon = toFiniteNumber(place?.lon);
|
||||
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
|
||||
const imageUrl = String(props.imageUrl || '');
|
||||
if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin
|
||||
const imageUrl = pinnedCameraUrl(props.imageUrl, TFL_IMAGE_ORIGIN);
|
||||
if (!imageUrl) continue; // official-bucket pin
|
||||
|
||||
// "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id).
|
||||
const rawId = String(place?.id || '').replace(/^JamCams_/, '');
|
||||
@@ -1127,13 +1131,15 @@ export function loadTallinnSourcesFromCatalog({
|
||||
// Rough Estonia/Tallinn metro sanity (allows nearby suburbs already in the pack).
|
||||
if (lat < 59.2 || lat > 59.7 || lon < 24.3 || lon > 25.4) continue;
|
||||
|
||||
const imageUrl =
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
typeof item.url === 'string'
|
||||
? item.url.trim()
|
||||
? item.url
|
||||
: typeof item.snapshotUrl === 'string'
|
||||
? item.snapshotUrl.trim()
|
||||
: '';
|
||||
if (!imageUrl.startsWith(TALLINN_IMAGE_ORIGIN)) continue;
|
||||
? item.snapshotUrl
|
||||
: '',
|
||||
TALLINN_IMAGE_ORIGIN,
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
|
||||
const extractedHeading = toFiniteNumber(item.headingDeg, NaN);
|
||||
const hasHeading = Number.isFinite(extractedHeading);
|
||||
@@ -1235,8 +1241,8 @@ export function parseTarkteeDatexImages(xml) {
|
||||
body,
|
||||
);
|
||||
if (!refMatch || !urlMatch) continue;
|
||||
const url = urlMatch[1].trim();
|
||||
if (!url.startsWith(TARKTEE_IMAGE_ORIGIN)) continue;
|
||||
const url = pinnedCameraUrl(urlMatch[1], TARKTEE_IMAGE_ORIGIN);
|
||||
if (!url) continue;
|
||||
out.set(refMatch[1], url);
|
||||
}
|
||||
return out;
|
||||
@@ -1422,14 +1428,16 @@ export function loadWarendorfSourcesFromCatalog({
|
||||
for (const item of rows) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const id = typeof item.id === 'string' ? item.id.trim() : '';
|
||||
const url =
|
||||
const raw =
|
||||
typeof item.url === 'string'
|
||||
? item.url.trim()
|
||||
? item.url
|
||||
: typeof item.snapshotUrl === 'string'
|
||||
? item.snapshotUrl.trim()
|
||||
? item.snapshotUrl
|
||||
: '';
|
||||
if (!id || !WARENDORF_IMAGE_ORIGINS.some((o) => url.startsWith(o)))
|
||||
continue;
|
||||
const url = WARENDORF_IMAGE_ORIGINS.map((o) =>
|
||||
pinnedCameraUrl(raw, o),
|
||||
).find(Boolean);
|
||||
if (!id || !url) continue;
|
||||
const lat = typeof item.lat === 'number' ? item.lat : NaN;
|
||||
const lon = typeof item.lon === 'number' ? item.lon : NaN;
|
||||
if (!isPlausibleLatLon(lat, lon)) continue;
|
||||
@@ -1483,8 +1491,8 @@ export function nswCameraToSource(feature) {
|
||||
const lat = typeof coords?.[1] === 'number' ? coords[1] : NaN;
|
||||
if (!isLikelyNswCoordinate(lat, lon)) return null;
|
||||
const props = feature?.properties || {};
|
||||
const url = String(props.href || '').trim();
|
||||
if (!url.startsWith(NSW_IMAGE_ORIGIN)) return null;
|
||||
const url = pinnedCameraUrl(props.href, NSW_IMAGE_ORIGIN);
|
||||
if (!url) return null;
|
||||
// "N-E" -> "NE" for the compass lookup.
|
||||
const direction = String(props.direction || '')
|
||||
.trim()
|
||||
@@ -1578,8 +1586,7 @@ export function normalizeCalgaryImageUrl(raw) {
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
|
||||
parsed.protocol = 'https:';
|
||||
const upgraded = parsed.toString();
|
||||
return upgraded.startsWith(CALGARY_IMAGE_ORIGIN) ? upgraded : null;
|
||||
return pinnedCameraUrl(parsed.toString(), CALGARY_IMAGE_ORIGIN);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2085,8 +2092,7 @@ export function normalizeHkImageUrl(raw) {
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
|
||||
parsed.protocol = 'https:';
|
||||
const upgraded = parsed.toString();
|
||||
return upgraded.startsWith(HK_IMAGE_ORIGIN) ? upgraded : null;
|
||||
return pinnedCameraUrl(parsed.toString(), HK_IMAGE_ORIGIN);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2423,10 +2429,16 @@ export async function loadKingCountySourcesFromOpenData() {
|
||||
// county image origin, so they are skipped.
|
||||
if (owner === 'WSDOT') continue;
|
||||
// The handler is served over HTTPS; the feed still writes http://.
|
||||
const imageUrl = String(attrs.ImageURL || '')
|
||||
.trim()
|
||||
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
|
||||
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
|
||||
const imageUrl = pinnedCameraUrl(
|
||||
String(attrs.ImageURL || '')
|
||||
.trim()
|
||||
.replace(
|
||||
/^http:\/\/info\.kingcounty\.gov\//,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
),
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
);
|
||||
if (!imageUrl) continue;
|
||||
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
|
||||
? row.geometry.coordinates
|
||||
: [];
|
||||
@@ -2586,8 +2598,8 @@ export async function loadWsdotSourcesFromOpenData() {
|
||||
const attrs = row?.attributes;
|
||||
const rawId = attrs?.CameraID;
|
||||
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
|
||||
const imageUrl = String(attrs.ImageURL || '').trim();
|
||||
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
|
||||
const imageUrl = pinnedCameraUrl(attrs.ImageURL, WSDOT_IMAGE_ORIGIN);
|
||||
if (!imageUrl) continue;
|
||||
const x = row?.geometry?.x;
|
||||
const y = row?.geometry?.y;
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
|
||||
|
||||
@@ -236,13 +236,26 @@ export function gbfsProxy({ ratePerMinute = GBFS_RATE_PER_MIN } = {}) {
|
||||
}
|
||||
|
||||
const upstream = await fetchGbfsUpstream(upstreamUrl.toString());
|
||||
// Only JSON is relayed, re-serialized under a fixed type: neither the
|
||||
// upstream's text nor its declared type reaches the browser.
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(upstream.body);
|
||||
} catch {
|
||||
res.writeHead(502, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'GBFS upstream returned non-JSON' }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(upstream.status, {
|
||||
'Content-Type': upstream.contentType,
|
||||
'Content-Type': 'application/json; charset=utf-8',
|
||||
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
|
||||
'X-GBFS-Upstream': upstreamUrl.hostname,
|
||||
'X-GBFS-Cache': 'MISS',
|
||||
});
|
||||
res.end(upstream.body);
|
||||
res.end(JSON.stringify(payload));
|
||||
} catch (error) {
|
||||
if (error?.name === 'AbortError') {
|
||||
res.writeHead(504, {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
import { readRequestBodyCapped } from './common/request.js';
|
||||
import { admitSameSite } from './common/same-site.js';
|
||||
import {
|
||||
OVERPASS_MAX_BODY_BYTES,
|
||||
OVERPASS_MAX_CONCURRENT,
|
||||
@@ -27,6 +28,14 @@ const _overpassInFlight = new Map();
|
||||
|
||||
let _overpassConcurrent = 0;
|
||||
|
||||
// Bound body-reading and validation work, including malformed requests and
|
||||
// cache hits. Keep the tighter upstream budget below independent of this gate.
|
||||
const _overpassRequestLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 300,
|
||||
globalMax: 900,
|
||||
});
|
||||
|
||||
const _overpassRateLimiter = makeRateLimiter({
|
||||
windowMs: 60_000,
|
||||
max: 90,
|
||||
@@ -94,6 +103,17 @@ function overpassProxy({ routing = {} } = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (!_overpassRequestLimiter(clientKey(req))) {
|
||||
res.writeHead(429, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
'Retry-After': '60',
|
||||
});
|
||||
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect POST body with a hard byte cap (Overpass QL queries are small)
|
||||
let body;
|
||||
try {
|
||||
|
||||
@@ -18,6 +18,29 @@ function isOverpassBoundaryQuery(cacheKey) {
|
||||
return /is_in\s*\(|\bpivot\b/i.test(String(cacheKey || ''));
|
||||
}
|
||||
|
||||
// Input has already had quoted literals and comments removed. Consume each
|
||||
// character once: a regex that searches for a closing bracket from every `[` is
|
||||
// quadratic on an unterminated filter. Nested or unmatched brackets are invalid.
|
||||
function stripTagFilters(text) {
|
||||
const parts = [];
|
||||
let start = 0;
|
||||
let inFilter = false;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
if (text[i] === '[') {
|
||||
if (inFilter) return null;
|
||||
parts.push(text.slice(start, i), ' ');
|
||||
inFilter = true;
|
||||
} else if (text[i] === ']') {
|
||||
if (!inFilter) return null;
|
||||
inFilter = false;
|
||||
start = i + 1;
|
||||
}
|
||||
}
|
||||
if (inFilter) return null;
|
||||
parts.push(text.slice(start));
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate + clamp an Overpass form body. Defends the generic proxy against
|
||||
* planet-scale abuse: requires exactly one `data` query in which EVERY element
|
||||
@@ -50,6 +73,10 @@ function sanitizeOverpassBody(rawBody) {
|
||||
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
|
||||
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
|
||||
const stripped = stripOverpassNoise(data);
|
||||
const withoutTags = stripTagFilters(stripped);
|
||||
if (withoutTags === null) {
|
||||
return { ok: false, error: 'Malformed Overpass tag filter' };
|
||||
}
|
||||
|
||||
// Reject oversized radii in EVERY around form — point `around:r,lat,lon` AND the
|
||||
// input-set form `around.set:r` — and parse the full numeric token so scientific
|
||||
@@ -93,20 +120,19 @@ function sanitizeOverpassBody(rawBody) {
|
||||
// statement. So `way[...]->.a` (global assigned to a set) is rejected, while the
|
||||
// app's `is_in(...)->.a; area.a[...]` and `area(id)->.x; rel(pivot.x)` validate.
|
||||
const boundedSets = new Set();
|
||||
for (let stmt of stripped.split(';')) {
|
||||
for (let stmt of withoutTags.split(';')) {
|
||||
stmt = stmt.trim();
|
||||
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
|
||||
|
||||
// Strip output-set assignments (NOT input bounds), then strip bracket tag
|
||||
// filters so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) can never be
|
||||
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
|
||||
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
|
||||
// misread as a spatial bound. Bounds live in (...) / function calls / set
|
||||
// refs, never inside [...], so the probe loses nothing real.
|
||||
const outSets = [];
|
||||
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
|
||||
outSets.push(name);
|
||||
return ' ';
|
||||
});
|
||||
const probe = body.replace(/\[[^\]]*\]/g, ' ');
|
||||
|
||||
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
|
||||
// between the selector and the area filter (way["highway"](area.a)) can't hide
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { Readable } from 'node:stream';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { overpassProxy } from '../overpass.js';
|
||||
import { sanitizeOverpassBody } from './query.js';
|
||||
|
||||
const form = (query) => `data=${encodeURIComponent(query)}`;
|
||||
|
||||
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
|
||||
for (const query of [
|
||||
'node[',
|
||||
'node]',
|
||||
'node[[name]](around:20,1,1);out;',
|
||||
'node[name=around:20];out;',
|
||||
'node[is_in];out;',
|
||||
'node[name="(around:20,1,1)"];out;',
|
||||
'node(around:20,1,1);way[name];out;',
|
||||
'way[highway](area.a);out;',
|
||||
'node[->.a](around:20,1,1);way.a;out;',
|
||||
]) {
|
||||
assert.equal(sanitizeOverpassBody(form(query)).ok, false, query);
|
||||
}
|
||||
});
|
||||
|
||||
test('bounded queries preserve quoted brackets, comments, multiple tags and timeout clamping', () => {
|
||||
for (const query of [
|
||||
'[out:json][timeout:999];node[name="[Town]; ]"](around:20,1,1);out;',
|
||||
'/* [ */way[highway][name](30.1,-97.9,30.5,-97.5);out geom;',
|
||||
'is_in(30.2,-97.7)->.a;area.a[boundary="administrative"];out tags;',
|
||||
'area(3600039152)->.x;rel(pivot.x);out geom;',
|
||||
]) {
|
||||
const result = sanitizeOverpassBody(form(query));
|
||||
assert.equal(result.ok, true, query);
|
||||
assert.equal(
|
||||
new URLSearchParams(result.body).get('data'),
|
||||
query.replace('[timeout:999]', '[timeout:30]'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
|
||||
// A child deadline can stop synchronous regex backtracking; a test timeout in
|
||||
// the same event loop cannot. The old parser takes several seconds for this.
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
'--input-type=module',
|
||||
'-e',
|
||||
`import { sanitizeOverpassBody } from ${JSON.stringify(new URL('./query.js', import.meta.url).href)};
|
||||
for (let i = 0; i < 20; i++) {
|
||||
if (sanitizeOverpassBody('data=node' + '['.repeat(24000)).ok) process.exit(1);
|
||||
}`,
|
||||
],
|
||||
{ timeout: 2000, encoding: 'utf8' },
|
||||
);
|
||||
assert.ifError(result.error);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
});
|
||||
|
||||
function handler(hook = 'configureServer') {
|
||||
let route;
|
||||
overpassProxy()[hook]({
|
||||
middlewares: {
|
||||
use(path, fn) {
|
||||
if (path === '/api/overpass') route = fn;
|
||||
},
|
||||
},
|
||||
});
|
||||
return route;
|
||||
}
|
||||
|
||||
async function call(
|
||||
route,
|
||||
{
|
||||
headers = {},
|
||||
peer = '127.0.0.1',
|
||||
body = 'data=node[',
|
||||
forbidRead = false,
|
||||
method = 'POST',
|
||||
url = '/',
|
||||
} = {},
|
||||
) {
|
||||
let reads = 0;
|
||||
const req = new Readable({
|
||||
read() {
|
||||
reads++;
|
||||
assert.equal(forbidRead, false, 'refused request must not read its body');
|
||||
this.push(Buffer.from(body));
|
||||
this.push(null);
|
||||
},
|
||||
});
|
||||
Object.assign(req, {
|
||||
method,
|
||||
url,
|
||||
headers: { host: 'localhost:4173', ...headers },
|
||||
socket: { remoteAddress: peer },
|
||||
});
|
||||
const res = {
|
||||
statusCode: 200,
|
||||
headers: {},
|
||||
setHeader(key, value) {
|
||||
this.headers[key.toLowerCase()] = value;
|
||||
},
|
||||
writeHead(status, values) {
|
||||
this.statusCode = status;
|
||||
for (const [key, value] of Object.entries(values))
|
||||
this.setHeader(key, value);
|
||||
},
|
||||
end(value) {
|
||||
this.body = value;
|
||||
},
|
||||
};
|
||||
await route(req, res);
|
||||
return { ...res, reads };
|
||||
}
|
||||
|
||||
for (const hook of ['configureServer', 'configurePreviewServer']) {
|
||||
test(`${hook}: cross-site requests are rejected before consuming a body or fetching`, async (t) => {
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler(hook);
|
||||
for (const headers of [
|
||||
{ origin: 'https://foreign.example', 'content-type': 'text/plain' },
|
||||
{ 'sec-fetch-site': 'cross-site' },
|
||||
{ 'sec-fetch-site': 'same-site' },
|
||||
{ origin: 'null' },
|
||||
{ 'x-forwarded-for': '127.0.0.1' },
|
||||
]) {
|
||||
const response = await call(route, { headers, forbidRead: true });
|
||||
assert.equal(response.statusCode, 403);
|
||||
assert.equal(response.reads, 0);
|
||||
assert.equal(response.headers['cache-control'], 'no-store');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
|
||||
const route = handler();
|
||||
for (const headers of [
|
||||
{},
|
||||
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
|
||||
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
|
||||
]) {
|
||||
const response = await call(route, { headers });
|
||||
assert.equal(response.statusCode, 400);
|
||||
assert.equal(response.reads, 1);
|
||||
assert.match(response.body, /Malformed Overpass tag filter/);
|
||||
}
|
||||
const response = await call(route, {
|
||||
method: 'GET',
|
||||
url: '/status',
|
||||
forbidRead: true,
|
||||
});
|
||||
assert.equal(response.statusCode, 200);
|
||||
assert.equal(typeof JSON.parse(response.body).configured, 'boolean');
|
||||
});
|
||||
|
||||
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
|
||||
let now = Date.now() + 120000;
|
||||
t.mock.method(Date, 'now', () => now);
|
||||
t.mock.method(globalThis, 'fetch', () =>
|
||||
assert.fail('unexpected upstream fetch'),
|
||||
);
|
||||
const route = handler();
|
||||
const peer = 'admission-limit-test';
|
||||
for (let i = 0; i < 300; i++) {
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
}
|
||||
const denied = await call(route, { peer, forbidRead: true });
|
||||
assert.equal(denied.statusCode, 429);
|
||||
assert.equal(denied.reads, 0);
|
||||
assert.equal(denied.headers['retry-after'], '60');
|
||||
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
|
||||
now += 60001;
|
||||
assert.equal((await call(route, { peer })).statusCode, 400);
|
||||
});
|
||||
Reference in new issue
Block a user