Merge remote-tracking branch 'origin/main' into fix/1273

# Conflicts:
#	docs/CURRENT-STATE.md
#	src/tooling/liveProviders.test.mjs
This commit is contained in:
Said Turksever committed 2026-10-23 12:09:32 +01:00
commit 98af799cb4
325 files changed
+20500 -1424

No files matched your search

+64 -47
View File
@@ -1,4 +1,7 @@
import { readResponseTextCapped } from '../common/http.js';
import {
coalesceProxyRequest,
readResponseTextCapped,
} from '../common/http.js';
/**
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
@@ -19,6 +22,7 @@ import { readResponseTextCapped } from '../common/http.js';
export function adsbLolProxy() {
/** @type {string|null} Cached upstream JSON body. */
let _cache = null;
const inFlight = new Map();
/** @type {number} Epoch-ms when the cache was populated. */
let _cacheAt = 0;
/** @type {number} Epoch-ms until which upstream is not contacted. */
@@ -104,60 +108,73 @@ export function adsbLolProxy() {
);
return;
}
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const refresh = coalesceProxyRequest(inFlight, 'global', async () => {
const result = (status, body, cacheStatus, extra = {}) => ({
status,
body,
cacheStatus,
extra,
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
return result(200, body, 'MISS');
}
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
return result(200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
});
}
}
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
_cooldownStatus = 0;
serve(res, 200, body, 'MISS');
return;
}
if (upstream.status === 429 || upstream.status >= 500) {
const failedAt = Date.now();
_cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
_cooldownStatus = upstream.status;
console.warn(
`[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
return result(
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
);
if (_cache) {
// Do not wait for a failing response body before serving usable data.
upstream.body?.cancel().catch(() => {});
serve(res, 200, _cache, 'STALE', {
'X-ADS-B-Upstream-Status': String(upstream.status),
'X-ADS-B-Retry-After-Seconds': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
});
return;
}
}
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
});
const response = await refresh.promise;
serve(
res,
upstream.status,
body,
'MISS',
_cooldownUntil > Date.now()
? {
'Retry-After': String(
Math.ceil((_cooldownUntil - Date.now()) / 1000),
),
}
: {},
response.status,
response.body,
response.cacheStatus,
response.extra,
);
} catch (e) {
console.error('[adsb.lol Proxy]', e.message);
+141 -111
View File
@@ -67,15 +67,24 @@ async function fetchOpenSkyStates(headers) {
try {
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
if (upstream.status === 429) {
if (upstream.status === 429 || upstream.status >= 500) {
// Start the cooldown from the headers and drop the rejected body
// unread, so a stalled or oversized 429 body cannot skip it.
_openskyCooldownUntil =
Date.now() + openSkyRateLimitCooldownMs(upstream);
Date.now() +
(upstream.status === 429
? openSkyRateLimitCooldownMs(upstream)
: 15000);
_openskyCooldownStatus = upstream.status;
void upstream.body?.cancel().catch(() => {});
return {
upstream,
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
body: JSON.stringify({
error:
upstream.status === 429
? 'OpenSky rate limited.'
: 'OpenSky upstream unavailable.',
}),
};
}
const body = await readResponseTextCapped(
@@ -109,6 +118,8 @@ async function fetchOpenSkyStates(headers) {
let _openskyTtlMs = OPENSKY_CACHE_MS;
/** @type {number} Epoch-ms before which no upstream fetch is attempted. */
let _openskyCooldownUntil = 0;
let _openskyCooldownStatus = 429;
const _openskySnapshotInFlight = new Map();
/**
* Picks the cache TTL from the remaining daily credit budget.
* Client polls every 30 s, so tiers ≤30 s cost the same 480 credits/h; the
@@ -460,7 +471,9 @@ export function openSkyProxy() {
requestedMode: cachedMeta.requestedMode || requestedMode,
usedMode: cachedMeta.usedMode || 'unknown',
reason: isStale
? 'rate_limited_serving_stale'
? _openskyCooldownStatus === 429
? 'rate_limited_serving_stale'
: `upstream_http_${_openskyCooldownStatus}_serving_stale`
: cachedMeta.reason || 'cached',
staleSeconds: isStale
? (now - _openskyCacheTime) / 1000
@@ -487,80 +500,140 @@ export function openSkyProxy() {
)
return;
res.writeHead(
429,
_openskyCooldownStatus,
buildOpenSkyHeaders({
cacheStatus: 'COOLDOWN',
requestedMode,
usedMode: 'none',
reason: 'rate_limited',
reason:
_openskyCooldownStatus === 429
? 'rate_limited'
: `upstream_http_${_openskyCooldownStatus}`,
retryAfterSeconds: (_openskyCooldownUntil - now) / 1000,
}),
);
res.end(
JSON.stringify({
error: 'OpenSky rate limited; proxy cooling down.',
error: 'OpenSky upstream unavailable; proxy cooling down.',
}),
);
return;
}
const basicUser = process.env.OPENSKY_USERNAME || '';
const basicPass = process.env.OPENSKY_PASSWORD || '';
const hasBasicCreds = Boolean(basicUser && basicPass);
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
const refresh = coalesceProxyRequest(
_openskySnapshotInFlight,
'global',
async () => {
const basicUser = process.env.OPENSKY_USERNAME || '';
const basicPass = process.env.OPENSKY_PASSWORD || '';
const hasBasicCreds = Boolean(basicUser && basicPass);
const headers = { Accept: 'application/json' };
let usedMode = 'anon';
let reason = 'forced_anonymous';
if (requestedMode === 'basic') {
if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'basic_credentials';
} else {
reason = 'missing_basic_creds';
}
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'oauth_unavailable_fallback_basic';
} else {
reason = 'missing_oauth_and_basic_creds';
}
}
if (requestedMode === 'basic') {
if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'basic_credentials';
} else {
reason = 'missing_basic_creds';
}
} else if (requestedMode === 'oauth') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else {
reason = 'oauth_invalid_or_missing';
}
} else if (requestedMode === 'auto') {
const token = await getOpenSkyToken();
if (token) {
headers.Authorization = `Bearer ${token}`;
usedMode = 'oauth';
reason = 'oauth_token';
} else if (hasBasicCreds) {
headers.Authorization = `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`;
usedMode = 'basic';
reason = 'oauth_unavailable_fallback_basic';
} else {
reason = 'missing_oauth_and_basic_creds';
}
}
let { upstream, body } = await fetchOpenSkyStates(headers);
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
if (
(upstream.status === 401 || upstream.status === 403) &&
requestedMode === 'auto' &&
usedMode === 'oauth' &&
hasBasicCreds
) {
const retryHeaders = {
Accept: 'application/json',
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
};
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
usedMode = 'basic';
reason = 'oauth_rejected_fallback_basic';
}
let { upstream, body } = await fetchOpenSkyStates(headers);
// Auto-mode fallback: if OAuth was rejected, retry with Basic credentials
if (
(upstream.status === 401 || upstream.status === 403) &&
requestedMode === 'auto' &&
usedMode === 'oauth' &&
hasBasicCreds
) {
const retryHeaders = {
Accept: 'application/json',
Authorization: `Basic ${Buffer.from(`${basicUser}:${basicPass}`).toString('base64')}`,
};
({ upstream, body } = await fetchOpenSkyStates(retryHeaders));
usedMode = 'basic';
reason = 'oauth_rejected_fallback_basic';
}
const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null;
const sourceEpochMs = upstream.ok
? openSkySourceEpochMs(body)
: null;
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'basic' &&
reason === 'basic_credentials'
) {
reason = 'basic_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
return {
upstream,
body,
usedMode,
reason,
hasBasicCreds,
sourceEpochMs,
};
},
);
let { upstream, body, usedMode, reason, hasBasicCreds, sourceEpochMs } =
await refresh.promise;
if (
upstream.ok &&
openSkySourceIsStale(sourceEpochMs, now) &&
@@ -571,20 +644,16 @@ export function openSkyProxy() {
'opensky_snapshot_stale_regional_fallback',
))
) {
// Keep the last global snapshot available as a fail-soft cache,
// but do not label or render it as a fresh live result.
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = { requestedMode, usedMode, reason };
return;
}
if (upstream.status === 429) {
reason = 'rate_limited';
if (upstream.status === 429 || upstream.status >= 500) {
reason =
upstream.status === 429
? 'rate_limited'
: `upstream_http_${upstream.status}`;
// Credit governor: fetchOpenSkyStates already started the cooldown
// from OpenSky's retry-after — no upstream attempts until then.
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
const cooldownMs = Math.max(0, _openskyCooldownUntil - Date.now());
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
@@ -594,7 +663,7 @@ export function openSkyProxy() {
cacheStatus: 'STALE',
requestedMode,
usedMode,
reason: 'rate_limited_serving_stale',
reason: `${reason}_serving_stale`,
staleSeconds: (now - _openskyCacheTime) / 1000,
retryAfterSeconds: cooldownMs / 1000,
}),
@@ -652,45 +721,6 @@ export function openSkyProxy() {
}
}
// Refine the reason string to reflect the actual outcome
if (upstream.ok && reason === 'forced_anonymous') {
reason = 'anonymous_ok';
} else if (
upstream.ok &&
usedMode === 'basic' &&
reason === 'basic_credentials'
) {
reason = 'basic_ok';
} else if (
upstream.ok &&
usedMode === 'oauth' &&
reason === 'oauth_token'
) {
reason = 'oauth_ok';
}
// Only cache successful responses — error responses (401/403/429/5xx)
// should not be served from cache on subsequent requests
if (upstream.ok) {
_openskyCacheBody = body;
_openskyCacheStatus = upstream.status;
_openskyCacheTime = now;
_openskyCacheSourceEpochMs = sourceEpochMs;
_openskyCacheMeta = {
requestedMode,
usedMode,
reason,
};
// Credit governor: adapt the cache TTL to the remaining daily
// budget so a continuously-open app stretches its polls instead of
// exhausting the quota mid-day. Success also clears any cooldown.
const remaining = Number(
upstream.headers.get('x-rate-limit-remaining'),
);
_openskyTtlMs = openskyAdaptiveTtlMs(remaining);
_openskyCooldownUntil = 0;
}
res.writeHead(
upstream.status,
buildOpenSkyHeaders({
+64 -34
View File
@@ -1,11 +1,9 @@
import { createCctvCatalog } from './cctv/catalog.js';
import {
normalizeFeedType,
isVideoFeedType,
toFiniteNumber,
} from './cctv/normalize.js';
import { normalizeFeedType, isVideoFeedType } from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
cctvLiveMediaType,
cctvRasterImageType,
proxyMediaResponse,
fetchCctvImageFromUpstream,
fetchTxdotSnapshot,
@@ -17,6 +15,8 @@ import {
CCTV_MAX_SOURCES_CEILING,
} from './cctv/constants.js';
import { sanitizeCctvRangeHeader } from './cctv/range.js';
import { admitSameSiteKeyless } from './common/same-site.js';
import { PROXY_SIGNALS } from '../../src/localRequestGate.mjs';
import { createHlsPuller } from './cctv/stream.js';
import { googleServerApiKey } from './places/google-key.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
@@ -116,8 +116,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' },
signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS),
});
const svType = svResp.headers.get('content-type') || '';
if (!svResp.ok || !svType.startsWith('image/')) return null;
const svType = cctvRasterImageType(svResp.headers.get('content-type'));
if (!svResp.ok || !svType) {
try {
await svResp.body?.cancel();
} catch {
/* no-op */
}
return null;
}
return {
ok: true,
@@ -135,11 +142,15 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
});
server.middlewares.use('/api/cctv', async (req, res) => {
try {
const url = new URL(req.url || '/', 'http://localhost');
// Refuse cross-site requests before any catalog or upstream work. The
// keyless gate keeps the routes reachable behind a reverse proxy; the
// key-bearing Street View fallback below has its own check.
if (admitSameSiteKeyless(req, res)) return;
const sources = await getCctvSources();
const sourceById = new Map(
sources.map((source) => [source.id, source]),
);
const url = new URL(req.url || '/', 'http://localhost');
if (url.pathname === '/sources') {
const body = {
@@ -217,8 +228,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const mediaUrl = source?.url || '';
const feedType = normalizeFeedType(source?.feedType || 'image');
// Live feeds only. A still camera is served by /frame, whose fetch
// keeps redirects on the registered origin, accepts only image/* and
// caps the body; relaying it here would bypass all three.
// keeps redirects on the registered origin, accepts only raster
// images and caps the body; relaying it here would bypass all three.
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
res.writeHead(404, {
'Content-Type': 'application/json',
@@ -382,7 +393,6 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
}
return;
}
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok) {
try {
await upstream.body?.cancel();
@@ -407,31 +417,42 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return;
}
if (
isVideoFeedType(feedType) &&
!(
contentType.startsWith('video/') ||
contentType.includes('mpegurl')
)
) {
// The relayed type is the feed kind's own, checked before the
// camera counts as healthy and before any byte is relayed.
const contentType = cctvLiveMediaType(
feedType,
upstream.headers.get('content-type'),
);
if (!contentType) {
try {
await upstream.body?.cancel();
} catch {
/* already closed */
}
setHealth(cameraId, {
status: 'degraded',
sourceKind: 'upstream',
label: source?.provider || 'Configured source',
message: `Unexpected media type ${contentType || 'unknown'}`,
message: 'Unexpected media type',
});
} else {
setHealth(cameraId, {
status: 'ok',
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
label: source?.provider || 'Configured source',
message: isVideoFeedType(feedType)
? 'Live stream connected'
: 'Snapshot feed connected',
res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'Media proxy failed' }));
return;
}
setHealth(cameraId, {
status: 'ok',
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
label: source?.provider || 'Configured source',
message: isVideoFeedType(feedType)
? 'Live stream connected'
: 'Snapshot feed connected',
});
await proxyMediaResponse(res, upstream, {
contentType,
sourceHeader: isVideoFeedType(feedType)
? 'live-media'
: 'upstream-image',
@@ -524,13 +545,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return;
}
const sv = await streetViewFallback({
lat,
lon,
heading,
fov,
pitch,
});
// The Street View fallback spends the Google key, so a forwarded
// request (whose client this server cannot see) gets the placeholder,
// as if no key were configured. Any forwarding header counts, even an
// empty one.
const forwarded = PROXY_SIGNALS.some(
(name) => req.headers?.[name] !== undefined,
);
const sv = forwarded
? null
: await streetViewFallback({
lat,
lon,
heading,
fov,
pitch,
});
if (sv?.ok) {
setHealth(cameraId, {
status: 'degraded',
+1 -1
View File
@@ -18,7 +18,7 @@ export const CCTV_MAX_SOURCES_CEILING = 5000;
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
const TXDOT_ORIGIN = 'https://its.txdot.gov';
export const TXDOT_CCTV_STATUS_URL = (district) =>
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
+1 -1
View File
@@ -3,7 +3,7 @@ import path from 'node:path';
import { poseHash } from '../../../src/data/cctvFootprint.js';
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
export const DEFAULT_GROUND_HEIGHTS_FILE =
const DEFAULT_GROUND_HEIGHTS_FILE =
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
+131 -10
View File
@@ -1,13 +1,21 @@
import { Readable } from 'node:stream';
import { hashSeed, escapeXml } from './normalize.js';
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
import {
CALGARY_IMAGE_ORIGIN,
CCTV_FRAME_FETCH_TIMEOUT_MS,
CCTV_FRAME_MAX_BODY_BYTES,
CCTV_MEDIA_FETCH_TIMEOUT_MS,
CCTV_MEDIA_IDLE_TIMEOUT_MS,
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
} from './constants.js';
/**
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
@@ -84,7 +92,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
* @param {ReadableStream|NodeJS.ReadableStream|null} body
* @returns {import('stream').Readable|null}
*/
export function toReadable(body) {
function toReadable(body) {
if (!body) return null;
if (typeof body.pipe === 'function') return body;
if (typeof body.getReader === 'function') {
@@ -93,6 +101,73 @@ export function toReadable(body) {
return null;
}
/**
* Split a Content-Type value into its lower-case essence and parameters.
*
* @param {string|null|undefined} value
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
* value is missing or is not a `type/subtype` media type.
*/
function parseMediaType(value) {
const [head, ...rest] = String(value ?? '').split(';');
const essence = head.trim().toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
const params = new Map();
for (const part of rest) {
const eq = part.indexOf('=');
if (eq <= 0) continue;
const name = part.slice(0, eq).trim().toLowerCase();
let paramValue = part.slice(eq + 1).trim();
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
if (name && !params.has(name)) params.set(name, paramValue);
}
return { essence, params };
}
/** Still-image types a camera frame may be served as. */
const CCTV_RASTER_IMAGE_TYPES = new Set([
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'image/avif',
]);
/**
* The canonical type of a raster camera frame, or null for any other type.
*
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvRasterImageType(contentType) {
const essence = parseMediaType(contentType)?.essence;
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
}
/**
* The type to relay a live feed as, or null when the upstream type does not
* match the feed kind. MJPEG keeps its part boundary, without which the body
* cannot be split into frames.
*
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvLiveMediaType(feedType, contentType) {
const parsed = parseMediaType(contentType);
if (!parsed) return null;
if (feedType === 'mp4' || feedType === 'webm')
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
return null;
const boundary = parsed.params.get('boundary') || '';
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
return null;
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
? `multipart/x-mixed-replace; boundary=${boundary}`
: `multipart/x-mixed-replace; boundary="${boundary}"`;
}
/**
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
*
@@ -104,6 +179,8 @@ export function toReadable(body) {
* @param {Response} upstream - fetch() Response object.
* @param {object} [opts]
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
* @param {string} [opts.contentType] - Validated type to send instead of the
* upstream's own.
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
* allowed between upstream chunks before the stream is released. Injectable
* only to keep the deadline unit-testable.
@@ -114,10 +191,10 @@ export async function proxyMediaResponse(
{
sourceHeader = 'upstream',
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
contentType = upstream.headers.get('content-type') ||
'application/octet-stream',
} = {},
) {
const contentType =
upstream.headers.get('content-type') || 'application/octet-stream';
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
const contentLength = upstream.headers.get('content-length');
const contentRange = upstream.headers.get('content-range');
@@ -340,7 +417,14 @@ export async function fetchCctvMediaUpstream(
if (downstream?.aborted) controller.abort();
else downstream?.addEventListener?.('abort', onDownstreamAbort);
try {
return await fetchImpl(url, { headers, signal: controller.signal });
// Redirects stay on the registered origin, as for frames.
const upstream = await fetchWithinHost(
url,
{ headers, signal: controller.signal },
fetchImpl,
);
if (!upstream) throw new Error('Media redirect refused');
return upstream;
} finally {
clearTimeout(timeoutId);
downstream?.removeEventListener?.('abort', onDownstreamAbort);
@@ -428,9 +512,40 @@ export async function fetchTxdotSnapshot(
/** Redirect hops the frame path will follow, and only within the same host. */
const MAX_SAME_HOST_REDIRECTS = 2;
/**
* Directories camera catalogs are pinned to, by host. A redirect on one of
* these hosts must stay inside the pinned directory too, so a same-origin
* redirect cannot leave it (another bucket on a shared storage host).
*/
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
prefix,
]);
}
/** Whether a redirect target stays inside its host's pinned directory. */
function withinPinnedPrefix(url) {
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
}
/**
* Fetch a registered frame URL following redirects ONLY within the original
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
* on a pinned camera host, within its pinned directory.
* Default redirect-following would let an upstream steer a host-pinned
* request, and its host-specific headers, to any origin, another port, or a
* plaintext downgrade.
@@ -471,7 +586,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
} catch {
return null;
}
if (next.origin !== origin) return null;
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
current = next;
}
return null;
@@ -543,9 +658,15 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok || !contentType.startsWith('image/')) {
controller.abort();
const contentType = cctvRasterImageType(
upstream.headers.get('content-type'),
);
if (!upstream.ok || !contentType) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
+38 -3
View File
@@ -64,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
* @param {string} text
* @returns {string}
*/
export function normalizeKey(text) {
function normalizeKey(text) {
return String(text || '')
.trim()
.toLowerCase()
@@ -80,7 +80,7 @@ export function normalizeKey(text) {
* @param {string} value
* @returns {{lat:number, lon:number}}
*/
export function parsePointString(value) {
function parsePointString(value) {
const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
@@ -100,7 +100,7 @@ export function parsePointString(value) {
* @param {string|object|null} value
* @returns {{lat:number, lon:number}}
*/
export function coerceLatLon(value) {
function coerceLatLon(value) {
if (!value) return { lat: NaN, lon: NaN };
if (typeof value === 'string') {
@@ -659,3 +659,38 @@ export function isLikelyTorontoCoordinate(lat, lon) {
lon <= -79.1
);
}
/**
* A camera URL under a pinned directory, by parsed URL rather than string
* prefix: the same scheme and exact host as `prefix`, no credentials, the
* default port, and a normalized path inside the prefix's directory. Dot
* segments (including encoded ones) and backslashes are resolved before the
* path is compared, and an encoded slash or backslash is refused.
*
* @param {string} raw - Camera URL from a catalog or redirect.
* @param {string} prefix - Pinned origin and directory, ending in `/`.
* @returns {string|null} The normalized URL, or null when it is not pinned.
*/
export function pinnedCameraUrl(raw, prefix) {
let url;
let pin;
try {
url = new URL(String(raw ?? '').trim());
pin = new URL(prefix);
} catch {
return null;
}
if (
url.protocol !== pin.protocol ||
url.hostname !== pin.hostname ||
url.username ||
url.password ||
url.port ||
!url.pathname.startsWith(
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
) ||
/%2f|%5c/i.test(url.pathname)
)
return null;
return url.href;
}
+41 -30
View File
@@ -133,7 +133,6 @@ import {
fallbackHeadingFromId,
isLikelyFinlandCoordinate,
fintrafficCameraName,
hashSeed,
isPlausibleLatLon,
isLikelyBcCoordinate,
isLikelyTexasCoordinate,
@@ -149,6 +148,7 @@ import {
cameraDisplayCode,
rowArrayToObject,
prioritizeSources,
pinnedCameraUrl,
} from './normalize.js';
import { directionToHeading } from '../../../src/data/directionText.js';
import {
@@ -310,9 +310,12 @@ export async function loadCaltransSourcesFromOpenData() {
const lon = toFiniteNumber(loc.longitude);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
const imageUrl = String(cctv.imageData?.static?.currentImageURL || '');
// Official-host pin (see JSDoc). Also drops records with no still image.
if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue;
const imageUrl = pinnedCameraUrl(
cctv.imageData?.static?.currentImageURL,
'https://cwwp2.dot.ca.gov/',
);
if (!imageUrl) continue;
const locationName = String(loc.locationName || '').trim();
// Leading token of locationName is the stable camera code ("TV102 -- I-580 : …").
@@ -417,8 +420,8 @@ export async function loadTflSourcesFromOpenData() {
const lat = toFiniteNumber(place?.lat);
const lon = toFiniteNumber(place?.lon);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
const imageUrl = String(props.imageUrl || '');
if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin
const imageUrl = pinnedCameraUrl(props.imageUrl, TFL_IMAGE_ORIGIN);
if (!imageUrl) continue; // official-bucket pin
// "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id).
const rawId = String(place?.id || '').replace(/^JamCams_/, '');
@@ -1127,13 +1130,15 @@ export function loadTallinnSourcesFromCatalog({
// Rough Estonia/Tallinn metro sanity (allows nearby suburbs already in the pack).
if (lat < 59.2 || lat > 59.7 || lon < 24.3 || lon > 25.4) continue;
const imageUrl =
const imageUrl = pinnedCameraUrl(
typeof item.url === 'string'
? item.url.trim()
? item.url
: typeof item.snapshotUrl === 'string'
? item.snapshotUrl.trim()
: '';
if (!imageUrl.startsWith(TALLINN_IMAGE_ORIGIN)) continue;
? item.snapshotUrl
: '',
TALLINN_IMAGE_ORIGIN,
);
if (!imageUrl) continue;
const extractedHeading = toFiniteNumber(item.headingDeg, NaN);
const hasHeading = Number.isFinite(extractedHeading);
@@ -1235,8 +1240,8 @@ export function parseTarkteeDatexImages(xml) {
body,
);
if (!refMatch || !urlMatch) continue;
const url = urlMatch[1].trim();
if (!url.startsWith(TARKTEE_IMAGE_ORIGIN)) continue;
const url = pinnedCameraUrl(urlMatch[1], TARKTEE_IMAGE_ORIGIN);
if (!url) continue;
out.set(refMatch[1], url);
}
return out;
@@ -1422,14 +1427,16 @@ export function loadWarendorfSourcesFromCatalog({
for (const item of rows) {
if (!item || typeof item !== 'object') continue;
const id = typeof item.id === 'string' ? item.id.trim() : '';
const url =
const raw =
typeof item.url === 'string'
? item.url.trim()
? item.url
: typeof item.snapshotUrl === 'string'
? item.snapshotUrl.trim()
? item.snapshotUrl
: '';
if (!id || !WARENDORF_IMAGE_ORIGINS.some((o) => url.startsWith(o)))
continue;
const url = WARENDORF_IMAGE_ORIGINS.map((o) =>
pinnedCameraUrl(raw, o),
).find(Boolean);
if (!id || !url) continue;
const lat = typeof item.lat === 'number' ? item.lat : NaN;
const lon = typeof item.lon === 'number' ? item.lon : NaN;
if (!isPlausibleLatLon(lat, lon)) continue;
@@ -1483,8 +1490,8 @@ export function nswCameraToSource(feature) {
const lat = typeof coords?.[1] === 'number' ? coords[1] : NaN;
if (!isLikelyNswCoordinate(lat, lon)) return null;
const props = feature?.properties || {};
const url = String(props.href || '').trim();
if (!url.startsWith(NSW_IMAGE_ORIGIN)) return null;
const url = pinnedCameraUrl(props.href, NSW_IMAGE_ORIGIN);
if (!url) return null;
// "N-E" -> "NE" for the compass lookup.
const direction = String(props.direction || '')
.trim()
@@ -1578,8 +1585,7 @@ export function normalizeCalgaryImageUrl(raw) {
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
parsed.protocol = 'https:';
const upgraded = parsed.toString();
return upgraded.startsWith(CALGARY_IMAGE_ORIGIN) ? upgraded : null;
return pinnedCameraUrl(parsed.toString(), CALGARY_IMAGE_ORIGIN);
}
/**
@@ -1862,7 +1868,7 @@ export function normalizeCataloniaImageUrl(rawUrl) {
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
* @returns {string}
*/
export function cataloniaCameraId(imageUrl) {
function cataloniaCameraId(imageUrl) {
const parsed = new URL(imageUrl);
const slug = (text) =>
text
@@ -2085,8 +2091,7 @@ export function normalizeHkImageUrl(raw) {
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
parsed.protocol = 'https:';
const upgraded = parsed.toString();
return upgraded.startsWith(HK_IMAGE_ORIGIN) ? upgraded : null;
return pinnedCameraUrl(parsed.toString(), HK_IMAGE_ORIGIN);
}
/**
@@ -2423,10 +2428,16 @@ export async function loadKingCountySourcesFromOpenData() {
// county image origin, so they are skipped.
if (owner === 'WSDOT') continue;
// The handler is served over HTTPS; the feed still writes http://.
const imageUrl = String(attrs.ImageURL || '')
.trim()
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
const imageUrl = pinnedCameraUrl(
String(attrs.ImageURL || '')
.trim()
.replace(
/^http:\/\/info\.kingcounty\.gov\//,
KING_COUNTY_IMAGE_ORIGIN,
),
KING_COUNTY_IMAGE_ORIGIN,
);
if (!imageUrl) continue;
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
? row.geometry.coordinates
: [];
@@ -2586,8 +2597,8 @@ export async function loadWsdotSourcesFromOpenData() {
const attrs = row?.attributes;
const rawId = attrs?.CameraID;
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
const imageUrl = String(attrs.ImageURL || '').trim();
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
const imageUrl = pinnedCameraUrl(attrs.ImageURL, WSDOT_IMAGE_ORIGIN);
if (!imageUrl) continue;
const x = row?.geometry?.x;
const y = row?.geometry?.y;
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
+1 -1
View File
@@ -70,7 +70,7 @@ export async function fetchHlsBytes(
}
}
export function sameOriginHlsUrl(value, base) {
function sameOriginHlsUrl(value, base) {
const url = new URL(value, base);
if (
!['https:', 'http:'].includes(url.protocol) ||
+3 -3
View File
@@ -12,14 +12,14 @@ export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
export const DEFAULT_TAIWAN_MAX_SOURCES = 600;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
/** After a failed download, wait this long before asking again. The catalog
* host does not answer from some networks outside Taiwan, and every attempt
* there holds the shared catalog refresh until the connect timeout. */
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
/** The table is republished daily; a good copy stands in for a failed
* download for up to this long. */
export const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// The table publishes each camera's MJPEG stream. The same camera servers
// answer a single JPEG for the same camera ID on a sibling path, which is what
@@ -50,7 +50,7 @@ const ANCHORS = [
/** Taiwan main island with slack for the coast; the freeway network has no
* offshore-island cameras. */
export function isLikelyTaiwanCoordinate(lat, lon) {
function isLikelyTaiwanCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
+15 -2
View File
@@ -236,13 +236,26 @@ export function gbfsProxy({ ratePerMinute = GBFS_RATE_PER_MIN } = {}) {
}
const upstream = await fetchGbfsUpstream(upstreamUrl.toString());
// Only JSON is relayed, re-serialized under a fixed type: neither the
// upstream's text nor its declared type reaches the browser.
let payload;
try {
payload = JSON.parse(upstream.body);
} catch {
res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'GBFS upstream returned non-JSON' }));
return;
}
res.writeHead(upstream.status, {
'Content-Type': upstream.contentType,
'Content-Type': 'application/json; charset=utf-8',
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
'X-GBFS-Upstream': upstreamUrl.hostname,
'X-GBFS-Cache': 'MISS',
});
res.end(upstream.body);
res.end(JSON.stringify(payload));
} catch (error) {
if (error?.name === 'AbortError') {
res.writeHead(504, {
+9 -9
View File
@@ -27,13 +27,13 @@ import { normalizeDump1090Aircraft } from '../../src/sources/adsbRecords.js';
*/
export const LOCAL_RECEIVERS_ROUTE = '/api/local-receivers/aircraft';
export const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
export const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
export const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
export const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
export const LOCAL_RECEIVER_CACHE_MS = 1_000;
const LOCAL_RECEIVER_FEEDS_ENV = 'LOCAL_RECEIVER_FEEDS';
const LOCAL_RECEIVER_BANDS = Object.freeze(['1090', '978']);
const LOCAL_RECEIVER_TIMEOUT_MS = 2_000;
const LOCAL_RECEIVER_MAX_BODY_BYTES = 2 * 1024 * 1024;
const LOCAL_RECEIVER_CACHE_MS = 1_000;
/** A feed whose own `now` is older than this is reported stale. */
export const LOCAL_RECEIVER_STALE_MS = 10_000;
const LOCAL_RECEIVER_STALE_MS = 10_000;
const MAX_FEEDS = 8;
const LOG_PREFIX = '[local-receivers]';
@@ -139,7 +139,7 @@ class FeedError extends Error {
* @param {string} address Resolved IP address.
* @returns {boolean}
*/
export function isLocalReceiverAddress(address) {
function isLocalReceiverAddress(address) {
const value = String(address || '').toLowerCase();
if (value === '::1') return true;
const mapped = /^::ffff:(\d{1,3}(?:\.\d{1,3}){3})$/.exec(value);
@@ -152,7 +152,7 @@ export function isLocalReceiverAddress(address) {
* @param {typeof dnsLookup} lookupImpl `dns.promises.lookup`-compatible.
* @returns {Promise<Array<{address:string, family:number}>>}
*/
export async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
async function resolveLocalReceiverAddresses(hostname, lookupImpl) {
let resolved;
try {
resolved = await lookupImpl(hostname, { all: true, verbatim: true });
@@ -274,7 +274,7 @@ function fetchLocalReceiverFeed(url, options) {
* @param {object} options
* @returns {Promise<object>} Parsed JSON document.
*/
export async function fetchLocalReceiverDocument(
async function fetchLocalReceiverDocument(
url,
{
fetchImpl = fetchLocalReceiverFeed,
+1 -1
View File
@@ -42,7 +42,7 @@ export class TileUpstreamError extends Error {
}
/** Seconds a Retry-After header asks for (an integer or an HTTP date), else null. */
export function retryAfterSeconds(header, now = Date.now()) {
function retryAfterSeconds(header, now = Date.now()) {
const text = String(header ?? '').trim();
if (!text) return null;
const seconds = /^\d+$/.test(text)
+1 -1
View File
@@ -145,7 +145,7 @@ const CODEX_LOGIN_ENV =
/^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
export function codexLoginEnvironment(environment, codexHome) {
function codexLoginEnvironment(environment, codexHome) {
const env = {};
for (const [name, value] of Object.entries(environment || {}))
if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
+1 -1
View File
@@ -157,4 +157,4 @@ function createDebugLogHandler({
};
}
export { createDebugLogHandler, omitVoiceContent };
export { createDebugLogHandler };
+20
View File
@@ -1,5 +1,6 @@
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
import { readRequestBodyCapped } from './common/request.js';
import { admitSameSite } from './common/same-site.js';
import {
OVERPASS_MAX_BODY_BYTES,
OVERPASS_MAX_CONCURRENT,
@@ -27,6 +28,14 @@ const _overpassInFlight = new Map();
let _overpassConcurrent = 0;
// Bound body-reading and validation work, including malformed requests and
// cache hits. Keep the tighter upstream budget below independent of this gate.
const _overpassRequestLimiter = makeRateLimiter({
windowMs: 60_000,
max: 300,
globalMax: 900,
});
const _overpassRateLimiter = makeRateLimiter({
windowMs: 60_000,
max: 90,
@@ -94,6 +103,17 @@ function overpassProxy({ routing = {} } = {}) {
return;
}
if (admitSameSite(req, res)) return;
if (!_overpassRequestLimiter(clientKey(req))) {
res.writeHead(429, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
'Retry-After': '60',
});
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
// Collect POST body with a hard byte cap (Overpass QL queries are small)
let body;
try {
+31 -5
View File
@@ -18,6 +18,29 @@ function isOverpassBoundaryQuery(cacheKey) {
return /is_in\s*\(|\bpivot\b/i.test(String(cacheKey || ''));
}
// Input has already had quoted literals and comments removed. Consume each
// character once: a regex that searches for a closing bracket from every `[` is
// quadratic on an unterminated filter. Nested or unmatched brackets are invalid.
function stripTagFilters(text) {
const parts = [];
let start = 0;
let inFilter = false;
for (let i = 0; i < text.length; i += 1) {
if (text[i] === '[') {
if (inFilter) return null;
parts.push(text.slice(start, i), ' ');
inFilter = true;
} else if (text[i] === ']') {
if (!inFilter) return null;
inFilter = false;
start = i + 1;
}
}
if (inFilter) return null;
parts.push(text.slice(start));
return parts.join('');
}
/**
* Validate + clamp an Overpass form body. Defends the generic proxy against
* planet-scale abuse: requires exactly one `data` query in which EVERY element
@@ -50,6 +73,10 @@ function sanitizeOverpassBody(rawBody) {
// Blank quoted literals + strip comments in one lexer pass so a fake bound or a
// `//` inside a string can't hide an unbounded selector (or satisfy a bound).
const stripped = stripOverpassNoise(data);
const withoutTags = stripTagFilters(stripped);
if (withoutTags === null) {
return { ok: false, error: 'Malformed Overpass tag filter' };
}
// Reject oversized radii in EVERY around form — point `around:r,lat,lon` AND the
// input-set form `around.set:r` — and parse the full numeric token so scientific
@@ -93,20 +120,19 @@ function sanitizeOverpassBody(rawBody) {
// statement. So `way[...]->.a` (global assigned to a set) is rejected, while the
// app's `is_in(...)->.a; area.a[...]` and `area(id)->.x; rel(pivot.x)` validate.
const boundedSets = new Set();
for (let stmt of stripped.split(';')) {
for (let stmt of withoutTags.split(';')) {
stmt = stmt.trim();
if (!stmt || stmt.startsWith('[') || /^out\b/.test(stmt)) continue;
// Strip output-set assignments (NOT input bounds), then strip bracket tag
// filters so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) can never be
// Strip output-set assignments (NOT input bounds). Bracket tag filters were
// removed above so a tag KEY/value (e.g. `way[is_in]`, `node[around]`) cannot be
// misread as a spatial bound. Bounds live in (...) / function calls / set
// refs, never inside [...], so the probe loses nothing real.
const outSets = [];
const body = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
const probe = stmt.replace(/->\s*\.(\w+)/g, (_, name) => {
outSets.push(name);
return ' ';
});
const probe = body.replace(/\[[^\]]*\]/g, ' ');
// Reject element-in-area scans on the TAG-STRIPPED probe, so a tag filter
// between the selector and the area filter (way["highway"](area.a)) can't hide
@@ -0,0 +1,178 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { Readable } from 'node:stream';
import { spawnSync } from 'node:child_process';
import { overpassProxy } from '../overpass.js';
import { sanitizeOverpassBody } from './query.js';
const form = (query) => `data=${encodeURIComponent(query)}`;
test('bracket validation rejects malformed filters without trusting their contents as bounds', () => {
for (const query of [
'node[',
'node]',
'node[[name]](around:20,1,1);out;',
'node[name=around:20];out;',
'node[is_in];out;',
'node[name="(around:20,1,1)"];out;',
'node(around:20,1,1);way[name];out;',
'way[highway](area.a);out;',
'node[->.a](around:20,1,1);way.a;out;',
]) {
assert.equal(sanitizeOverpassBody(form(query)).ok, false, query);
}
});
test('bounded queries preserve quoted brackets, comments, multiple tags and timeout clamping', () => {
for (const query of [
'[out:json][timeout:999];node[name="[Town]; ]"](around:20,1,1);out;',
'/* [ */way[highway][name](30.1,-97.9,30.5,-97.5);out geom;',
'is_in(30.2,-97.7)->.a;area.a[boundary="administrative"];out tags;',
'area(3600039152)->.x;rel(pivot.x);out geom;',
]) {
const result = sanitizeOverpassBody(form(query));
assert.equal(result.ok, true, query);
assert.equal(
new URLSearchParams(result.body).get('data'),
query.replace('[timeout:999]', '[timeout:30]'),
);
}
});
test('a burst of maximum-size unterminated filters completes within a bounded process deadline', () => {
// A child deadline can stop synchronous regex backtracking; a test timeout in
// the same event loop cannot. The old parser takes several seconds for this.
const result = spawnSync(
process.execPath,
[
'--input-type=module',
'-e',
`import { sanitizeOverpassBody } from ${JSON.stringify(new URL('./query.js', import.meta.url).href)};
for (let i = 0; i < 20; i++) {
if (sanitizeOverpassBody('data=node' + '['.repeat(24000)).ok) process.exit(1);
}`,
],
{ timeout: 2000, encoding: 'utf8' },
);
assert.ifError(result.error);
assert.equal(result.status, 0, result.stderr);
});
function handler(hook = 'configureServer') {
let route;
overpassProxy()[hook]({
middlewares: {
use(path, fn) {
if (path === '/api/overpass') route = fn;
},
},
});
return route;
}
async function call(
route,
{
headers = {},
peer = '127.0.0.1',
body = 'data=node[',
forbidRead = false,
method = 'POST',
url = '/',
} = {},
) {
let reads = 0;
const req = new Readable({
read() {
reads++;
assert.equal(forbidRead, false, 'refused request must not read its body');
this.push(Buffer.from(body));
this.push(null);
},
});
Object.assign(req, {
method,
url,
headers: { host: 'localhost:4173', ...headers },
socket: { remoteAddress: peer },
});
const res = {
statusCode: 200,
headers: {},
setHeader(key, value) {
this.headers[key.toLowerCase()] = value;
},
writeHead(status, values) {
this.statusCode = status;
for (const [key, value] of Object.entries(values))
this.setHeader(key, value);
},
end(value) {
this.body = value;
},
};
await route(req, res);
return { ...res, reads };
}
for (const hook of ['configureServer', 'configurePreviewServer']) {
test(`${hook}: cross-site requests are rejected before consuming a body or fetching`, async (t) => {
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler(hook);
for (const headers of [
{ origin: 'https://foreign.example', 'content-type': 'text/plain' },
{ 'sec-fetch-site': 'cross-site' },
{ 'sec-fetch-site': 'same-site' },
{ origin: 'null' },
{ 'x-forwarded-for': '127.0.0.1' },
]) {
const response = await call(route, { headers, forbidRead: true });
assert.equal(response.statusCode, 403);
assert.equal(response.reads, 0);
assert.equal(response.headers['cache-control'], 'no-store');
}
});
}
test('same-origin, LAN and non-browser requests still reach validation; GET status stays available', async () => {
const route = handler();
for (const headers of [
{},
{ origin: 'http://localhost:4173', 'sec-fetch-site': 'same-origin' },
{ host: '192.168.1.10:4173', origin: 'http://192.168.1.10:4173' },
]) {
const response = await call(route, { headers });
assert.equal(response.statusCode, 400);
assert.equal(response.reads, 1);
assert.match(response.body, /Malformed Overpass tag filter/);
}
const response = await call(route, {
method: 'GET',
url: '/status',
forbidRead: true,
});
assert.equal(response.statusCode, 200);
assert.equal(typeof JSON.parse(response.body).configured, 'boolean');
});
test('invalid requests spend admission quota before parsing, including on an unconfigured server', async (t) => {
let now = Date.now() + 120000;
t.mock.method(Date, 'now', () => now);
t.mock.method(globalThis, 'fetch', () =>
assert.fail('unexpected upstream fetch'),
);
const route = handler();
const peer = 'admission-limit-test';
for (let i = 0; i < 300; i++) {
assert.equal((await call(route, { peer })).statusCode, 400);
}
const denied = await call(route, { peer, forbidRead: true });
assert.equal(denied.statusCode, 429);
assert.equal(denied.reads, 0);
assert.equal(denied.headers['retry-after'], '60');
assert.equal((await call(route, { peer: 'other-peer' })).statusCode, 400);
now += 60001;
assert.equal((await call(route, { peer })).statusCode, 400);
});
+1 -1
View File
@@ -26,7 +26,7 @@ import {
RADIO_UUID_RE,
RADIO_FALLBACK_MIRRORS,
} from './constants.js';
export async function mapRadioConcurrent(values, concurrency, mapper) {
async function mapRadioConcurrent(values, concurrency, mapper) {
const results = new Array(values.length);
let cursor = 0;
const workers = Array.from(
+5 -5
View File
@@ -20,8 +20,8 @@ import { admitSameSiteKeyless } from './common/same-site.js';
* cache and serve-stale shape. Cache entries and stale fallback are keyed per
* 5dp point, so reordered and partially overlapping batches reuse prior work.
* Only missing/stale points go upstream; the response is rebuilt in exact
* request order. Larger requests are chunked sequentially, and one failing
* chunk does not discard the chunks that resolved.
* request order. Each request is bounded to one 64-point upstream chunk;
* larger client workloads are split by the browser terrain service.
*
* Bounds, because every distinct 5dp point is its own key: coordinates must be
* in WGS-84 range (rejected at parse), the point cache has an entry ceiling
@@ -55,7 +55,7 @@ export function terrainHeightsProxy(options = {}) {
// 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
// that range.
const UPSTREAM_CHUNK = 64;
const MAX_POINTS = 2000;
const MAX_POINTS = UPSTREAM_CHUNK;
/**
* Entry ceiling for the point cache, and with it the file written from it.
*
@@ -255,7 +255,6 @@ export function terrainHeightsProxy(options = {}) {
res.end(JSON.stringify({ error: 'Rate limit exceeded' }));
return;
}
await loadDiskOnce();
const parsedUrl = new URL(req.url || '', 'http://internal');
const rawPoints = parsedUrl.searchParams.get('points');
const points = parseTerrainPoints(rawPoints);
@@ -267,12 +266,13 @@ export function terrainHeightsProxy(options = {}) {
return;
}
if (points.length > MAX_POINTS) {
send(500, {
send(413, {
error: `too many points (${points.length}); max ${MAX_POINTS} per request`,
});
return;
}
await loadDiskOnce();
const outcome = await resolveTerrainHeightRequest({
points,
cache: mem,
-1
View File
@@ -1,5 +1,4 @@
export {
createTransitHistory,
TRANSIT_HISTORY_LIMITS,
validTransitIdentifier,
} from '../../src/sources/transitHistoryStore.js';
+1 -1
View File
@@ -3,7 +3,7 @@ import { decodeWindGribMessage } from './decode.js';
import { resampleWeatherSnapshot, weatherScalarMetadata } from './grid.js';
/** Keyless ECMWF Open Data root for real-time IFS forecasts. */
export const IFS_BASE = 'https://data.ecmwf.int/forecasts';
const IFS_BASE = 'https://data.ecmwf.int/forecasts';
/**
* Select the latest IFS cycle that should be available.