benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-04 02:14:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: adopt formatting for CCTV and radio provider modules

Browse Source
This commit is contained in:
Sameh Khamis committed 2026-09-12 12:09:04 -07:00
1 parent 79b4929b88
commit c69416ff6b
13 files changed
+862 -364
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/format-scope.json
+13 -1
View File
@@ -69,5 +69,17 @@
"src/data/tomtomTiles.test.mjs", "src/data/tomtomTiles.test.mjs",
"src/data/firmsCsv.js", "src/data/firmsCsv.js",
"src/data/firmsCsv.test.mjs", "src/data/firmsCsv.test.mjs",
"src/tooling/environmentProviders.test.mjs" "src/tooling/environmentProviders.test.mjs",
"server/providers/cctv.js",
"server/providers/cctv/catalog.js",
"server/providers/cctv/constants.js",
"server/providers/cctv/media.js",
"server/providers/cctv/normalize.js",
"server/providers/cctv/sources.js",
"server/providers/radio.js",
"server/providers/radio/catalog.js",
"server/providers/radio/constants.js",
"server/providers/radio/stations.js",
"server/providers/radio/transport.js",
"src/tooling/mediaProviders.test.mjs"
] ]
server/providers/cctv.js
+117 -33
View File
@@ -1,6 +1,14 @@
import { createCctvCatalog } from './cctv/catalog.js'; import { createCctvCatalog } from './cctv/catalog.js';
import { normalizeFeedType, isVideoFeedType, toFiniteNumber } from './cctv/normalize.js'; import {
import { buildSyntheticCctvSvg, proxyMediaResponse, fetchCctvImageFromUpstream } from './cctv/media.js'; normalizeFeedType,
isVideoFeedType,
toFiniteNumber,
} from './cctv/normalize.js';
import {
buildSyntheticCctvSvg,
proxyMediaResponse,
fetchCctvImageFromUpstream,
} from './cctv/media.js';
import { CCTV_FRAME_FETCH_TIMEOUT_MS } from './cctv/constants.js'; import { CCTV_FRAME_FETCH_TIMEOUT_MS } from './cctv/constants.js';
import { googleServerApiKey } from './places/google-key.js'; import { googleServerApiKey } from './places/google-key.js';
export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
@@ -58,7 +66,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
: null, : null,
frameUrl: `/api/cctv/frame/${encodeURIComponent(cameraId)}`, frameUrl: `/api/cctv/frame/${encodeURIComponent(cameraId)}`,
provider: source?.provider || '', provider: source?.provider || '',
sourceKind: source?.sourceKind || (source?.url ? 'configured' : 'fallback'), sourceKind:
source?.sourceKind || (source?.url ? 'configured' : 'fallback'),
}; };
}; };
@@ -71,14 +80,24 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
*/ */
const streetViewFallback = async ({ lat, lon, heading, fov, pitch }) => { const streetViewFallback = async ({ lat, lon, heading, fov, pitch }) => {
const streetViewKey = googleServerApiKey(); const streetViewKey = googleServerApiKey();
if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon)) return null; if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon))
return null;
try { try {
const sv = new URL('https://maps.googleapis.com/maps/api/streetview'); const sv = new URL('https://maps.googleapis.com/maps/api/streetview');
sv.searchParams.set('size', '960x540'); sv.searchParams.set('size', '960x540');
sv.searchParams.set('location', `${lat},${lon}`); sv.searchParams.set('location', `${lat},${lon}`);
sv.searchParams.set('heading', String(Number.isFinite(heading) ? heading : 0)); sv.searchParams.set(
sv.searchParams.set('fov', String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80)); 'heading',
sv.searchParams.set('pitch', String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0)); String(Number.isFinite(heading) ? heading : 0),
);
sv.searchParams.set(
'fov',
String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80),
);
sv.searchParams.set(
'pitch',
String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0),
);
sv.searchParams.set('source', 'outdoor'); sv.searchParams.set('source', 'outdoor');
sv.searchParams.set('return_error_code', 'true'); sv.searchParams.set('return_error_code', 'true');
sv.searchParams.set('key', streetViewKey); sv.searchParams.set('key', streetViewKey);
@@ -106,7 +125,9 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
server.middlewares.use('/api/cctv', async (req, res) => { server.middlewares.use('/api/cctv', async (req, res) => {
try { try {
const sources = await getCctvSources(); const sources = await getCctvSources();
const sourceById = new Map(sources.map((source) => [source.id, source])); const sourceById = new Map(
sources.map((source) => [source.id, source]),
);
const url = new URL(req.url || '/', 'http://localhost'); const url = new URL(req.url || '/', 'http://localhost');
if (url.pathname === '/sources') { if (url.pathname === '/sources') {
@@ -127,33 +148,47 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
mountHeightM: source.mountHeightM, mountHeightM: source.mountHeightM,
groundElevationM: source.groundElevationM, groundElevationM: source.groundElevationM,
feedType: normalizeFeedType(source.feedType), feedType: normalizeFeedType(source.feedType),
sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
poseSource: source.poseSource, poseSource: source.poseSource,
license: source.license, license: source.license,
})), })),
}; };
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify(body)); res.end(JSON.stringify(body));
return; return;
} }
if (url.pathname === '/health') { if (url.pathname === '/health') {
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ cameras: listHealth() })); res.end(JSON.stringify({ cameras: listHealth() }));
return; return;
} }
if (url.pathname.startsWith('/stream/')) { if (url.pathname.startsWith('/stream/')) {
const cameraId = decodeURIComponent(url.pathname.replace('/stream/', '').trim()) || 'camera'; const cameraId =
decodeURIComponent(url.pathname.replace('/stream/', '').trim()) ||
'camera';
const source = sourceById.get(cameraId); const source = sourceById.get(cameraId);
const payload = buildStreamPayload(source, cameraId); const payload = buildStreamPayload(source, cameraId);
res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(200, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify(payload)); res.end(JSON.stringify(payload));
return; return;
} }
if (url.pathname.startsWith('/media/')) { if (url.pathname.startsWith('/media/')) {
const cameraId = decodeURIComponent(url.pathname.replace('/media/', '').trim()) || 'camera'; const cameraId =
decodeURIComponent(url.pathname.replace('/media/', '').trim()) ||
'camera';
const source = sourceById.get(cameraId); const source = sourceById.get(cameraId);
const mediaUrl = source?.url || ''; const mediaUrl = source?.url || '';
const feedType = normalizeFeedType(source?.feedType || 'image'); const feedType = normalizeFeedType(source?.feedType || 'image');
@@ -165,13 +200,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
label: source?.provider || 'No upstream URL', label: source?.provider || 'No upstream URL',
message: 'No stream URL configured', message: 'No stream URL configured',
}); });
res.writeHead(404, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(404, {
res.end(JSON.stringify({ error: 'No media URL configured for this camera' })); 'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(
JSON.stringify({
error: 'No media URL configured for this camera',
}),
);
return; return;
} }
try { try {
const upstreamHeaders = { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }; const upstreamHeaders = {
'User-Agent': 'gods-eye-view-cctv-proxy/1.0',
};
const requestRange = req.headers?.range; const requestRange = req.headers?.range;
if (requestRange) upstreamHeaders.Range = requestRange; if (requestRange) upstreamHeaders.Range = requestRange;
const upstream = await fetch(mediaUrl, { const upstream = await fetch(mediaUrl, {
@@ -185,12 +229,25 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
label: source?.provider || 'Configured source', label: source?.provider || 'Configured source',
message: `Upstream HTTP ${upstream.status}`, message: `Upstream HTTP ${upstream.status}`,
}); });
res.writeHead(upstream.status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(upstream.status, {
res.end(JSON.stringify({ error: `Upstream returned ${upstream.status}` })); 'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(
JSON.stringify({
error: `Upstream returned ${upstream.status}`,
}),
);
return; return;
} }
if (isVideoFeedType(feedType) && !(contentType.startsWith('video/') || contentType.includes('mpegurl'))) { if (
isVideoFeedType(feedType) &&
!(
contentType.startsWith('video/') ||
contentType.includes('mpegurl')
)
) {
setHealth(cameraId, { setHealth(cameraId, {
status: 'degraded', status: 'degraded',
sourceKind: 'upstream', sourceKind: 'upstream',
@@ -202,12 +259,16 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
status: 'ok', status: 'ok',
sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot', sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot',
label: source?.provider || 'Configured source', label: source?.provider || 'Configured source',
message: isVideoFeedType(feedType) ? 'Live stream connected' : 'Snapshot feed connected', message: isVideoFeedType(feedType)
? 'Live stream connected'
: 'Snapshot feed connected',
}); });
} }
await proxyMediaResponse(res, upstream, { await proxyMediaResponse(res, upstream, {
sourceHeader: isVideoFeedType(feedType) ? 'live-media' : 'upstream-image', sourceHeader: isVideoFeedType(feedType)
? 'live-media'
: 'upstream-image',
}); });
return; return;
} catch (error) { } catch (error) {
@@ -217,7 +278,10 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
label: source?.provider || 'Configured source', label: source?.provider || 'Configured source',
message: error?.message || 'Media fetch failed', message: error?.message || 'Media fetch failed',
}); });
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'Media proxy failed' })); res.end(JSON.stringify({ error: 'Media proxy failed' }));
return; return;
} }
@@ -229,23 +293,33 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return; return;
} }
const cameraId = decodeURIComponent(url.pathname.replace('/frame/', '').trim()) || 'camera'; const cameraId =
decodeURIComponent(url.pathname.replace('/frame/', '').trim()) ||
'camera';
const source = sourceById.get(cameraId); const source = sourceById.get(cameraId);
const label = url.searchParams.get('label') || source?.name || cameraId; const label =
url.searchParams.get('label') || source?.name || cameraId;
const city = url.searchParams.get('city') || source?.city || ''; const city = url.searchParams.get('city') || source?.city || '';
const lat = Number(url.searchParams.get('lat') || source?.lat); const lat = Number(url.searchParams.get('lat') || source?.lat);
const lon = Number(url.searchParams.get('lon') || source?.lon); const lon = Number(url.searchParams.get('lon') || source?.lon);
const heading = Number(url.searchParams.get('heading') || source?.headingDeg); const heading = Number(
url.searchParams.get('heading') || source?.headingDeg,
);
const fov = Number(url.searchParams.get('fov') || source?.fovDeg); const fov = Number(url.searchParams.get('fov') || source?.fovDeg);
const pitch = Number(url.searchParams.get('pitch') || source?.pitchDeg); const pitch = Number(
url.searchParams.get('pitch') || source?.pitchDeg,
);
// Only use server-registered upstream URLs — never accept client-supplied URLs // Only use server-registered upstream URLs — never accept client-supplied URLs
// (prevents SSRF via ?upstream= query parameter) // (prevents SSRF via ?upstream= query parameter)
const upstreamCandidate = const upstreamCandidate =
source?.snapshotUrl source?.snapshotUrl ||
|| (!isVideoFeedType(normalizeFeedType(source?.feedType)) ? source?.url : ''); (!isVideoFeedType(normalizeFeedType(source?.feedType))
? source?.url
: '');
const upstreamImage = await fetchCctvImageFromUpstream(upstreamCandidate); const upstreamImage =
await fetchCctvImageFromUpstream(upstreamCandidate);
if (upstreamImage?.ok) { if (upstreamImage?.ok) {
setHealth(cameraId, { setHealth(cameraId, {
status: 'ok', status: 'ok',
@@ -262,7 +336,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
return; return;
} }
const sv = await streetViewFallback({ lat, lon, heading, fov, pitch }); const sv = await streetViewFallback({
lat,
lon,
heading,
fov,
pitch,
});
if (sv?.ok) { if (sv?.ok) {
setHealth(cameraId, { setHealth(cameraId, {
status: 'degraded', status: 'degraded',
@@ -283,14 +363,18 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
cameraId, cameraId,
label, label,
city, city,
status: source?.url ? 'UPSTREAM UNAVAILABLE' : 'NO UPSTREAM CONFIGURED', status: source?.url
? 'UPSTREAM UNAVAILABLE'
: 'NO UPSTREAM CONFIGURED',
}); });
setHealth(cameraId, { setHealth(cameraId, {
status: 'degraded', status: 'degraded',
sourceKind: 'synthetic', sourceKind: 'synthetic',
label: source?.provider || 'Synthetic fallback', label: source?.provider || 'Synthetic fallback',
message: source?.url ? 'Upstream unavailable' : 'No source configured', message: source?.url
? 'Upstream unavailable'
: 'No source configured',
}); });
res.writeHead(200, { res.writeHead(200, {
server/providers/cctv/catalog.js
+135 -94
View File
@@ -1,8 +1,16 @@
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import { DEFAULT_CCTV_SOURCE_FILE, DEFAULT_CCTV_MAX_SOURCES, CCTV_SOURCE_CACHE_MS } from './constants.js'; import {
DEFAULT_CCTV_SOURCE_FILE,
DEFAULT_CCTV_MAX_SOURCES,
CCTV_SOURCE_CACHE_MS,
} from './constants.js';
import { normalizeSourceItem } from './normalize.js'; import { normalizeSourceItem } from './normalize.js';
import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, loadTflSourcesFromOpenData } from './sources.js'; import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
loadTflSourcesFromOpenData,
} from './sources.js';
/** /**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
* *
@@ -19,7 +27,11 @@ function loadSourcesFromFile(sourceRoot) {
const parsed = JSON.parse(raw); const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed : []; return Array.isArray(parsed) ? parsed : [];
} catch (error) { } catch (error) {
console.warn('[CCTV] failed to read source file:', resolved, error?.message || error); console.warn(
'[CCTV] failed to read source file:',
resolved,
error?.message || error,
);
return []; return [];
} }
} }
@@ -42,99 +54,128 @@ function loadSourcesFromEnv() {
/** Create an independent catalog rooted in the consuming application. */ /** Create an independent catalog rooted in the consuming application. */
export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
/** @type {Array<object>} Cached merged + normalized CCTV source list. */ /** @type {Array<object>} Cached merged + normalized CCTV source list. */
let _cctvSourceCache = []; let _cctvSourceCache = [];
/** @type {number} Epoch-ms when the source cache was last refreshed. */ /** @type {number} Epoch-ms when the source cache was last refreshed. */
let _cctvSourceCacheAt = 0; let _cctvSourceCacheAt = 0;
/** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent /** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
* callers so a post-TTL burst launches ONE refetch, not one per request. */ * callers so a post-TTL burst launches ONE refetch, not one per request. */
let _cctvSourceInflight = null; let _cctvSourceInflight = null;
/** /**
* Assemble and cache the merged CCTV source list. * Assemble and cache the merged CCTV source list.
* *
* Merges sources from three origins (Austin Open Data, local file, * Merges sources from three origins (Austin Open Data, local file,
* env variable), deduplicates by ID, applies the global max cap, and * env variable), deduplicates by ID, applies the global max cap, and
* caches for CCTV_SOURCE_CACHE_MS. * caches for CCTV_SOURCE_CACHE_MS.
* *
* @returns {Promise<Array<object>>} Deduplicated, capped source list. * @returns {Promise<Array<object>>} Deduplicated, capped source list.
*/ */
async function getCctvSources() { async function getCctvSources() {
const now = Date.now(); const now = Date.now();
if (_cctvSourceCache.length && now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS) { if (
_cctvSourceCache.length &&
now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS
) {
return _cctvSourceCache;
}
// Single-flight: a burst of requests arriving past the TTL shares ONE refresh
// instead of each launching the full multi-provider refetch. The `.finally`
// clears the ref so the next post-TTL cycle starts fresh.
if (_cctvSourceInflight) return _cctvSourceInflight;
_cctvSourceInflight = refreshCctvSources().finally(() => {
_cctvSourceInflight = null;
});
return _cctvSourceInflight;
}
/**
* Assemble and cache the merged CCTV source list from file/env + live packs.
* Always resolves (loaders self-catch to []); on a fully-empty refresh with a
* good prior catalog it serves stale rather than blanking the CCTV layer.
*
* @returns {Promise<Array<object>>} Deduplicated, capped source list.
*/
async function refreshCctvSources() {
const fromFile = loadSourcesFromFile(sourceRoot);
const fromEnv = loadSourcesFromEnv();
const forceAustin =
String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1';
const preferAustin =
String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0';
// Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack
// is configured and live packs aren't forced — same gate that governed the
// Austin-only fetch, now governing all three. Each pack fails independently.
const needsLiveSources =
forceAustin || (fromFile.length + fromEnv.length === 0 && preferAustin);
const tflEnabled =
String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0';
let fromAustin = [];
let fromCaltrans = [];
let fromTfl = [];
if (needsLiveSources) {
const [austinResult, caltransResult, tflResult] =
await Promise.allSettled([
loadAustinSourcesFromOpenData(),
loadCaltransSourcesFromOpenData(),
tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]),
]);
fromAustin =
austinResult.status === 'fulfilled' ? austinResult.value : [];
fromCaltrans =
caltransResult.status === 'fulfilled' ? caltransResult.value : [];
fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : [];
}
// Live sources first so file/env overrides win on duplicate IDs (Map last-write).
const merged = [
...fromAustin,
...fromCaltrans,
...fromTfl,
...fromFile,
...fromEnv,
];
// Deduplicate by camera ID (last-write wins because of Map.set)
const byId = new Map();
for (const item of merged) {
if (!item || typeof item !== 'object') continue;
const normalized = normalizeSourceItem(item);
if (!normalized.id) continue;
byId.set(normalized.id, normalized);
}
const mergedSources = Array.from(byId.values());
const maxRaw = Number(
process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
: DEFAULT_CCTV_MAX_SOURCES;
if (mergedSources.length > maxCount) {
console.warn(
`[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`,
);
}
const capped =
mergedSources.length > maxCount
? mergedSources.slice(0, maxCount)
: mergedSources;
if (capped.length > 0 || _cctvSourceCache.length === 0) {
_cctvSourceCache = capped;
} else {
// Every source came back empty (all live packs timed out / upstream outage)
// but a good catalog is already cached — serve it stale rather than blanking
// every CCTV route. Advancing the timestamp waits one TTL before retrying,
// which (with single-flight) bounds load on a persistently-down upstream.
console.warn(
`[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`,
);
}
_cctvSourceCacheAt = Date.now();
return _cctvSourceCache; return _cctvSourceCache;
} }
// Single-flight: a burst of requests arriving past the TTL shares ONE refresh
// instead of each launching the full multi-provider refetch. The `.finally`
// clears the ref so the next post-TTL cycle starts fresh.
if (_cctvSourceInflight) return _cctvSourceInflight;
_cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; });
return _cctvSourceInflight;
}
/** return getCctvSources;
* Assemble and cache the merged CCTV source list from file/env + live packs.
* Always resolves (loaders self-catch to []); on a fully-empty refresh with a
* good prior catalog it serves stale rather than blanking the CCTV layer.
*
* @returns {Promise<Array<object>>} Deduplicated, capped source list.
*/
async function refreshCctvSources() {
const fromFile = loadSourcesFromFile(sourceRoot);
const fromEnv = loadSourcesFromEnv();
const forceAustin = String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1';
const preferAustin = String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0';
// Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack
// is configured and live packs aren't forced — same gate that governed the
// Austin-only fetch, now governing all three. Each pack fails independently.
const needsLiveSources = forceAustin || ((fromFile.length + fromEnv.length) === 0 && preferAustin);
const tflEnabled = String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0';
let fromAustin = [];
let fromCaltrans = [];
let fromTfl = [];
if (needsLiveSources) {
const [austinResult, caltransResult, tflResult] = await Promise.allSettled([
loadAustinSourcesFromOpenData(),
loadCaltransSourcesFromOpenData(),
tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]),
]);
fromAustin = austinResult.status === 'fulfilled' ? austinResult.value : [];
fromCaltrans = caltransResult.status === 'fulfilled' ? caltransResult.value : [];
fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : [];
}
// Live sources first so file/env overrides win on duplicate IDs (Map last-write).
const merged = [...fromAustin, ...fromCaltrans, ...fromTfl, ...fromFile, ...fromEnv];
// Deduplicate by camera ID (last-write wins because of Map.set)
const byId = new Map();
for (const item of merged) {
if (!item || typeof item !== 'object') continue;
const normalized = normalizeSourceItem(item);
if (!normalized.id) continue;
byId.set(normalized.id, normalized);
}
const mergedSources = Array.from(byId.values());
const maxRaw = Number(process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES);
const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) : DEFAULT_CCTV_MAX_SOURCES;
if (mergedSources.length > maxCount) {
console.warn(`[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`);
}
const capped = mergedSources.length > maxCount ? mergedSources.slice(0, maxCount) : mergedSources;
if (capped.length > 0 || _cctvSourceCache.length === 0) {
_cctvSourceCache = capped;
} else {
// Every source came back empty (all live packs timed out / upstream outage)
// but a good catalog is already cached — serve it stale rather than blanking
// every CCTV route. Advancing the timestamp waits one TTL before retrying,
// which (with single-flight) bounds load on a persistently-down upstream.
console.warn(`[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`);
}
_cctvSourceCacheAt = Date.now();
return _cctvSourceCache;
}
return getCctvSources;
} }
server/providers/cctv/constants.js
+4 -2
View File
@@ -1,6 +1,7 @@
export const DEFAULT_CCTV_SOURCE_FILE = 'config/cctv_sources.austin.json'; export const DEFAULT_CCTV_SOURCE_FILE = 'config/cctv_sources.austin.json';
/** Austin Open Data portal endpoint for traffic camera records. */ /** Austin Open Data portal endpoint for traffic camera records. */
export const DEFAULT_AUSTIN_ROWS_URL = 'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD'; export const DEFAULT_AUSTIN_ROWS_URL =
'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD';
/** Default cap on Austin cameras after distance-based prioritization. */ /** Default cap on Austin cameras after distance-based prioritization. */
export const DEFAULT_AUSTIN_MAX_SOURCES = 250; export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
/** Global cap on total CCTV sources served by the proxy. */ /** Global cap on total CCTV sources served by the proxy. */
@@ -22,7 +23,8 @@ export const CALTRANS_ANCHORS = [
]; ];
/** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */ /** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */
export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam'; export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam';
export const TFL_IMAGE_ORIGIN = 'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'; export const TFL_IMAGE_ORIGIN =
'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/';
export const DEFAULT_TFL_MAX_SOURCES = 250; export const DEFAULT_TFL_MAX_SOURCES = 250;
export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 }; export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 };
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL) infrequent. Frames are fetched per-request and are unaffected. */ /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL) infrequent. Frames are fetched per-request and are unaffected. */
server/providers/cctv/media.js
+27 -10
View File
@@ -97,8 +97,13 @@ export function toReadable(body) {
* @param {object} [opts] * @param {object} [opts]
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header. * @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
*/ */
export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstream' } = {}) { export async function proxyMediaResponse(
const contentType = upstream.headers.get('content-type') || 'application/octet-stream'; res,
upstream,
{ sourceHeader = 'upstream' } = {},
) {
const contentType =
upstream.headers.get('content-type') || 'application/octet-stream';
const cacheControl = upstream.headers.get('cache-control') || 'no-store'; const cacheControl = upstream.headers.get('cache-control') || 'no-store';
const contentLength = upstream.headers.get('content-length'); const contentLength = upstream.headers.get('content-length');
const contentRange = upstream.headers.get('content-range'); const contentRange = upstream.headers.get('content-range');
@@ -116,10 +121,20 @@ export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstre
// Live MJPEG/HLS streams are unbounded by design and send no content-length, // Live MJPEG/HLS streams are unbounded by design and send no content-length,
// so they pipe normally (piping streams to the client, never buffering). // so they pipe normally (piping streams to the client, never buffering).
const MEDIA_DECLARED_CAP_BYTES = 64 * 1024 * 1024; const MEDIA_DECLARED_CAP_BYTES = 64 * 1024 * 1024;
if (Number.isFinite(Number(contentLength)) && Number(contentLength) > MEDIA_DECLARED_CAP_BYTES) { if (
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); Number.isFinite(Number(contentLength)) &&
Number(contentLength) > MEDIA_DECLARED_CAP_BYTES
) {
res.writeHead(502, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify({ error: 'Upstream media exceeds size cap' })); res.end(JSON.stringify({ error: 'Upstream media exceeds size cap' }));
try { await upstream.body?.cancel(); } catch { /* no-op */ } try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return; return;
} }
@@ -151,14 +166,16 @@ export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstre
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout. * @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>} * @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
*/ */
export async function fetchCctvImageFromUpstream(url, { export async function fetchCctvImageFromUpstream(
fetchImpl = fetch, url,
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS, { fetchImpl = fetch, timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS } = {},
} = {}) { ) {
if (!url || !/^https?:\/\//i.test(url)) return null; if (!url || !/^https?:\/\//i.test(url)) return null;
const controller = new AbortController(); const controller = new AbortController();
const timeoutId = setTimeout(() => { const timeoutId = setTimeout(() => {
controller.abort(new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError')); controller.abort(
new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError'),
);
}, timeoutMs); }, timeoutMs);
try { try {
const upstream = await fetchImpl(url, { const upstream = await fetchImpl(url, {
server/providers/cctv/normalize.js
+57 -20
View File
@@ -39,7 +39,9 @@ export function escapeXml(text) {
* @returns {string} Normalized feed type. * @returns {string} Normalized feed type.
*/ */
export function normalizeFeedType(value) { export function normalizeFeedType(value) {
const raw = String(value || '').trim().toLowerCase(); const raw = String(value || '')
.trim()
.toLowerCase();
if (!raw) return 'image'; if (!raw) return 'image';
if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image';
if (raw === 'mjpg') return 'mjpeg'; if (raw === 'mjpg') return 'mjpeg';
@@ -93,7 +95,9 @@ export function normalizeKey(text) {
* @returns {{lat:number, lon:number}} * @returns {{lat:number, lon:number}}
*/ */
export function parsePointString(value) { export function parsePointString(value) {
const match = String(value || '').match(/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i); const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
if (!match) return { lat: NaN, lon: NaN }; if (!match) return { lat: NaN, lon: NaN };
return { return {
lon: toFiniteNumber(match[1]), lon: toFiniteNumber(match[1]),
@@ -123,11 +127,16 @@ export function coerceLatLon(value) {
const lat = toFiniteNumber( const lat = toFiniteNumber(
value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat, value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat,
NaN NaN,
); );
const lon = toFiniteNumber( const lon = toFiniteNumber(
value.longitude ?? value.lon ?? value.lng ?? value.x ?? value.Longitude ?? value.Lon, value.longitude ??
NaN value.lon ??
value.lng ??
value.x ??
value.Longitude ??
value.Lon,
NaN,
); );
return { lat, lon }; return { lat, lon };
} }
@@ -152,16 +161,24 @@ export function extractAustinCoords(record) {
]; ];
for (const candidate of candidates) { for (const candidate of candidates) {
const parsed = coerceLatLon(candidate); const parsed = coerceLatLon(candidate);
if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon)) return parsed; if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon))
return parsed;
} }
const lat = toFiniteNumber( const lat = toFiniteNumber(
record.latitude ?? record.lat ?? record.camera_latitude ?? record.location_latitude, record.latitude ??
NaN record.lat ??
record.camera_latitude ??
record.location_latitude,
NaN,
); );
const lon = toFiniteNumber( const lon = toFiniteNumber(
record.longitude ?? record.lon ?? record.lng ?? record.camera_longitude ?? record.location_longitude, record.longitude ??
NaN record.lon ??
record.lng ??
record.camera_longitude ??
record.location_longitude,
NaN,
); );
return { lat, lon }; return { lat, lon };
} }
@@ -239,11 +256,19 @@ export function extractAustinName(record, cameraId) {
* @returns {number} Heading in degrees [0..360), or NaN if unknown. * @returns {number} Heading in degrees [0..360), or NaN if unknown.
*/ */
export function extractAustinHeading(record) { export function extractAustinHeading(record) {
const direct = toFiniteNumber(record.heading_deg ?? record.heading ?? record.bearing, NaN); const direct = toFiniteNumber(
record.heading_deg ?? record.heading ?? record.bearing,
NaN,
);
if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360; if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360;
// Dedicated direction fields: bare cardinal words ("West") are real facings. // Dedicated direction fields: bare cardinal words ("West") are real facings.
const directionKeys = ['direction', 'travel_direction', 'facing', 'facing_direction']; const directionKeys = [
'direction',
'travel_direction',
'facing',
'facing_direction',
];
for (const key of directionKeys) { for (const key of directionKeys) {
const heading = directionToHeading(record[key], true); const heading = directionToHeading(record[key], true);
if (Number.isFinite(heading)) return heading; if (Number.isFinite(heading)) return heading;
@@ -260,7 +285,9 @@ export function extractAustinHeading(record) {
record.cross_street, record.cross_street,
record.description, record.description,
record.name, record.name,
].filter(Boolean).join(' '); ]
.filter(Boolean)
.join(' ');
const inferred = directionToHeading(nameProbe); const inferred = directionToHeading(nameProbe);
if (Number.isFinite(inferred)) return inferred; if (Number.isFinite(inferred)) return inferred;
@@ -276,7 +303,7 @@ export function extractAustinHeading(record) {
*/ */
export function isLikelyAustinCoordinate(lat, lon) { export function isLikelyAustinCoordinate(lat, lon) {
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.40; return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.4;
} }
/** /**
@@ -324,18 +351,26 @@ export function rowArrayToObject(row, columns) {
export function prioritizeSources(cameras, maxCount, anchors) { export function prioritizeSources(cameras, maxCount, anchors) {
const list = Array.isArray(cameras) ? cameras : []; const list = Array.isArray(cameras) ? cameras : [];
const anchorList = (Array.isArray(anchors) ? anchors : []).filter( const anchorList = (Array.isArray(anchors) ? anchors : []).filter(
(a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon) (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon),
); );
if (!Number.isFinite(maxCount) || maxCount <= 0 || list.length <= maxCount || !anchorList.length) { if (
!Number.isFinite(maxCount) ||
maxCount <= 0 ||
list.length <= maxCount ||
!anchorList.length
) {
return list; return list;
} }
const scored = list.map((camera, idx) => { const scored = list.map((camera, idx) => {
const lat = Number(camera?.lat); const lat = Number(camera?.lat);
const lon = Number(camera?.lon); const lon = Number(camera?.lon);
const distKm = Number.isFinite(lat) && Number.isFinite(lon) const distKm =
? Math.min(...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon))) Number.isFinite(lat) && Number.isFinite(lon)
: Number.POSITIVE_INFINITY; ? Math.min(
...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon)),
)
: Number.POSITIVE_INFINITY;
return { camera, idx, distKm }; return { camera, idx, distKm };
}); });
@@ -363,7 +398,9 @@ export function normalizeSourceItem(item) {
lat: toFiniteNumber(item.lat), lat: toFiniteNumber(item.lat),
lon: toFiniteNumber(item.lon), lon: toFiniteNumber(item.lon),
headingDeg: toFiniteNumber(item.headingDeg), headingDeg: toFiniteNumber(item.headingDeg),
headingConfidence: String(item.headingConfidence || item.headingSource || '').toLowerCase(), headingConfidence: String(
item.headingConfidence || item.headingSource || '',
).toLowerCase(),
pitchDeg: toFiniteNumber(item.pitchDeg), pitchDeg: toFiniteNumber(item.pitchDeg),
fovDeg: toFiniteNumber(item.fovDeg), fovDeg: toFiniteNumber(item.fovDeg),
rangeM: toFiniteNumber(item.rangeM), rangeM: toFiniteNumber(item.rangeM),
server/providers/cctv/sources.js
+99 -26
View File
@@ -1,5 +1,28 @@
import { DEFAULT_AUSTIN_ROWS_URL, DEFAULT_AUSTIN_MAX_SOURCES, AUSTIN_DOWNTOWN, CALTRANS_CCTV_URL, DEFAULT_CALTRANS_DISTRICTS, DEFAULT_CALTRANS_MAX_SOURCES, CALTRANS_ANCHORS, TFL_JAMCAM_URL, TFL_IMAGE_ORIGIN, DEFAULT_TFL_MAX_SOURCES, LONDON_CENTER, CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; import {
import { toFiniteNumber, extractAustinCoords, extractAustinCameraId, extractAustinName, extractAustinHeading, isLikelyAustinCoordinate, fallbackHeadingFromId, rowArrayToObject, prioritizeSources } from './normalize.js'; DEFAULT_AUSTIN_ROWS_URL,
DEFAULT_AUSTIN_MAX_SOURCES,
AUSTIN_DOWNTOWN,
CALTRANS_CCTV_URL,
DEFAULT_CALTRANS_DISTRICTS,
DEFAULT_CALTRANS_MAX_SOURCES,
CALTRANS_ANCHORS,
TFL_JAMCAM_URL,
TFL_IMAGE_ORIGIN,
DEFAULT_TFL_MAX_SOURCES,
LONDON_CENTER,
CCTV_SOURCE_FETCH_TIMEOUT_MS,
} from './constants.js';
import {
toFiniteNumber,
extractAustinCoords,
extractAustinCameraId,
extractAustinName,
extractAustinHeading,
isLikelyAustinCoordinate,
fallbackHeadingFromId,
rowArrayToObject,
prioritizeSources,
} from './normalize.js';
import { directionToHeading } from '../../../src/data/directionText.js'; import { directionToHeading } from '../../../src/data/directionText.js';
/** /**
* Fetch and parse Austin traffic camera records from the city Open Data portal. * Fetch and parse Austin traffic camera records from the city Open Data portal.
@@ -14,13 +37,18 @@ import { directionToHeading } from '../../../src/data/directionText.js';
export async function loadAustinSourcesFromOpenData() { export async function loadAustinSourcesFromOpenData() {
const endpoint = process.env.CCTV_AUSTIN_ROWS_URL || DEFAULT_AUSTIN_ROWS_URL; const endpoint = process.env.CCTV_AUSTIN_ROWS_URL || DEFAULT_AUSTIN_ROWS_URL;
try { try {
const resp = await fetch(endpoint, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); const resp = await fetch(endpoint, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) { if (!resp.ok) {
console.warn('[CCTV] Austin source download failed:', resp.status); console.warn('[CCTV] Austin source download failed:', resp.status);
return []; return [];
} }
const payload = await resp.json(); const payload = await resp.json();
const columns = Array.isArray(payload?.meta?.view?.columns) ? payload.meta.view.columns : []; const columns = Array.isArray(payload?.meta?.view?.columns)
? payload.meta.view.columns
: [];
const rows = Array.isArray(payload?.data) ? payload.data : []; const rows = Array.isArray(payload?.data) ? payload.data : [];
if (!columns.length || !rows.length) return []; if (!columns.length || !rows.length) return [];
@@ -35,7 +63,9 @@ export async function loadAustinSourcesFromOpenData() {
// REMOVED and VOID rows whose frame URLs never resolve — those cameras // REMOVED and VOID rows whose frame URLs never resolve — those cameras
// would render as permanent Street View / synthetic fallbacks. Tolerate // would render as permanent Street View / synthetic fallbacks. Tolerate
// a missing column (keep the row) so a schema change fails open. // a missing column (keep the row) so a schema change fails open.
const status = String(record.camera_status || '').trim().toUpperCase(); const status = String(record.camera_status || '')
.trim()
.toUpperCase();
if (status && status !== 'TURNED_ON') continue; if (status && status !== 'TURNED_ON') continue;
const { lat, lon } = extractAustinCoords(record); const { lat, lon } = extractAustinCoords(record);
@@ -44,7 +74,9 @@ export async function loadAustinSourcesFromOpenData() {
const extractedHeading = extractAustinHeading(record); const extractedHeading = extractAustinHeading(record);
const hasHeading = Number.isFinite(extractedHeading); const hasHeading = Number.isFinite(extractedHeading);
const headingDeg = hasHeading ? extractedHeading : fallbackHeadingFromId(cameraId); const headingDeg = hasHeading
? extractedHeading
: fallbackHeadingFromId(cameraId);
cameras.push({ cameras.push({
id: cameraId, id: cameraId,
name: extractAustinName(record, cameraId), name: extractAustinName(record, cameraId),
@@ -68,18 +100,29 @@ export async function loadAustinSourcesFromOpenData() {
}); });
} }
const unique = Array.from(new Map(cameras.map((camera) => [camera.id, camera])).values()); const unique = Array.from(
const maxRaw = Number(process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES); new Map(cameras.map((camera) => [camera.id, camera])).values(),
const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(300, Math.floor(maxRaw))) : DEFAULT_AUSTIN_MAX_SOURCES; );
const maxRaw = Number(
process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(300, Math.floor(maxRaw)))
: DEFAULT_AUSTIN_MAX_SOURCES;
const prioritized = prioritizeSources(unique, maxCount, [AUSTIN_DOWNTOWN]); const prioritized = prioritizeSources(unique, maxCount, [AUSTIN_DOWNTOWN]);
if (prioritized.length < unique.length) { if (prioritized.length < unique.length) {
console.log(`[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`); console.log(
`[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`,
);
} else { } else {
console.log('[CCTV] Loaded Austin camera sources:', prioritized.length); console.log('[CCTV] Loaded Austin camera sources:', prioritized.length);
} }
return prioritized; return prioritized;
} catch (error) { } catch (error) {
console.warn('[CCTV] Austin source download error:', error?.message || error); console.warn(
'[CCTV] Austin source download error:',
error?.message || error,
);
return []; return [];
} }
} }
@@ -96,7 +139,8 @@ export async function loadAustinSourcesFromOpenData() {
* @returns {Promise<Array<object>>} Normalized camera source objects. * @returns {Promise<Array<object>>} Normalized camera source objects.
*/ */
export async function loadCaltransSourcesFromOpenData() { export async function loadCaltransSourcesFromOpenData() {
const districtsRaw = process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS; const districtsRaw =
process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS;
const districts = String(districtsRaw) const districts = String(districtsRaw)
.split(',') .split(',')
.map((token) => Number(token.trim())) .map((token) => Number(token.trim()))
@@ -105,18 +149,24 @@ export async function loadCaltransSourcesFromOpenData() {
const settled = await Promise.allSettled( const settled = await Promise.allSettled(
districts.map(async (district) => { districts.map(async (district) => {
const resp = await fetch(CALTRANS_CCTV_URL(district), { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); const resp = await fetch(CALTRANS_CCTV_URL(district), {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) throw new Error(`D${district} HTTP ${resp.status}`); if (!resp.ok) throw new Error(`D${district} HTTP ${resp.status}`);
const payload = await resp.json(); const payload = await resp.json();
const rows = Array.isArray(payload?.data) ? payload.data : []; const rows = Array.isArray(payload?.data) ? payload.data : [];
return { district, rows }; return { district, rows };
}) }),
); );
const cameras = []; const cameras = [];
for (const result of settled) { for (const result of settled) {
if (result.status !== 'fulfilled') { if (result.status !== 'fulfilled') {
console.warn('[CCTV] Caltrans district fetch failed:', result.reason?.message || result.reason); console.warn(
'[CCTV] Caltrans district fetch failed:',
result.reason?.message || result.reason,
);
continue; continue;
} }
const { district, rows } = result.value; const { district, rows } = result.value;
@@ -135,13 +185,17 @@ export async function loadCaltransSourcesFromOpenData() {
const locationName = String(loc.locationName || '').trim(); const locationName = String(loc.locationName || '').trim();
// Leading token of locationName is the stable camera code ("TV102 -- I-580 : …"). // Leading token of locationName is the stable camera code ("TV102 -- I-580 : …").
const codeMatch = /^([A-Za-z0-9_-]+)\s*--/.exec(locationName); const codeMatch = /^([A-Za-z0-9_-]+)\s*--/.exec(locationName);
const code = (codeMatch ? codeMatch[1] : `x${cameras.length}`).toLowerCase(); const code = (
codeMatch ? codeMatch[1] : `x${cameras.length}`
).toLowerCase();
const cameraId = `ca-d${district}-${code}`; const cameraId = `ca-d${district}-${code}`;
// loc.direction is a dedicated field ("West", "South") → allow bare words. // loc.direction is a dedicated field ("West", "South") → allow bare words.
const heading = directionToHeading(loc.direction, true); const heading = directionToHeading(loc.direction, true);
const hasHeading = Number.isFinite(heading); const hasHeading = Number.isFinite(heading);
const label = locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') || `Caltrans D${district} ${code}`; const label =
locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') ||
`Caltrans D${district} ${code}`;
cameras.push({ cameras.push({
id: cameraId, id: cameraId,
name: loc.nearbyPlace ? `${label} (${loc.nearbyPlace})` : label, name: loc.nearbyPlace ? `${label} (${loc.nearbyPlace})` : label,
@@ -168,7 +222,9 @@ export async function loadCaltransSourcesFromOpenData() {
// so it must be right-ish on its own. // so it must be right-ish on its own.
groundElevationM: (() => { groundElevationM: (() => {
const ft = toFiniteNumber(loc.elevation, NaN); const ft = toFiniteNumber(loc.elevation, NaN);
return Number.isFinite(ft) ? Math.max(-100, Math.min(4000, ft * 0.3048)) : 150; return Number.isFinite(ft)
? Math.max(-100, Math.min(4000, ft * 0.3048))
: 150;
})(), })(),
feedType: 'image', feedType: 'image',
url: imageUrl, url: imageUrl,
@@ -179,10 +235,16 @@ export async function loadCaltransSourcesFromOpenData() {
} }
} }
const maxRaw = Number(process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES); const maxRaw = Number(
const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_CALTRANS_MAX_SOURCES; process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_CALTRANS_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, CALTRANS_ANCHORS); const prioritized = prioritizeSources(cameras, maxCount, CALTRANS_ANCHORS);
console.log(`[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`); console.log(
`[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`,
);
return prioritized; return prioritized;
} }
@@ -199,8 +261,13 @@ export async function loadCaltransSourcesFromOpenData() {
export async function loadTflSourcesFromOpenData() { export async function loadTflSourcesFromOpenData() {
try { try {
const appKey = String(process.env.TFL_APP_KEY || '').trim(); const appKey = String(process.env.TFL_APP_KEY || '').trim();
const url = appKey ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}` : TFL_JAMCAM_URL; const url = appKey
const resp = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}`
: TFL_JAMCAM_URL;
const resp = await fetch(url, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) { if (!resp.ok) {
console.warn('[CCTV] TfL JamCam download failed:', resp.status); console.warn('[CCTV] TfL JamCam download failed:', resp.status);
return []; return [];
@@ -251,10 +318,16 @@ export async function loadTflSourcesFromOpenData() {
}); });
} }
const maxRaw = Number(process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES); const maxRaw = Number(
const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_TFL_MAX_SOURCES; process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_TFL_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, [LONDON_CENTER]); const prioritized = prioritizeSources(cameras, maxCount, [LONDON_CENTER]);
console.log(`[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`); console.log(
`[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`,
);
return prioritized; return prioritized;
} catch (error) { } catch (error) {
console.warn('[CCTV] TfL JamCam download error:', error?.message || error); console.warn('[CCTV] TfL JamCam download error:', error?.message || error);
server/providers/radio.js
+5 -1
View File
@@ -1,7 +1,11 @@
import { createRadioProxyMiddleware } from './radio/catalog.js'; import { createRadioProxyMiddleware } from './radio/catalog.js';
export { createRadioProxyMiddleware }; export { createRadioProxyMiddleware };
export { isPublicRadioAddress } from './radio/transport.js'; export { isPublicRadioAddress } from './radio/transport.js';
export { normalizeRadioBrowserStation, publicRadioStation, publicRadioHttpsUrl } from './radio/stations.js'; export {
normalizeRadioBrowserStation,
publicRadioStation,
publicRadioHttpsUrl,
} from './radio/stations.js';
export function radioBrowserProxy() { export function radioBrowserProxy() {
const middleware = createRadioProxyMiddleware(); const middleware = createRadioProxyMiddleware();
const install = (server) => { const install = (server) => {
server/providers/radio/catalog.js
+192 -75
View File
@@ -2,25 +2,53 @@ import { lookup as lookupDns } from 'node:dns/promises';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { readResponseTextCapped } from '../common/http.js'; import { readResponseTextCapped } from '../common/http.js';
import { normalizeRadioBrowserStation, publicRadioStation, cleanRadioText } from './stations.js'; import {
import { radioMirrorOrigin, radioProxyDestination, resolveRadioProxyAddresses, fetchPinnedRadioResponse } from './transport.js'; normalizeRadioBrowserStation,
import { RADIO_DIRECTORY_CACHE_MS, RADIO_DIRECTORY_STALE_MS, RADIO_MIRROR_CACHE_MS, RADIO_FETCH_TIMEOUT_MS, RADIO_RESPONSE_MAX_BYTES, RADIO_DIRECTORY_LIMIT, RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES, RADIO_CATALOG_HEALTHY_MIN_STATIONS, RADIO_USER_AGENT, RADIO_UUID_RE, RADIO_FALLBACK_MIRRORS } from './constants.js'; publicRadioStation,
cleanRadioText,
} from './stations.js';
import {
radioMirrorOrigin,
radioProxyDestination,
resolveRadioProxyAddresses,
fetchPinnedRadioResponse,
} from './transport.js';
import {
RADIO_DIRECTORY_CACHE_MS,
RADIO_DIRECTORY_STALE_MS,
RADIO_MIRROR_CACHE_MS,
RADIO_FETCH_TIMEOUT_MS,
RADIO_RESPONSE_MAX_BYTES,
RADIO_DIRECTORY_LIMIT,
RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES,
RADIO_CATALOG_HEALTHY_MIN_STATIONS,
RADIO_USER_AGENT,
RADIO_UUID_RE,
RADIO_FALLBACK_MIRRORS,
} from './constants.js';
export async function mapRadioConcurrent(values, concurrency, mapper) { export async function mapRadioConcurrent(values, concurrency, mapper) {
const results = new Array(values.length); const results = new Array(values.length);
let cursor = 0; let cursor = 0;
const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { const workers = Array.from(
for (;;) { { length: Math.min(concurrency, values.length) },
const index = cursor++; async () => {
if (index >= values.length) return; for (;;) {
results[index] = await mapper(values[index], index); const index = cursor++;
} if (index >= values.length) return;
}); results[index] = await mapper(values[index], index);
}
},
);
await Promise.all(workers); await Promise.all(workers);
return results; return results;
} }
/** Create the testable Connect middleware backing `/api/radio`. */ /** Create the testable Connect middleware backing `/api/radio`. */
export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = lookupDns, now = Date.now } = {}) { export function createRadioProxyMiddleware({
fetchImpl = null,
lookupImpl = lookupDns,
now = Date.now,
} = {}) {
let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 }; let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 };
let mirrorPromise = null; let mirrorPromise = null;
let catalogCache = null; let catalogCache = null;
@@ -35,11 +63,15 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) { async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) {
const destination = radioProxyDestination(url); const destination = radioProxyDestination(url);
if (!destination) throw new Error('Radio Browser destination is not permitted'); if (!destination)
throw new Error('Radio Browser destination is not permitted');
const controller = new AbortController(); const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), RADIO_FETCH_TIMEOUT_MS); const timer = setTimeout(() => controller.abort(), RADIO_FETCH_TIMEOUT_MS);
try { try {
const addresses = await resolveRadioProxyAddresses(destination.hostname, lookupImpl); const addresses = await resolveRadioProxyAddresses(
destination.hostname,
lookupImpl,
);
const options = { const options = {
headers: { Accept: 'application/json', 'User-Agent': RADIO_USER_AGENT }, headers: { Accept: 'application/json', 'User-Agent': RADIO_USER_AGENT },
signal: controller.signal, signal: controller.signal,
@@ -49,10 +81,15 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
? await fetchImpl(destination.href, options) ? await fetchImpl(destination.href, options)
: await fetchPinnedRadioResponse(destination, options, addresses); : await fetchPinnedRadioResponse(destination, options, addresses);
if (response.status >= 300 && response.status < 400) { if (response.status >= 300 && response.status < 400) {
try { await response.body?.cancel?.(); } catch { /* no-op */ } try {
await response.body?.cancel?.();
} catch {
/* no-op */
}
throw new Error('Radio Browser redirects are refused'); throw new Error('Radio Browser redirects are refused');
} }
if (!response.ok) throw new Error(`Radio Browser returned ${response.status}`); if (!response.ok)
throw new Error(`Radio Browser returned ${response.status}`);
const text = await readResponseTextCapped(response, maxBytes); const text = await readResponseTextCapped(response, maxBytes);
return JSON.parse(text); return JSON.parse(text);
} finally { } finally {
@@ -61,20 +98,40 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
} }
async function mirrors() { async function mirrors() {
if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS) return mirrorCache.origins; if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS)
return mirrorCache.origins;
if (!mirrorPromise) { if (!mirrorPromise) {
mirrorPromise = (async () => { mirrorPromise = (async () => {
try { try {
const rows = await fetchJson('https://all.api.radio-browser.info/json/servers', 256 * 1024); const rows = await fetchJson(
const discovered = [...new Set((Array.isArray(rows) ? rows : []).map((row) => radioMirrorOrigin(row?.name)).filter(Boolean))]; 'https://all.api.radio-browser.info/json/servers',
256 * 1024,
);
const discovered = [
...new Set(
(Array.isArray(rows) ? rows : [])
.map((row) => radioMirrorOrigin(row?.name))
.filter(Boolean),
),
];
if (discovered.length) { if (discovered.length) {
mirrorCache = { origins: [...discovered, ...RADIO_FALLBACK_MIRRORS.filter((origin) => !discovered.includes(origin))], cachedAt: now() }; mirrorCache = {
origins: [
...discovered,
...RADIO_FALLBACK_MIRRORS.filter(
(origin) => !discovered.includes(origin),
),
],
cachedAt: now(),
};
} }
} catch { } catch {
mirrorCache = { ...mirrorCache, cachedAt: now() }; mirrorCache = { ...mirrorCache, cachedAt: now() };
} }
return mirrorCache.origins; return mirrorCache.origins;
})().finally(() => { mirrorPromise = null; }); })().finally(() => {
mirrorPromise = null;
});
} }
return mirrorPromise; return mirrorPromise;
} }
@@ -92,68 +149,113 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
} }
async function refreshCatalog() { async function refreshCatalog() {
const queries = [null, 'news', 'talk', 'weather', 'emergency', 'scanner', 'aviation', 'marine', 'traffic']; const queries = [
const outcomes = await mapRadioConcurrent(queries, 3, async (tag, index) => { null,
const params = new URLSearchParams({ 'news',
has_geo_info: 'true', 'talk',
is_https: 'true', 'weather',
hidebroken: 'true', 'emergency',
order: 'clickcount', 'scanner',
reverse: 'true', 'aviation',
limit: index === 0 ? '1800' : '220', 'marine',
}); 'traffic',
if (tag) params.set('tag', tag); ];
try { const outcomes = await mapRadioConcurrent(
const rows = await fetchPath(`/json/stations/search?${params}`); queries,
if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); 3,
if (!rows.every((row) => ( async (tag, index) => {
row const params = new URLSearchParams({
&& typeof row === 'object' has_geo_info: 'true',
&& !Array.isArray(row) is_https: 'true',
&& typeof row.stationuuid === 'string' hidebroken: 'true',
&& typeof row.name === 'string' order: 'clickcount',
&& (typeof row.url_resolved === 'string' || typeof row.url === 'string') reverse: 'true',
))) throw new Error('Radio Browser catalog contained a malformed station row'); limit: index === 0 ? '1800' : '220',
const stations = rows.map(normalizeRadioBrowserStation).filter(Boolean); });
const requestedTag = cleanRadioText(tag, 80) if (tag) params.set('tag', tag);
.toLocaleLowerCase() try {
.replace(/[_-]+/g, ' ') const rows = await fetchPath(`/json/stations/search?${params}`);
.replace(/\s+/g, ' ') if (!Array.isArray(rows))
.trim(); throw new Error('Radio Browser catalog payload was not an array');
const requestedTagCovered = !requestedTag || stations.some((station) => ( if (
station.tags.some((stationTag) => stationTag === requestedTag || stationTag.includes(requestedTag)) !rows.every(
)); (row) =>
return { row &&
// Query coverage is based on accepted rows, not merely a payload that typeof row === 'object' &&
// happens to match the upstream schema. Specialist responses must !Array.isArray(row) &&
// also contain an accepted station tagged for the requested category. typeof row.stationuuid === 'string' &&
succeeded: stations.length > 0 && requestedTagCovered, typeof row.name === 'string' &&
stations, (typeof row.url_resolved === 'string' ||
}; typeof row.url === 'string'),
} catch { )
return { succeeded: false, stations: [] }; )
} throw new Error(
}); 'Radio Browser catalog contained a malformed station row',
);
const stations = rows
.map(normalizeRadioBrowserStation)
.filter(Boolean);
const requestedTag = cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim();
const requestedTagCovered =
!requestedTag ||
stations.some((station) =>
station.tags.some(
(stationTag) =>
stationTag === requestedTag ||
stationTag.includes(requestedTag),
),
);
return {
// Query coverage is based on accepted rows, not merely a payload that
// happens to match the upstream schema. Specialist responses must
// also contain an accepted station tagged for the requested category.
succeeded: stations.length > 0 && requestedTagCovered,
stations,
};
} catch {
return { succeeded: false, stations: [] };
}
},
);
const resultSets = outcomes.map((outcome) => outcome.stations); const resultSets = outcomes.map((outcome) => outcome.stations);
const selected = []; const selected = [];
const seen = new Set(); const seen = new Set();
const take = (station) => { const take = (station) => {
if (!station || seen.has(station.id) || selected.length >= RADIO_DIRECTORY_LIMIT) return; if (
!station ||
seen.has(station.id) ||
selected.length >= RADIO_DIRECTORY_LIMIT
)
return;
seen.add(station.id); seen.add(station.id);
selected.push(station); selected.push(station);
}; };
// Seed specialist station-tag queries before popularity fill so operational // Seed specialist station-tag queries before popularity fill so operational
// categories remain represented even when global click charts skew musical. // categories remain represented even when global click charts skew musical.
for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take); for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take);
resultSets.flat().sort((a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name)).forEach(take); resultSets
.flat()
.sort(
(a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name),
)
.forEach(take);
const timestamp = now(); const timestamp = now();
const successfulQueries = outcomes.filter((outcome) => outcome.succeeded).length; const successfulQueries = outcomes.filter(
const broadQueryHealthy = outcomes[0].succeeded && outcomes[0].stations.length > 0; (outcome) => outcome.succeeded,
).length;
const broadQueryHealthy =
outcomes[0].succeeded && outcomes[0].stations.length > 0;
const healthReasons = []; const healthReasons = [];
if (!broadQueryHealthy) healthReasons.push('broad-query-unhealthy'); if (!broadQueryHealthy) healthReasons.push('broad-query-unhealthy');
if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES) healthReasons.push('query-coverage-below-policy'); if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES)
if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); healthReasons.push('query-coverage-below-policy');
if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS)
healthReasons.push('station-coverage-below-policy');
const degraded = healthReasons.length > 0; const degraded = healthReasons.length > 0;
const coverage = { const coverage = {
successfulQueries, successfulQueries,
@@ -171,14 +273,18 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
coverage, coverage,
}; };
if (degraded && catalogCache) { if (degraded && catalogCache) {
const error = new Error('Radio Browser catalog refresh did not meet health policy'); const error = new Error(
'Radio Browser catalog refresh did not meet health policy',
);
error.radioCatalogDegraded = true; error.radioCatalogDegraded = true;
error.radioDegradedReason = nextCatalog.degradedReason; error.radioDegradedReason = nextCatalog.degradedReason;
error.radioCoverage = coverage; error.radioCoverage = coverage;
throw error; throw error;
} }
if (degraded && !selected.length) { if (degraded && !selected.length) {
const error = new Error('Radio Browser catalog refresh returned no usable stations'); const error = new Error(
'Radio Browser catalog refresh returned no usable stations',
);
error.radioCatalogDegraded = true; error.radioCatalogDegraded = true;
error.radioDegradedReason = nextCatalog.degradedReason; error.radioDegradedReason = nextCatalog.degradedReason;
error.radioCoverage = coverage; error.radioCoverage = coverage;
@@ -197,16 +303,24 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
} }
async function getCatalog() { async function getCatalog() {
if (catalogCache && now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS) { if (
catalogCache &&
now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS
) {
return { ...catalogCache, stale: false }; return { ...catalogCache, stale: false };
} }
if (!refreshPromise) { if (!refreshPromise) {
refreshPromise = refreshCatalog().finally(() => { refreshPromise = null; }); refreshPromise = refreshCatalog().finally(() => {
refreshPromise = null;
});
} }
try { try {
return { ...await refreshPromise, stale: false }; return { ...(await refreshPromise), stale: false };
} catch (error) { } catch (error) {
if (catalogCache && now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS) { if (
catalogCache &&
now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS
) {
return { return {
...catalogCache, ...catalogCache,
stale: true, stale: true,
@@ -220,7 +334,10 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look
} }
function sendJson(res, status, body) { function sendJson(res, status, body) {
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }); res.writeHead(status, {
'Content-Type': 'application/json; charset=utf-8',
'Cache-Control': 'no-store',
});
res.end(JSON.stringify(body)); res.end(JSON.stringify(body));
} }
server/providers/radio/constants.js
+7 -3
View File
@@ -5,9 +5,13 @@ export const RADIO_FETCH_TIMEOUT_MS = 12_000;
export const RADIO_RESPONSE_MAX_BYTES = 4 * 1024 * 1024; export const RADIO_RESPONSE_MAX_BYTES = 4 * 1024 * 1024;
export const RADIO_DIRECTORY_LIMIT = 750; export const RADIO_DIRECTORY_LIMIT = 750;
export const RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES = 5; export const RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES = 5;
export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(RADIO_DIRECTORY_LIMIT / 2); export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; RADIO_DIRECTORY_LIMIT / 2,
export const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; );
export const RADIO_USER_AGENT =
'GodsEyeView/1.0 (Radio Browser directory client)';
export const RADIO_UUID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const RADIO_FALLBACK_MIRRORS = Object.freeze([ export const RADIO_FALLBACK_MIRRORS = Object.freeze([
'https://de1.api.radio-browser.info', 'https://de1.api.radio-browser.info',
'https://de2.api.radio-browser.info', 'https://de2.api.radio-browser.info',
server/providers/radio/stations.js
+74 -38
View File
@@ -1,40 +1,56 @@
import { RADIO_UUID_RE } from './constants.js'; import { RADIO_UUID_RE } from './constants.js';
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js'; import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js';
export function cleanRadioText(value, maxLength) { export function cleanRadioText(value, maxLength) {
return String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, maxLength).trim(); return String(value ?? '')
.replace(/[\u0000-\u001f\u007f]/g, ' ')
.replace(/\s+/g, ' ')
.trim()
.slice(0, maxLength)
.trim();
} }
export function isNonGlobalIpv4(hostname) { export function isNonGlobalIpv4(hostname) {
const pieces = hostname.split('.'); const pieces = hostname.split('.');
if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) return false; if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece)))
return false;
const values = pieces.map(Number); const values = pieces.map(Number);
if (values.some((value) => value > 255)) return true; if (values.some((value) => value > 255)) return true;
const [a, b, c] = values; const [a, b, c] = values;
return a === 0 || a === 10 || a === 127 || a >= 224 return (
|| (a === 100 && b >= 64 && b <= 127) a === 0 ||
|| (a === 169 && b === 254) a === 10 ||
|| (a === 172 && b >= 16 && b <= 31) a === 127 ||
|| (a === 192 && b === 0) a >= 224 ||
|| (a === 192 && b === 88 && c === 99) (a === 100 && b >= 64 && b <= 127) ||
|| (a === 192 && b === 168) (a === 169 && b === 254) ||
|| (a === 198 && (b === 18 || b === 19)) (a === 172 && b >= 16 && b <= 31) ||
|| (a === 198 && b === 51 && c === 100) (a === 192 && b === 0) ||
|| (a === 203 && b === 0 && c === 113); (a === 192 && b === 88 && c === 99) ||
(a === 192 && b === 168) ||
(a === 198 && (b === 18 || b === 19)) ||
(a === 198 && b === 51 && c === 100) ||
(a === 203 && b === 0 && c === 113)
);
} }
/** Return a normalized public HTTPS URL, or null for local/private targets. */ /** Return a normalized public HTTPS URL, or null for local/private targets. */
export function publicRadioHttpsUrl(value) { export function publicRadioHttpsUrl(value) {
try { try {
const url = new URL(String(value ?? '')); const url = new URL(String(value ?? ''));
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, ''); const hostname = url.hostname
if (url.protocol !== 'https:' || url.username || url.password || !hostname) return null; .toLowerCase()
.replace(/^\[|\]$/g, '')
.replace(/\.$/, '');
if (url.protocol !== 'https:' || url.username || url.password || !hostname)
return null;
if ( if (
hostname === 'localhost' hostname === 'localhost' ||
|| hostname.endsWith('.localhost') hostname.endsWith('.localhost') ||
|| hostname.endsWith('.local') hostname.endsWith('.local') ||
|| isNonGlobalIpv4(hostname) isNonGlobalIpv4(hostname) ||
|| hostname.includes(':') hostname.includes(':')
) return null; )
return null;
url.hash = ''; url.hash = '';
return url.href; return url.href;
} catch { } catch {
@@ -45,25 +61,40 @@ export function publicRadioHttpsUrl(value) {
/** Normalize one Radio Browser station and omit favicons and unsafe streams. */ /** Normalize one Radio Browser station and omit favicons and unsafe streams. */
export function normalizeRadioBrowserStation(raw) { export function normalizeRadioBrowserStation(raw) {
const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase();
const lat = raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); const lat =
const lon = raw?.geo_long === null || raw?.geo_long === '' ? null : Number(raw?.geo_long); raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat);
const lon =
raw?.geo_long === null || raw?.geo_long === ''
? null
: Number(raw?.geo_long);
const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); const codec = cleanRadioText(raw?.codec, 16).toUpperCase();
const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url); const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url);
if ( if (
!RADIO_UUID_RE.test(id) !RADIO_UUID_RE.test(id) ||
|| Number(raw?.lastcheckok) !== 1 Number(raw?.lastcheckok) !== 1 ||
|| Number(raw?.hls) === 1 Number(raw?.hls) === 1 ||
|| !Number.isFinite(lat) || lat < -90 || lat > 90 !Number.isFinite(lat) ||
|| !Number.isFinite(lon) || lon < -180 || lon > 180 lat < -90 ||
|| !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) lat > 90 ||
|| !streamUrl !Number.isFinite(lon) ||
) return null; lon < -180 ||
lon > 180 ||
!/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) ||
!streamUrl
)
return null;
const name = cleanRadioText(raw?.name, 140); const name = cleanRadioText(raw?.name, 140);
if (!name) return null; if (!name) return null;
const tags = String(raw?.tags ?? '') const tags = String(raw?.tags ?? '')
.split(',') .split(',')
.map((tag) => cleanRadioText(tag, 80).toLocaleLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim()) .map((tag) =>
cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim(),
)
.filter(Boolean) .filter(Boolean)
.filter((tag, index, all) => all.indexOf(tag) === index) .filter((tag, index, all) => all.indexOf(tag) === index)
.slice(0, 24); .slice(0, 24);
@@ -74,9 +105,10 @@ export function normalizeRadioBrowserStation(raw) {
.slice(0, 8); .slice(0, 8);
const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase();
const normalizedCode = normalizeRadioCountryInput(rawCountryCode); const normalizedCode = normalizeRadioCountryInput(rawCountryCode);
const normalizedCountry = normalizedCode.valid && !normalizedCode.empty const normalizedCountry =
? normalizedCode normalizedCode.valid && !normalizedCode.empty
: normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); ? normalizedCode
: normalizeRadioCountryInput(cleanRadioText(raw?.country, 80));
const bitrate = Number(raw?.bitrate); const bitrate = Number(raw?.bitrate);
return { return {
id, id,
@@ -88,13 +120,17 @@ export function normalizeRadioBrowserStation(raw) {
tags, tags,
languages, languages,
state: cleanRadioText(raw?.state, 80), state: cleanRadioText(raw?.state, 80),
country: normalizedCountry.valid && !normalizedCountry.empty country:
? normalizedCountry.name normalizedCountry.valid && !normalizedCountry.empty
: cleanRadioText(raw?.country, 80), ? normalizedCountry.name
: cleanRadioText(raw?.country, 80),
countryCode: normalizedCountry.valid ? normalizedCountry.code : '', countryCode: normalizedCountry.valid ? normalizedCountry.code : '',
metadataTrust: 'untrusted-community', metadataTrust: 'untrusted-community',
codec, codec,
bitrate: Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 ? bitrate : null, bitrate:
Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024
? bitrate
: null,
clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)),
}; };
} }
server/providers/radio/transport.js
+88 -50
View File
@@ -2,44 +2,67 @@ import https from 'node:https';
import { Readable } from 'node:stream'; import { Readable } from 'node:stream';
import { isNonGlobalIpv4 } from './stations.js'; import { isNonGlobalIpv4 } from './stations.js';
export function radioMirrorOrigin(value) { export function radioMirrorOrigin(value) {
const hostname = String(value ?? '').toLowerCase().replace(/\.$/, ''); const hostname = String(value ?? '')
.toLowerCase()
.replace(/\.$/, '');
if (!/^[a-z0-9-]+\.api\.radio-browser\.info$/.test(hostname)) return null; if (!/^[a-z0-9-]+\.api\.radio-browser\.info$/.test(hostname)) return null;
return `https://${hostname}`; return `https://${hostname}`;
} }
/** Return whether a resolved Radio Browser address is safe for an outbound request. */ /** Return whether a resolved Radio Browser address is safe for an outbound request. */
export function isPublicRadioAddress(value) { export function isPublicRadioAddress(value) {
const address = String(value ?? '').trim().toLowerCase().replace(/^\[|\]$/g, ''); const address = String(value ?? '')
.trim()
.toLowerCase()
.replace(/^\[|\]$/g, '');
if (!address) return false; if (!address) return false;
if (!address.includes(':')) { if (!address.includes(':')) {
const ipv4 = address.split('.'); const ipv4 = address.split('.');
return ipv4.length === 4 return (
&& ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) ipv4.length === 4 &&
&& !isNonGlobalIpv4(address); ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) &&
!isNonGlobalIpv4(address)
);
} }
const pieces = address.split('::'); const pieces = address.split('::');
if (pieces.length > 2) return false; if (pieces.length > 2) return false;
const left = pieces[0] ? pieces[0].split(':') : []; const left = pieces[0] ? pieces[0].split(':') : [];
const right = pieces[1] ? pieces[1].split(':') : []; const right = pieces[1] ? pieces[1].split(':') : [];
const missing = 8 - left.length - right.length; const missing = 8 - left.length - right.length;
if ((pieces.length === 1 && missing !== 0) || (pieces.length === 2 && missing < 1)) return false; if (
(pieces.length === 1 && missing !== 0) ||
(pieces.length === 2 && missing < 1)
)
return false;
const groups = [...left, ...Array(Math.max(0, missing)).fill('0'), ...right]; const groups = [...left, ...Array(Math.max(0, missing)).fill('0'), ...right];
if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) return false; if (
const numeric = groups.reduce((total, group) => (total << 16n) | BigInt(`0x${group}`), 0n); groups.length !== 8 ||
const inCidr = (base, prefix) => { groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))
const shift = 128n - BigInt(prefix); )
return (numeric >> shift) === (base >> shift); return false;
}; const numeric = groups.reduce(
const base = (text) => text.split(':').reduce( (total, group) => (total << 16n) | BigInt(`0x${group}`),
(total, group) => (total << 16n) | BigInt(`0x${group || '0'}`),
0n, 0n,
); );
const inCidr = (base, prefix) => {
const shift = 128n - BigInt(prefix);
return numeric >> shift === base >> shift;
};
const base = (text) =>
text
.split(':')
.reduce(
(total, group) => (total << 16n) | BigInt(`0x${group || '0'}`),
0n,
);
const cidr = (text, prefix) => inCidr(base(text), prefix); const cidr = (text, prefix) => inCidr(base(text), prefix);
return cidr('2000:0:0:0:0:0:0:0', 3) return (
&& !cidr('2001:0:0:0:0:0:0:0', 23) cidr('2000:0:0:0:0:0:0:0', 3) &&
&& !cidr('2001:db8:0:0:0:0:0:0', 32) !cidr('2001:0:0:0:0:0:0:0', 23) &&
&& !cidr('2002:0:0:0:0:0:0:0', 16) !cidr('2001:db8:0:0:0:0:0:0', 32) &&
&& !cidr('3fff:0:0:0:0:0:0:0', 20); !cidr('2002:0:0:0:0:0:0:0', 16) &&
!cidr('3fff:0:0:0:0:0:0:0', 20)
);
} }
export function radioProxyDestination(value) { export function radioProxyDestination(value) {
@@ -51,16 +74,18 @@ export function radioProxyDestination(value) {
} }
const origin = radioMirrorOrigin(url.hostname); const origin = radioMirrorOrigin(url.hostname);
if ( if (
!origin !origin ||
|| url.origin !== origin url.origin !== origin ||
|| url.username url.username ||
|| url.password url.password ||
|| url.port url.port ||
|| url.hash url.hash
) return null; )
const discovery = url.hostname.toLowerCase() === 'all.api.radio-browser.info' return null;
&& url.pathname === '/json/servers' const discovery =
&& !url.search; url.hostname.toLowerCase() === 'all.api.radio-browser.info' &&
url.pathname === '/json/servers' &&
!url.search;
const directory = url.pathname === '/json/stations/search'; const directory = url.pathname === '/json/stations/search';
const click = /^\/json\/url\/[0-9a-f-]+$/i.test(url.pathname) && !url.search; const click = /^\/json\/url\/[0-9a-f-]+$/i.test(url.pathname) && !url.search;
return discovery || directory || click ? url : null; return discovery || directory || click ? url : null;
@@ -70,9 +95,15 @@ export async function resolveRadioProxyAddresses(hostname, lookupImpl) {
const resolved = await lookupImpl(hostname, { all: true, verbatim: true }); const resolved = await lookupImpl(hostname, { all: true, verbatim: true });
const rows = Array.isArray(resolved) ? resolved : [resolved]; const rows = Array.isArray(resolved) ? resolved : [resolved];
const addresses = rows const addresses = rows
.map((row) => ({ address: String(row?.address || ''), family: Number(row?.family) || undefined })) .map((row) => ({
address: String(row?.address || ''),
family: Number(row?.family) || undefined,
}))
.filter((row) => row.address); .filter((row) => row.address);
if (!addresses.length || addresses.some((row) => !isPublicRadioAddress(row.address))) { if (
!addresses.length ||
addresses.some((row) => !isPublicRadioAddress(row.address))
) {
throw new Error('Radio Browser resolved to a forbidden address'); throw new Error('Radio Browser resolved to a forbidden address');
} }
return addresses; return addresses;
@@ -81,26 +112,33 @@ export async function resolveRadioProxyAddresses(hostname, lookupImpl) {
export function fetchPinnedRadioResponse(url, options, addresses) { export function fetchPinnedRadioResponse(url, options, addresses) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const address = addresses[0]; const address = addresses[0];
const request = https.request(url, { const request = https.request(
method: 'GET', url,
headers: options.headers, {
signal: options.signal, method: 'GET',
lookup(_hostname, lookupOptions, callback) { headers: options.headers,
if (lookupOptions?.all) callback(null, addresses); signal: options.signal,
else callback(null, address.address, address.family); lookup(_hostname, lookupOptions, callback) {
if (lookupOptions?.all) callback(null, addresses);
else callback(null, address.address, address.family);
},
}, },
}, (response) => { (response) => {
const headers = new Headers(); const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) { for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value)) value.forEach((item) => headers.append(name, item)); if (Array.isArray(value))
else if (value !== undefined) headers.set(name, String(value)); value.forEach((item) => headers.append(name, item));
} else if (value !== undefined) headers.set(name, String(value));
resolve(new Response(Readable.toWeb(response), { }
status: response.statusCode || 500, resolve(
statusText: response.statusMessage || '', new Response(Readable.toWeb(response), {
headers, status: response.statusCode || 500,
})); statusText: response.statusMessage || '',
}); headers,
}),
);
},
);
request.on('error', reject); request.on('error', reject);
request.end(); request.end();
}); });
src/tooling/mediaProviders.test.mjs
+44 -11
View File
@@ -9,11 +9,21 @@ import { localProviderPlugins } from '../../server/providers/local.js';
function install(plugin) { function install(plugin) {
let handler; let handler;
plugin.configureServer({ middlewares: { use(_route, fn) { handler = fn; } } }); plugin.configureServer({
middlewares: {
use(_route, fn) {
handler = fn;
},
},
});
return async (url) => { return async (url) => {
const res = { const res = {
writeHead(status, headers) { Object.assign(this, { status, headers }); }, writeHead(status, headers) {
end(body) { this.body = body; }, Object.assign(this, { status, headers });
},
end(body) {
this.body = body;
},
}; };
await handler({ url, method: 'GET' }, res); await handler({ url, method: 'GET' }, res);
return res; return res;
@@ -24,14 +34,29 @@ function fixture(t, id) {
const root = mkdtempSync(path.join(tmpdir(), 'gev-cctv-')); const root = mkdtempSync(path.join(tmpdir(), 'gev-cctv-'));
t.after(() => rmSync(root, { recursive: true, force: true })); t.after(() => rmSync(root, { recursive: true, force: true }));
mkdirSync(path.join(root, 'config')); mkdirSync(path.join(root, 'config'));
writeFileSync(path.join(root, 'config/cctv_sources.austin.json'), JSON.stringify([ writeFileSync(
{ id, name: '<Camera & test>', lat: 30.27, lon: -97.74, feedType: 'video' }, path.join(root, 'config/cctv_sources.austin.json'),
])); JSON.stringify([
{
id,
name: '<Camera & test>',
lat: 30.27,
lon: -97.74,
feedType: 'video',
},
]),
);
return root; return root;
} }
function isolate(t) { function isolate(t) {
for (const name of ['CCTV_SOURCES_FILE', 'CCTV_SOURCES_JSON', 'CCTV_FORCE_AUSTIN', 'GOOGLE_MAPS_SERVER_API_KEY', 'GOOGLE_MAPS_API_KEY']) { for (const name of [
'CCTV_SOURCES_FILE',
'CCTV_SOURCES_JSON',
'CCTV_FORCE_AUSTIN',
'GOOGLE_MAPS_SERVER_API_KEY',
'GOOGLE_MAPS_API_KEY',
]) {
const previous = process.env[name]; const previous = process.env[name];
delete process.env[name]; delete process.env[name];
t.after(() => { t.after(() => {
@@ -39,7 +64,9 @@ function isolate(t) {
else process.env[name] = previous; else process.env[name] = previous;
}); });
} }
t.mock.method(globalThis, 'fetch', () => { throw Error('fixture must not fetch'); }); t.mock.method(globalThis, 'fetch', () => {
throw Error('fixture must not fetch');
});
} }
test('CCTV instances resolve their own application source root and isolate catalogs and health', async (t) => { test('CCTV instances resolve their own application source root and isolate catalogs and health', async (t) => {
@@ -49,8 +76,14 @@ test('CCTV instances resolve their own application source root and isolate catal
const [a, b] = await Promise.all([first('/sources'), second('/sources')]); const [a, b] = await Promise.all([first('/sources'), second('/sources')]);
assert.equal(a.status, 200); assert.equal(a.status, 200);
assert.equal(b.status, 200); assert.equal(b.status, 200);
assert.deepEqual(JSON.parse(a.body).sources.map(s => s.id), ['first']); assert.deepEqual(
assert.deepEqual(JSON.parse(b.body).sources.map(s => s.id), ['second']); JSON.parse(a.body).sources.map((s) => s.id),
['first'],
);
assert.deepEqual(
JSON.parse(b.body).sources.map((s) => s.id),
['second'],
);
const stream = JSON.parse((await first('/stream/first')).body); const stream = JSON.parse((await first('/stream/first')).body);
assert.equal(stream.feedType, 'mp4'); assert.equal(stream.feedType, 'mp4');
assert.equal(stream.mediaUrl, '/api/cctv/media/first'); assert.equal(stream.mediaUrl, '/api/cctv/media/first');
@@ -66,6 +99,6 @@ test('composition creates exactly one CCTV and radio provider without acquisitio
isolate(t); isolate(t);
const plugins = localProviderPlugins(); const plugins = localProviderPlugins();
for (const factory of [cctvProxy, radioBrowserProxy]) { for (const factory of [cctvProxy, radioBrowserProxy]) {
assert.equal(plugins.filter(p => p.name === factory().name).length, 1); assert.equal(plugins.filter((p) => p.name === factory().name).length, 1);
} }
}); });