From 79b4929b88dce2360b468a639cc946a35bac12d4 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Sat, 12 Sep 2026 12:09:02 -0700 Subject: [PATCH 1/2] refactor: extract CCTV media and radio directory providers --- CHANGELOG.md | 4 + docs/CURRENT-STATE.md | 10 + package.json | 10 +- scripts/package-boundaries.json | 31 + server/providers/cctv.js | 310 +++++ server/providers/cctv/catalog.js | 140 ++ server/providers/cctv/constants.js | 38 + server/providers/cctv/media.js | 180 +++ server/providers/cctv/normalize.js | 384 ++++++ server/providers/cctv/sources.js | 263 ++++ server/providers/local.js | 1834 +-------------------------- server/providers/radio.js | 15 + server/providers/radio/catalog.js | 277 ++++ server/providers/radio/constants.js | 15 + server/providers/radio/stations.js | 119 ++ server/providers/radio/transport.js | 107 ++ src/tooling/mediaProviders.test.mjs | 71 ++ 17 files changed, 1977 insertions(+), 1831 deletions(-) create mode 100644 server/providers/cctv.js create mode 100644 server/providers/cctv/catalog.js create mode 100644 server/providers/cctv/constants.js create mode 100644 server/providers/cctv/media.js create mode 100644 server/providers/cctv/normalize.js create mode 100644 server/providers/cctv/sources.js create mode 100644 server/providers/radio.js create mode 100644 server/providers/radio/catalog.js create mode 100644 server/providers/radio/constants.js create mode 100644 server/providers/radio/stations.js create mode 100644 server/providers/radio/transport.js create mode 100644 src/tooling/mediaProviders.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c70d9889..2b7500b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Extract CCTV catalog/media and Radio Browser directory providers into focused + Node modules, preserving their routes and policies and isolating CCTV catalogs + by provider instance and application root. + - Simplify POWER UP to one Google Maps entry. Keep the optional server key available through environment configuration without a second setup row or missing-key reminder. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 52a019edf..10f616517 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,15 @@ # God's Eye View Current State +## CCTV and radio provider modules + +CCTV catalog acquisition, source normalization and frame/media delivery now live +in separate modules. Each CCTV factory owns its catalog and health state; its +`sourceRoot` option resolves relative source files against the application root. +Radio Browser station normalization, restricted outbound transport and directory +caching are separate modules. Node-only package entries expose both provider +factories. Routes, payloads, fallback behavior and existing dev/preview hook +registration are preserved; browser rendering is unchanged. + ## Terrain, traffic, fire and bike-share provider modules Local composition now imports separate Node modules for Re:Earth heights, diff --git a/package.json b/package.json index b93c900ff..3d729940c 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "gods-eye-view", "private": true, "version": "0.1.1", - "description": "A real-time intelligence console for planet Earth — photorealistic 3D globe, live aircraft/ships/satellites/earthquakes/CCTV, and hands-free voice control. Runs in a browser.", + "description": "A real-time intelligence console for planet Earth \u2014 photorealistic 3D globe, live aircraft/ships/satellites/earthquakes/CCTV, and hands-free voice control. Runs in a browser.", "type": "module", "license": "MIT", "author": "Bilawal Sidhu", @@ -96,6 +96,12 @@ "./server/providers/gbfs": { "node": "./server/providers/gbfs.js" }, - "./sources/gbfs": "./src/data/gbfsSource.js" + "./sources/gbfs": "./src/data/gbfsSource.js", + "./server/providers/cctv": { + "node": "./server/providers/cctv.js" + }, + "./server/providers/radio": { + "node": "./server/providers/radio.js" + } } } diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 06d90f55f..504ff55e5 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -131,5 +131,36 @@ "exports": ["./sources/gbfs"], "modules": ["src/data/gbfsSource.js"], "external": [] + }, + "cctv-provider": { + "runtime": "node", + "exports": ["./server/providers/cctv"], + "modules": [ + "server/providers/cctv.js", + "server/providers/cctv/catalog.js", + "server/providers/cctv/constants.js", + "server/providers/cctv/media.js", + "server/providers/cctv/normalize.js", + "server/providers/cctv/sources.js", + "src/data/directionText.js", + "server/providers/common/geo.js", + "server/providers/places/google-key.js", + "scripts/google-server-key.mjs" + ], + "external": [] + }, + "radio-provider": { + "runtime": "node", + "exports": ["./server/providers/radio"], + "modules": [ + "server/providers/radio.js", + "server/providers/radio/catalog.js", + "server/providers/radio/constants.js", + "server/providers/radio/stations.js", + "server/providers/radio/transport.js", + "src/data/radioCountry.js", + "server/providers/common/http.js" + ], + "external": [] } } diff --git a/server/providers/cctv.js b/server/providers/cctv.js new file mode 100644 index 000000000..e884a9d72 --- /dev/null +++ b/server/providers/cctv.js @@ -0,0 +1,310 @@ +import { createCctvCatalog } from './cctv/catalog.js'; +import { normalizeFeedType, isVideoFeedType, toFiniteNumber } from './cctv/normalize.js'; +import { buildSyntheticCctvSvg, proxyMediaResponse, fetchCctvImageFromUpstream } from './cctv/media.js'; +import { CCTV_FRAME_FETCH_TIMEOUT_MS } from './cctv/constants.js'; +import { googleServerApiKey } from './places/google-key.js'; +export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; +/** + * Vite plugin: CCTV camera proxy with source registry, frame/media serving, + * fallback chain (upstream -> Street View -> synthetic SVG), and health tracking. + * + * Endpoints: + * GET /api/cctv/sources — list all registered camera sources + * GET /api/cctv/health — per-camera health/status report + * GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera + * GET /api/cctv/media/:id — proxy live video/image media from upstream + * GET /api/cctv/frame/:id — single frame with fallback chain + * + * @returns {import('vite').Plugin} + */ +export function cctvProxy({ sourceRoot = process.cwd() } = {}) { + const getCctvSources = createCctvCatalog({ sourceRoot }); + /** @type {Map} */ + const health = new Map(); + /** Cap on health map entries to prevent unbounded growth. Sized to cover the + * full served catalog (CCTV_MAX_SOURCES hard-bounds at 1200) so health/status + * observability isn't silently evicted for a default 800-camera catalog. */ + const HEALTH_MAX_ENTRIES = 1200; + + /** Update the health entry for a camera, evicting the oldest entry if at capacity. */ + const setHealth = (cameraId, patch) => { + // Evict oldest entries if the health map grows beyond the cap + if (!health.has(cameraId) && health.size >= HEALTH_MAX_ENTRIES) { + const oldest = health.keys().next().value; + health.delete(oldest); + } + const prev = health.get(cameraId) || {}; + health.set(cameraId, { + id: cameraId, + status: patch.status || prev.status || 'unknown', + sourceKind: patch.sourceKind || prev.sourceKind || 'unknown', + label: patch.label || prev.label || '', + message: patch.message || prev.message || '', + updatedAt: Date.now(), + }); + }; + + /** Snapshot all camera health entries as an array. */ + const listHealth = () => Array.from(health.values()); + + /** Build a JSON payload describing stream info (feedType, URLs) for a camera. */ + const buildStreamPayload = (source, cameraId) => { + const feedType = normalizeFeedType(source?.feedType || 'image'); + return { + id: cameraId, + feedType, + mediaUrl: isVideoFeedType(feedType) + ? `/api/cctv/media/${encodeURIComponent(cameraId)}` + : null, + frameUrl: `/api/cctv/frame/${encodeURIComponent(cameraId)}`, + provider: source?.provider || '', + sourceKind: source?.sourceKind || (source?.url ? 'configured' : 'fallback'), + }; + }; + + /** + * Fetch a Google Street View static image as a fallback frame. Server-side + * call, never reaches the browser — prefers GOOGLE_MAPS_SERVER_API_KEY + * (#33: a key scoped to Street View Static/Places, restricted by server IP + * rather than HTTP referrer) and falls back to the browser-exposed + * GOOGLE_MAPS_API_KEY for setups that haven't split the two yet. + */ + const streetViewFallback = async ({ lat, lon, heading, fov, pitch }) => { + const streetViewKey = googleServerApiKey(); + if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon)) return null; + try { + const sv = new URL('https://maps.googleapis.com/maps/api/streetview'); + sv.searchParams.set('size', '960x540'); + sv.searchParams.set('location', `${lat},${lon}`); + sv.searchParams.set('heading', String(Number.isFinite(heading) ? heading : 0)); + sv.searchParams.set('fov', String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80)); + sv.searchParams.set('pitch', String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0)); + sv.searchParams.set('source', 'outdoor'); + sv.searchParams.set('return_error_code', 'true'); + sv.searchParams.set('key', streetViewKey); + + const svResp = await fetch(sv.toString(), { + headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }, + signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS), + }); + const svType = svResp.headers.get('content-type') || ''; + if (!svResp.ok || !svType.startsWith('image/')) return null; + + return { + ok: true, + body: Buffer.from(await svResp.arrayBuffer()), + contentType: svType, + }; + } catch { + return null; + } + }; + + return { + name: 'cctv-proxy', + configureServer(server) { + server.middlewares.use('/api/cctv', async (req, res) => { + try { + const sources = await getCctvSources(); + const sourceById = new Map(sources.map((source) => [source.id, source])); + const url = new URL(req.url || '/', 'http://localhost'); + + if (url.pathname === '/sources') { + const body = { + sources: sources.map((source) => ({ + id: source.id, + name: source.name, + city: source.city, + cityId: source.cityId, + provider: source.provider, + lat: source.lat, + lon: source.lon, + headingDeg: source.headingDeg, + headingConfidence: source.headingConfidence || '', + pitchDeg: source.pitchDeg, + fovDeg: source.fovDeg, + rangeM: source.rangeM, + mountHeightM: source.mountHeightM, + groundElevationM: source.groundElevationM, + feedType: normalizeFeedType(source.feedType), + sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), + poseSource: source.poseSource, + license: source.license, + })), + }; + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(body)); + return; + } + + if (url.pathname === '/health') { + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ cameras: listHealth() })); + return; + } + + if (url.pathname.startsWith('/stream/')) { + const cameraId = decodeURIComponent(url.pathname.replace('/stream/', '').trim()) || 'camera'; + const source = sourceById.get(cameraId); + const payload = buildStreamPayload(source, cameraId); + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(payload)); + return; + } + + if (url.pathname.startsWith('/media/')) { + const cameraId = decodeURIComponent(url.pathname.replace('/media/', '').trim()) || 'camera'; + const source = sourceById.get(cameraId); + const mediaUrl = source?.url || ''; + const feedType = normalizeFeedType(source?.feedType || 'image'); + + if (!mediaUrl || !/^https?:\/\//i.test(mediaUrl)) { + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'fallback', + label: source?.provider || 'No upstream URL', + message: 'No stream URL configured', + }); + res.writeHead(404, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'No media URL configured for this camera' })); + return; + } + + try { + const upstreamHeaders = { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }; + const requestRange = req.headers?.range; + if (requestRange) upstreamHeaders.Range = requestRange; + const upstream = await fetch(mediaUrl, { + headers: upstreamHeaders, + }); + const contentType = upstream.headers.get('content-type') || ''; + if (!upstream.ok) { + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'upstream', + label: source?.provider || 'Configured source', + message: `Upstream HTTP ${upstream.status}`, + }); + res.writeHead(upstream.status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: `Upstream returned ${upstream.status}` })); + return; + } + + if (isVideoFeedType(feedType) && !(contentType.startsWith('video/') || contentType.includes('mpegurl'))) { + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'upstream', + label: source?.provider || 'Configured source', + message: `Unexpected media type ${contentType || 'unknown'}`, + }); + } else { + setHealth(cameraId, { + status: 'ok', + sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot', + label: source?.provider || 'Configured source', + message: isVideoFeedType(feedType) ? 'Live stream connected' : 'Snapshot feed connected', + }); + } + + await proxyMediaResponse(res, upstream, { + sourceHeader: isVideoFeedType(feedType) ? 'live-media' : 'upstream-image', + }); + return; + } catch (error) { + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'upstream', + label: source?.provider || 'Configured source', + message: error?.message || 'Media fetch failed', + }); + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Media proxy failed' })); + return; + } + } + + if (!url.pathname.startsWith('/frame/')) { + res.writeHead(404, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'not found' })); + return; + } + + const cameraId = decodeURIComponent(url.pathname.replace('/frame/', '').trim()) || 'camera'; + const source = sourceById.get(cameraId); + const label = url.searchParams.get('label') || source?.name || cameraId; + const city = url.searchParams.get('city') || source?.city || ''; + const lat = Number(url.searchParams.get('lat') || source?.lat); + const lon = Number(url.searchParams.get('lon') || source?.lon); + const heading = Number(url.searchParams.get('heading') || source?.headingDeg); + const fov = Number(url.searchParams.get('fov') || source?.fovDeg); + const pitch = Number(url.searchParams.get('pitch') || source?.pitchDeg); + + // Only use server-registered upstream URLs — never accept client-supplied URLs + // (prevents SSRF via ?upstream= query parameter) + const upstreamCandidate = + source?.snapshotUrl + || (!isVideoFeedType(normalizeFeedType(source?.feedType)) ? source?.url : ''); + + const upstreamImage = await fetchCctvImageFromUpstream(upstreamCandidate); + if (upstreamImage?.ok) { + setHealth(cameraId, { + status: 'ok', + sourceKind: 'snapshot', + label: source?.provider || 'Configured source', + message: 'Upstream snapshot active', + }); + res.writeHead(200, { + 'Content-Type': upstreamImage.contentType, + 'Cache-Control': 'no-store', + 'X-CCTV-Source': 'upstream-image', + }); + res.end(upstreamImage.body); + return; + } + + const sv = await streetViewFallback({ lat, lon, heading, fov, pitch }); + if (sv?.ok) { + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'streetview', + label: 'Google Street View', + message: 'Fallback Street View frame', + }); + res.writeHead(200, { + 'Content-Type': sv.contentType, + 'Cache-Control': 'no-store', + 'X-CCTV-Source': 'streetview', + }); + res.end(sv.body); + return; + } + + const svg = buildSyntheticCctvSvg({ + cameraId, + label, + city, + status: source?.url ? 'UPSTREAM UNAVAILABLE' : 'NO UPSTREAM CONFIGURED', + }); + + setHealth(cameraId, { + status: 'degraded', + sourceKind: 'synthetic', + label: source?.provider || 'Synthetic fallback', + message: source?.url ? 'Upstream unavailable' : 'No source configured', + }); + + res.writeHead(200, { + 'Content-Type': 'image/svg+xml', + 'Cache-Control': 'no-store', + 'X-CCTV-Source': 'synthetic', + }); + res.end(svg); + } catch (error) { + console.error('[CCTV Proxy]', error?.message || String(error)); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'CCTV proxy error' })); + } + }); + }, + }; +} diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js new file mode 100644 index 000000000..e27d06528 --- /dev/null +++ b/server/providers/cctv/catalog.js @@ -0,0 +1,140 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DEFAULT_CCTV_SOURCE_FILE, DEFAULT_CCTV_MAX_SOURCES, CCTV_SOURCE_CACHE_MS } from './constants.js'; +import { normalizeSourceItem } from './normalize.js'; +import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, loadTflSourcesFromOpenData } from './sources.js'; +/** + * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). + * + * @returns {Array} Array of raw source objects, or [] on error. + */ +function loadSourcesFromFile(sourceRoot) { + const sourceFile = process.env.CCTV_SOURCES_FILE || DEFAULT_CCTV_SOURCE_FILE; + const resolved = path.isAbsolute(sourceFile) + ? sourceFile + : path.resolve(sourceRoot, sourceFile); + try { + if (!fs.existsSync(resolved)) return []; + const raw = fs.readFileSync(resolved, 'utf8'); + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch (error) { + console.warn('[CCTV] failed to read source file:', resolved, error?.message || error); + return []; + } +} + +/** + * Load CCTV sources from the CCTV_SOURCES_JSON env variable (inline JSON). + * + * @returns {Array} Array of raw source objects, or [] if unset/invalid. + */ +function loadSourcesFromEnv() { + const raw = process.env.CCTV_SOURCES_JSON; + if (!raw) return []; + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } +} + +/** Create an independent catalog rooted in the consuming application. */ +export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { +/** @type {Array} Cached merged + normalized CCTV source list. */ +let _cctvSourceCache = []; +/** @type {number} Epoch-ms when the source cache was last refreshed. */ +let _cctvSourceCacheAt = 0; +/** @type {Promise>|null} In-flight refresh, shared by concurrent + * callers so a post-TTL burst launches ONE refetch, not one per request. */ +let _cctvSourceInflight = null; + +/** + * Assemble and cache the merged CCTV source list. + * + * Merges sources from three origins (Austin Open Data, local file, + * env variable), deduplicates by ID, applies the global max cap, and + * caches for CCTV_SOURCE_CACHE_MS. + * + * @returns {Promise>} Deduplicated, capped source list. + */ +async function getCctvSources() { + const now = Date.now(); + if (_cctvSourceCache.length && now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS) { + return _cctvSourceCache; + } + // Single-flight: a burst of requests arriving past the TTL shares ONE refresh + // instead of each launching the full multi-provider refetch. The `.finally` + // clears the ref so the next post-TTL cycle starts fresh. + if (_cctvSourceInflight) return _cctvSourceInflight; + _cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; }); + return _cctvSourceInflight; +} + +/** + * Assemble and cache the merged CCTV source list from file/env + live packs. + * Always resolves (loaders self-catch to []); on a fully-empty refresh with a + * good prior catalog it serves stale rather than blanking the CCTV layer. + * + * @returns {Promise>} Deduplicated, capped source list. + */ +async function refreshCctvSources() { + const fromFile = loadSourcesFromFile(sourceRoot); + const fromEnv = loadSourcesFromEnv(); + + const forceAustin = String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1'; + const preferAustin = String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0'; + // Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack + // is configured and live packs aren't forced — same gate that governed the + // Austin-only fetch, now governing all three. Each pack fails independently. + const needsLiveSources = forceAustin || ((fromFile.length + fromEnv.length) === 0 && preferAustin); + const tflEnabled = String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0'; + + let fromAustin = []; + let fromCaltrans = []; + let fromTfl = []; + if (needsLiveSources) { + const [austinResult, caltransResult, tflResult] = await Promise.allSettled([ + loadAustinSourcesFromOpenData(), + loadCaltransSourcesFromOpenData(), + tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]), + ]); + fromAustin = austinResult.status === 'fulfilled' ? austinResult.value : []; + fromCaltrans = caltransResult.status === 'fulfilled' ? caltransResult.value : []; + fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : []; + } + // Live sources first so file/env overrides win on duplicate IDs (Map last-write). + const merged = [...fromAustin, ...fromCaltrans, ...fromTfl, ...fromFile, ...fromEnv]; + + // Deduplicate by camera ID (last-write wins because of Map.set) + const byId = new Map(); + for (const item of merged) { + if (!item || typeof item !== 'object') continue; + const normalized = normalizeSourceItem(item); + if (!normalized.id) continue; + byId.set(normalized.id, normalized); + } + + const mergedSources = Array.from(byId.values()); + const maxRaw = Number(process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES); + const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) : DEFAULT_CCTV_MAX_SOURCES; + if (mergedSources.length > maxCount) { + console.warn(`[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`); + } + const capped = mergedSources.length > maxCount ? mergedSources.slice(0, maxCount) : mergedSources; + if (capped.length > 0 || _cctvSourceCache.length === 0) { + _cctvSourceCache = capped; + } else { + // Every source came back empty (all live packs timed out / upstream outage) + // but a good catalog is already cached — serve it stale rather than blanking + // every CCTV route. Advancing the timestamp waits one TTL before retrying, + // which (with single-flight) bounds load on a persistently-down upstream. + console.warn(`[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`); + } + _cctvSourceCacheAt = Date.now(); + return _cctvSourceCache; +} + +return getCctvSources; +} diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js new file mode 100644 index 000000000..b8d733348 --- /dev/null +++ b/server/providers/cctv/constants.js @@ -0,0 +1,38 @@ +export const DEFAULT_CCTV_SOURCE_FILE = 'config/cctv_sources.austin.json'; +/** Austin Open Data portal endpoint for traffic camera records. */ +export const DEFAULT_AUSTIN_ROWS_URL = 'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD'; +/** Default cap on Austin cameras after distance-based prioritization. */ +export const DEFAULT_AUSTIN_MAX_SOURCES = 250; +/** Global cap on total CCTV sources served by the proxy. */ +export const DEFAULT_CCTV_MAX_SOURCES = 900; +/** Reference point for Austin camera prioritization (Congress & 6th). */ +export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 }; +/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */ +export const CALTRANS_CCTV_URL = (district) => + `https://cwwp2.dot.ca.gov/data/d${district}/cctv/cctvStatusD${String(district).padStart(2, '0')}.json`; +/** Districts fetched by default: SF Bay (4), LA (7), San Diego (11), Sacramento (3). */ +export const DEFAULT_CALTRANS_DISTRICTS = '4,7,11,3'; +export const DEFAULT_CALTRANS_MAX_SOURCES = 300; +/** Prioritization anchors: downtown cores of the four default metros. */ +export const CALTRANS_ANCHORS = [ + { lat: 37.7793, lon: -122.4193 }, // San Francisco + { lat: 34.0537, lon: -118.2428 }, // Los Angeles + { lat: 32.7157, lon: -117.1611 }, // San Diego + { lat: 38.5816, lon: -121.4944 }, // Sacramento +]; +/** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */ +export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam'; +export const TFL_IMAGE_ORIGIN = 'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'; +export const DEFAULT_TFL_MAX_SOURCES = 250; +export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 }; +/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL) infrequent. Frames are fetched per-request and are unaffected. */ +export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; +/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one + * stalled upstream can't leave getCctvSources (and thus every CCTV route) + * pending forever — a hung fetch aborts, the loader returns [], and + * serve-stale/other packs take over. */ +export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000; +/** Individual CCTV image fetches must settle before the active 10-second + * client refresh cadence. A bounded miss can fall through to Street View or + * the synthetic frame instead of leaving the browser preview pending. */ +export const CCTV_FRAME_FETCH_TIMEOUT_MS = 8 * 1000; diff --git a/server/providers/cctv/media.js b/server/providers/cctv/media.js new file mode 100644 index 000000000..e0c298e1a --- /dev/null +++ b/server/providers/cctv/media.js @@ -0,0 +1,180 @@ +import { Readable } from 'node:stream'; +import { hashSeed, escapeXml } from './normalize.js'; +import { CCTV_FRAME_FETCH_TIMEOUT_MS } from './constants.js'; +/** + * Generate a synthetic SVG billboard image for a CCTV camera placeholder. + * + * Produces a 960x540 SVG with a deterministic gradient (hue derived from + * camera ID hash), scanline overlay, HUD-style grid, and text labels + * showing camera name, city, ID, status, and current timestamp. Used + * when no upstream image or Street View fallback is available. + * + * @param {object} opts + * @param {string} opts.cameraId + * @param {string} opts.label + * @param {string} [opts.city] + * @param {string} [opts.status] + * @returns {string} SVG markup string. + */ +export function buildSyntheticCctvSvg({ cameraId, label, city, status }) { + const seed = hashSeed(`${cameraId}:${label}:${city}`); + const hue = seed % 360; + const hue2 = (hue + 46) % 360; + const now = new Date(); + const ts = now.toISOString().replace('T', ' ').replace('Z', 'Z').slice(0, 20); + const safeLabel = escapeXml(label); + const safeCity = escapeXml(city || 'GLOBAL GRID'); + const safeId = escapeXml(cameraId); + const safeStatus = escapeXml(status || 'SYNTHETIC'); + + return ` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CCTV FEED PLACEHOLDER + ${safeLabel} · ${safeCity} + ${safeId} + ${escapeXml(ts)} + ${safeStatus} + +`.trim(); +} + +/** + * Coerce a fetch() response body to a Node.js Readable stream. + * + * Handles both Node-native streams (.pipe) and web ReadableStreams (.getReader). + * + * @param {ReadableStream|NodeJS.ReadableStream|null} body + * @returns {import('stream').Readable|null} + */ +export function toReadable(body) { + if (!body) return null; + if (typeof body.pipe === 'function') return body; + if (typeof body.getReader === 'function') { + return Readable.fromWeb(body); + } + return null; +} + +/** + * Pipe an upstream fetch Response (image or video) to the client HTTP response. + * + * Forwards Content-Type, Content-Length, Content-Range, Accept-Ranges, and + * Cache-Control headers from the upstream. Falls back to buffered arrayBuffer + * if the body is not streamable. + * + * @param {import('http').ServerResponse} res + * @param {Response} upstream - fetch() Response object. + * @param {object} [opts] + * @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header. + */ +export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstream' } = {}) { + const contentType = upstream.headers.get('content-type') || 'application/octet-stream'; + const cacheControl = upstream.headers.get('cache-control') || 'no-store'; + const contentLength = upstream.headers.get('content-length'); + const contentRange = upstream.headers.get('content-range'); + const acceptRanges = upstream.headers.get('accept-ranges'); + const headers = { + 'Content-Type': contentType, + 'Cache-Control': cacheControl, + 'X-CCTV-Source': sourceHeader, + }; + if (contentLength) headers['Content-Length'] = contentLength; + if (contentRange) headers['Content-Range'] = contentRange; + if (acceptRanges) headers['Accept-Ranges'] = acceptRanges; + + // Cheap defense: reject an upstream that DECLARES an oversized fixed body. + // Live MJPEG/HLS streams are unbounded by design and send no content-length, + // so they pipe normally (piping streams to the client, never buffering). + const MEDIA_DECLARED_CAP_BYTES = 64 * 1024 * 1024; + if (Number.isFinite(Number(contentLength)) && Number(contentLength) > MEDIA_DECLARED_CAP_BYTES) { + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Upstream media exceeds size cap' })); + try { await upstream.body?.cancel(); } catch { /* no-op */ } + return; + } + + res.writeHead(upstream.status, headers); + + const stream = toReadable(upstream.body); + if (!stream) { + const buf = Buffer.from(await upstream.arrayBuffer()); + res.end(buf); + return; + } + + stream.on('error', () => { + if (!res.writableEnded) res.end(); + }); + stream.pipe(res); +} + +/** + * Fetch one upstream CCTV image within the frame-refresh budget. + * + * A timeout is treated like every other upstream miss so the caller can + * continue through the Street View and synthetic fallback chain. `fetchImpl` + * and `timeoutMs` are injectable only to keep the timeout contract unit-testable. + * + * @param {string} url - Server-registered upstream image URL. + * @param {object} [options] + * @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation. + * @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout. + * @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>} + */ +export async function fetchCctvImageFromUpstream(url, { + fetchImpl = fetch, + timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS, +} = {}) { + if (!url || !/^https?:\/\//i.test(url)) return null; + const controller = new AbortController(); + const timeoutId = setTimeout(() => { + controller.abort(new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError')); + }, timeoutMs); + try { + const upstream = await fetchImpl(url, { + headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }, + signal: controller.signal, + }); + const contentType = upstream.headers.get('content-type') || ''; + if (!upstream.ok || !contentType.startsWith('image/')) return null; + return { + ok: true, + body: Buffer.from(await upstream.arrayBuffer()), + contentType, + }; + } catch { + return null; + } finally { + clearTimeout(timeoutId); + } +} diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js new file mode 100644 index 000000000..24efbe25a --- /dev/null +++ b/server/providers/cctv/normalize.js @@ -0,0 +1,384 @@ +import { directionToHeading } from '../../../src/data/directionText.js'; +import { haversineKm } from '../common/geo.js'; +/** + * FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random + * values (e.g. hue for synthetic SVG billboards, fallback heading angles). + * + * @param {string} text + * @returns {number} Unsigned 32-bit hash. + */ +export function hashSeed(text) { + let h = 2166136261 >>> 0; // FNV offset basis + for (let i = 0; i < text.length; i++) { + h ^= text.charCodeAt(i); + h = Math.imul(h, 16777619); // FNV prime + } + return h >>> 0; +} + +/** + * Escape special XML/HTML characters for safe embedding in SVG text nodes. + * + * @param {string} text + * @returns {string} + */ +export function escapeXml(text) { + return String(text || '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +/** + * Canonicalize a CCTV feed type string to one of: + * 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through. + * + * @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream'). + * @returns {string} Normalized feed type. + */ +export function normalizeFeedType(value) { + const raw = String(value || '').trim().toLowerCase(); + if (!raw) return 'image'; + if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; + if (raw === 'mjpg') return 'mjpeg'; + if (raw === 'video') return 'mp4'; + if (raw === 'stream') return 'hls'; + return raw; +} + +/** + * Check whether a normalized feed type represents streaming video. + * + * @param {string} feedType + * @returns {boolean} + */ +export function isVideoFeedType(feedType) { + return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls'; +} + +/** + * Coerce a value to a finite number, returning fallback if NaN/Infinity. + * + * @param {*} value + * @param {number} [fallback=NaN] + * @returns {number} + */ +export function toFiniteNumber(value, fallback = NaN) { + const num = Number(value); + return Number.isFinite(num) ? num : fallback; +} + +/** + * Normalize a column/field name to a lowercase snake_case key. + * + * @param {string} text + * @returns {string} + */ +export function normalizeKey(text) { + return String(text || '') + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, ''); +} + +/** + * Parse a WKT POINT string (e.g. "POINT(-97.74 30.27)") into lat/lon. + * + * WKT uses (lon lat) order; returned object uses {lat, lon}. + * + * @param {string} value + * @returns {{lat:number, lon:number}} + */ +export function parsePointString(value) { + const match = String(value || '').match(/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i); + if (!match) return { lat: NaN, lon: NaN }; + return { + lon: toFiniteNumber(match[1]), + lat: toFiniteNumber(match[2]), + }; +} + +/** + * Extract lat/lon from a variety of coordinate representations. + * + * Handles WKT POINT strings, and objects with latitude/lat/y or + * longitude/lon/lng/x properties (various casing). + * + * @param {string|object|null} value + * @returns {{lat:number, lon:number}} + */ +export function coerceLatLon(value) { + if (!value) return { lat: NaN, lon: NaN }; + + if (typeof value === 'string') { + return parsePointString(value); + } + + if (typeof value !== 'object') { + return { lat: NaN, lon: NaN }; + } + + const lat = toFiniteNumber( + value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat, + NaN + ); + const lon = toFiniteNumber( + value.longitude ?? value.lon ?? value.lng ?? value.x ?? value.Longitude ?? value.Lon, + NaN + ); + return { lat, lon }; +} + +/** + * Extract geographic coordinates from an Austin Open Data camera record. + * + * Tries several candidate fields (location, coordinates, the_geom, + * point, geocoded_column) via coerceLatLon, then falls back to + * explicit latitude/longitude scalar fields. + * + * @param {object} record - Flattened camera record. + * @returns {{lat:number, lon:number}} + */ +export function extractAustinCoords(record) { + const candidates = [ + record.location, + record.coordinates, + record.the_geom, + record.point, + record.geocoded_column, + ]; + for (const candidate of candidates) { + const parsed = coerceLatLon(candidate); + if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon)) return parsed; + } + + const lat = toFiniteNumber( + record.latitude ?? record.lat ?? record.camera_latitude ?? record.location_latitude, + NaN + ); + const lon = toFiniteNumber( + record.longitude ?? record.lon ?? record.lng ?? record.camera_longitude ?? record.location_longitude, + NaN + ); + return { lat, lon }; +} + +/** + * Extract a numeric camera ID from an Austin Open Data record. + * + * Tries well-known field names first, then scans any field whose key + * contains "camera"/"cam"/"device" + "id". + * + * @param {object} record - Flattened camera record. + * @returns {string} Numeric ID string, or '' if none found. + */ +export function extractAustinCameraId(record) { + const preferredKeys = [ + 'camera_id', + 'cameraid', + 'cam_id', + 'device_id', + 'intersection_id', + 'id', + ]; + for (const key of preferredKeys) { + const value = record[key]; + if (value == null) continue; + const asText = String(value).trim(); + if (!asText) continue; + if (/^\d+$/.test(asText)) return asText; + } + + for (const [key, value] of Object.entries(record)) { + if (!/camera|cam|device/.test(key)) continue; + if (!/id/.test(key)) continue; + const asText = String(value || '').trim(); + if (!asText) continue; + if (/^\d+$/.test(asText)) return asText; + } + + return ''; +} + +/** + * Extract a human-readable camera name from an Austin record. + * + * @param {object} record - Flattened camera record. + * @param {string} cameraId - Fallback identifier if no name field found. + * @returns {string} + */ +export function extractAustinName(record, cameraId) { + const preferredKeys = [ + 'camera_name', + 'location_name', + 'intersection_name', + 'location', + 'cross_street', + 'description', + 'name', + ]; + for (const key of preferredKeys) { + const value = record[key]; + if (typeof value !== 'string') continue; + const text = value.trim(); + if (text) return text; + } + return `Austin Camera ${cameraId}`; +} + +/** + * Extract camera heading (compass bearing) from an Austin record. + * + * Tries explicit numeric heading fields first, then direction-keyword + * fields, then infers from the camera name/description text. + * + * @param {object} record - Flattened camera record. + * @returns {number} Heading in degrees [0..360), or NaN if unknown. + */ +export function extractAustinHeading(record) { + const direct = toFiniteNumber(record.heading_deg ?? record.heading ?? record.bearing, NaN); + if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360; + + // Dedicated direction fields: bare cardinal words ("West") are real facings. + const directionKeys = ['direction', 'travel_direction', 'facing', 'facing_direction']; + for (const key of directionKeys) { + const heading = directionToHeading(record[key], true); + if (Number.isFinite(heading)) return heading; + } + + // Free-form name/intersection text: only explicit travel forms ("WESTBOUND"/ + // "WB") count — a bare "West" here is a street name ("5TH ST / WEST AVE"), not + // a facing, and must not promote the camera to a false high-confidence heading. + const nameProbe = [ + record.camera_name, + record.location_name, + record.intersection_name, + record.location, + record.cross_street, + record.description, + record.name, + ].filter(Boolean).join(' '); + const inferred = directionToHeading(nameProbe); + if (Number.isFinite(inferred)) return inferred; + + return NaN; +} + +/** + * Bounding-box sanity check: is this coordinate plausibly in the Austin metro area? + * + * @param {number} lat + * @param {number} lon + * @returns {boolean} + */ +export function isLikelyAustinCoordinate(lat, lon) { + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; + return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.40; +} + +/** + * Derive a deterministic fallback heading from a camera ID hash. + * + * Produces one of 16 evenly-spaced compass directions (0, 22.5, 45, ...). + * + * @param {string} cameraId + * @returns {number} Heading in degrees [0..360). + */ +export function fallbackHeadingFromId(cameraId) { + return (hashSeed(String(cameraId)) % 16) * 22.5; +} + +/** + * Convert a Socrata rows.json array row into a keyed object using column metadata. + * + * @param {Array} row - Array of cell values from the Socrata payload. + * @param {Array<{fieldName?:string, name?:string}>} columns - Column descriptors. + * @returns {object} Keyed record with normalized snake_case keys. + */ +export function rowArrayToObject(row, columns) { + const record = {}; + for (let idx = 0; idx < columns.length; idx++) { + const col = columns[idx]; + const key = normalizeKey(col.fieldName || col.name || `col_${idx}`); + if (!key) continue; + record[key] = row[idx]; + } + return record; +} + +/** + * Distance-prioritizes cameras to a cap: keeps the maxCount cameras closest + * to ANY of the given anchor points (min distance over anchors), tie-broken + * by original array order. Used by every live source pack (Austin: one + * downtown anchor; Caltrans: one anchor per major CA metro; TfL: central + * London) so a cap always keeps the densest, most interesting cores. + * + * @param {Array} cameras - Normalized camera source objects. + * @param {number} maxCount - Cap (<=0 or >= length disables). + * @param {Array<{lat:number,lon:number}>} anchors - At least one anchor. + * @returns {Array} Capped, priority-ordered camera list. + */ +export function prioritizeSources(cameras, maxCount, anchors) { + const list = Array.isArray(cameras) ? cameras : []; + const anchorList = (Array.isArray(anchors) ? anchors : []).filter( + (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon) + ); + if (!Number.isFinite(maxCount) || maxCount <= 0 || list.length <= maxCount || !anchorList.length) { + return list; + } + + const scored = list.map((camera, idx) => { + const lat = Number(camera?.lat); + const lon = Number(camera?.lon); + const distKm = Number.isFinite(lat) && Number.isFinite(lon) + ? Math.min(...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon))) + : Number.POSITIVE_INFINITY; + return { camera, idx, distKm }; + }); + + scored.sort((a, b) => { + if (a.distKm !== b.distKm) return a.distKm - b.distKm; + return a.idx - b.idx; + }); + + return scored.slice(0, maxCount).map((entry) => entry.camera); +} + +/** + * Normalize a raw CCTV source item into a canonical shape with safe defaults. + * + * @param {object} item - Raw source from file, env, or Austin Open Data. + * @returns {object} Normalized source with all expected fields populated. + */ +export function normalizeSourceItem(item) { + return { + id: String(item.id || '').trim(), + name: String(item.name || item.id || '').trim(), + city: String(item.city || ''), + cityId: String(item.cityId || ''), + provider: String(item.provider || 'Configured CCTV Source'), + lat: toFiniteNumber(item.lat), + lon: toFiniteNumber(item.lon), + headingDeg: toFiniteNumber(item.headingDeg), + headingConfidence: String(item.headingConfidence || item.headingSource || '').toLowerCase(), + pitchDeg: toFiniteNumber(item.pitchDeg), + fovDeg: toFiniteNumber(item.fovDeg), + rangeM: toFiniteNumber(item.rangeM), + mountHeightM: toFiniteNumber(item.mountHeightM), + groundElevationM: toFiniteNumber(item.groundElevationM), + feedType: normalizeFeedType(item.feedType || item.type || ''), + url: typeof item.url === 'string' ? item.url : '', + snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '', + license: String(item.license || item.licenseNote || ''), + sourceKind: String(item.sourceKind || item.kind || 'configured'), + // Optional CAL badge input (cctv-v2 design §3b/§9.2, additive-only per the + // global constraints — nothing else in this file changes): hand-authored + // file/env catalog entries may declare poseSource:'curated' so the panel + // badge can distinguish them from raw automated priors (e.g. Austin Open + // Data, which never sets this field). Passed through as-is to the client. + poseSource: item.poseSource === 'curated' ? 'curated' : undefined, + }; +} diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js new file mode 100644 index 000000000..81da1574a --- /dev/null +++ b/server/providers/cctv/sources.js @@ -0,0 +1,263 @@ +import { DEFAULT_AUSTIN_ROWS_URL, DEFAULT_AUSTIN_MAX_SOURCES, AUSTIN_DOWNTOWN, CALTRANS_CCTV_URL, DEFAULT_CALTRANS_DISTRICTS, DEFAULT_CALTRANS_MAX_SOURCES, CALTRANS_ANCHORS, TFL_JAMCAM_URL, TFL_IMAGE_ORIGIN, DEFAULT_TFL_MAX_SOURCES, LONDON_CENTER, CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; +import { toFiniteNumber, extractAustinCoords, extractAustinCameraId, extractAustinName, extractAustinHeading, isLikelyAustinCoordinate, fallbackHeadingFromId, rowArrayToObject, prioritizeSources } from './normalize.js'; +import { directionToHeading } from '../../../src/data/directionText.js'; +/** + * Fetch and parse Austin traffic camera records from the city Open Data portal. + * + * Downloads the Socrata rows.json payload, converts each row to a keyed + * record, extracts camera ID / coords / heading / name, validates against + * the Austin bounding box, deduplicates by ID, then distance-prioritizes + * to stay within CCTV_AUSTIN_MAX_SOURCES. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadAustinSourcesFromOpenData() { + const endpoint = process.env.CCTV_AUSTIN_ROWS_URL || DEFAULT_AUSTIN_ROWS_URL; + try { + const resp = await fetch(endpoint, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + if (!resp.ok) { + console.warn('[CCTV] Austin source download failed:', resp.status); + return []; + } + const payload = await resp.json(); + const columns = Array.isArray(payload?.meta?.view?.columns) ? payload.meta.view.columns : []; + const rows = Array.isArray(payload?.data) ? payload.data : []; + if (!columns.length || !rows.length) return []; + + const cameras = []; + for (const row of rows) { + if (!Array.isArray(row)) continue; + const record = rowArrayToObject(row, columns); + const cameraId = extractAustinCameraId(record); + if (!cameraId) continue; + + // Only live cameras: the dataset carries DESIRED (planned, not built), + // REMOVED and VOID rows whose frame URLs never resolve — those cameras + // would render as permanent Street View / synthetic fallbacks. Tolerate + // a missing column (keep the row) so a schema change fails open. + const status = String(record.camera_status || '').trim().toUpperCase(); + if (status && status !== 'TURNED_ON') continue; + + const { lat, lon } = extractAustinCoords(record); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + if (!isLikelyAustinCoordinate(lat, lon)) continue; + + const extractedHeading = extractAustinHeading(record); + const hasHeading = Number.isFinite(extractedHeading); + const headingDeg = hasHeading ? extractedHeading : fallbackHeadingFromId(cameraId); + cameras.push({ + id: cameraId, + name: extractAustinName(record, cameraId), + city: 'Austin', + cityId: 'austin', + provider: 'Austin Transportation & Public Works', + lat, + lon, + headingDeg, + headingConfidence: hasHeading ? 'high' : 'low', + pitchDeg: hasHeading ? -24 : -18, + fovDeg: hasHeading ? 56 : 44, + rangeM: hasHeading ? 210 : 145, + mountHeightM: hasHeading ? 10 : 8, + groundElevationM: 150, + feedType: 'image', + url: `https://cctv.austinmobility.io/image/${encodeURIComponent(cameraId)}.jpg`, + snapshotUrl: `https://cctv.austinmobility.io/image/${encodeURIComponent(cameraId)}.jpg`, + sourceKind: 'austin-open-data', + license: 'Public city traffic camera frame', + }); + } + + const unique = Array.from(new Map(cameras.map((camera) => [camera.id, camera])).values()); + const maxRaw = Number(process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES); + const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(300, Math.floor(maxRaw))) : DEFAULT_AUSTIN_MAX_SOURCES; + const prioritized = prioritizeSources(unique, maxCount, [AUSTIN_DOWNTOWN]); + if (prioritized.length < unique.length) { + console.log(`[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`); + } else { + console.log('[CCTV] Loaded Austin camera sources:', prioritized.length); + } + return prioritized; + } catch (error) { + console.warn('[CCTV] Austin source download error:', error?.message || error); + return []; + } +} + +/** + * Fetch Caltrans CCTV cameras for the configured districts (CCTV_CALTRANS_DISTRICTS, + * comma-separated 1..12; empty string disables the pack). One official JSON feed per + * district, identical schema statewide; keyless. Only inService cameras with finite + * coords and a cwwp2.dot.ca.gov https image URL are kept (the image-URL origin check + * is defense-in-depth: the proxy only ever fetches catalog URLs, and this pins the + * catalog to the official host). Districts fetch in parallel and fail independently + * (Promise.allSettled) — one district outage never darkens the others. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadCaltransSourcesFromOpenData() { + const districtsRaw = process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS; + const districts = String(districtsRaw) + .split(',') + .map((token) => Number(token.trim())) + .filter((n) => Number.isInteger(n) && n >= 1 && n <= 12); + if (!districts.length) return []; + + const settled = await Promise.allSettled( + districts.map(async (district) => { + const resp = await fetch(CALTRANS_CCTV_URL(district), { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + if (!resp.ok) throw new Error(`D${district} HTTP ${resp.status}`); + const payload = await resp.json(); + const rows = Array.isArray(payload?.data) ? payload.data : []; + return { district, rows }; + }) + ); + + const cameras = []; + for (const result of settled) { + if (result.status !== 'fulfilled') { + console.warn('[CCTV] Caltrans district fetch failed:', result.reason?.message || result.reason); + continue; + } + const { district, rows } = result.value; + for (const row of rows) { + const cctv = row?.cctv; + if (!cctv || String(cctv.inService).toLowerCase() !== 'true') continue; + const loc = cctv.location || {}; + const lat = toFiniteNumber(loc.latitude); + const lon = toFiniteNumber(loc.longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + + const imageUrl = String(cctv.imageData?.static?.currentImageURL || ''); + // Official-host pin (see JSDoc). Also drops records with no still image. + if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue; + + const locationName = String(loc.locationName || '').trim(); + // Leading token of locationName is the stable camera code ("TV102 -- I-580 : …"). + const codeMatch = /^([A-Za-z0-9_-]+)\s*--/.exec(locationName); + const code = (codeMatch ? codeMatch[1] : `x${cameras.length}`).toLowerCase(); + const cameraId = `ca-d${district}-${code}`; + + // loc.direction is a dedicated field ("West", "South") → allow bare words. + const heading = directionToHeading(loc.direction, true); + const hasHeading = Number.isFinite(heading); + const label = locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') || `Caltrans D${district} ${code}`; + cameras.push({ + id: cameraId, + name: loc.nearbyPlace ? `${label} (${loc.nearbyPlace})` : label, + city: String(loc.nearbyPlace || `Caltrans D${district}`), + cityId: `ca-d${district}`, + provider: 'Caltrans', + lat, + lon, + headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId), + headingConfidence: hasHeading ? 'high' : 'low', + // Same two fabricated pose personalities as Austin (design §1a): these are + // RAW PRIOR starting points; the client's one-shot ground snap + manual + // calibration own the truth. + pitchDeg: hasHeading ? -24 : -18, + fovDeg: hasHeading ? 56 : 44, + rangeM: hasHeading ? 210 : 145, + mountHeightM: hasHeading ? 10 : 8, + // loc.elevation is reported in FEET (verified: D3 maxes at 7427 ft ≈ + // 2264 m for the Sierra passes — as metres that would top Mt Whitney). + // Convert to metres and clamp to a sane CA-roads range so an occasional + // garbage upstream value can't fling a camera kilometres up. Prior only: + // the client one-shot snap corrects it on 3D-tile stacks — but on a + // no-tileset stack (keyless OSM) the snap misses and this height freezes, + // so it must be right-ish on its own. + groundElevationM: (() => { + const ft = toFiniteNumber(loc.elevation, NaN); + return Number.isFinite(ft) ? Math.max(-100, Math.min(4000, ft * 0.3048)) : 150; + })(), + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'caltrans-open-data', + license: 'Public Caltrans highway camera frame', + }); + } + } + + const maxRaw = Number(process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES); + const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_CALTRANS_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, CALTRANS_ANCHORS); + console.log(`[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`); + return prioritized; +} + +/** + * Fetch TfL JamCams (London). Keyless: the optional TFL_APP_KEY only raises the + * list-endpoint rate limit (frames come from TfL's public S3 bucket, which is not + * rate-limited); the 15-min source cache keeps list hits far below anonymous + * limits anyway. Only `available === "true"` cameras with finite coords and an + * image URL on the official bucket are kept. Attribution: "Powered by TfL Open + * Data" (registered in src/data/dataCredits.js). + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadTflSourcesFromOpenData() { + try { + const appKey = String(process.env.TFL_APP_KEY || '').trim(); + const url = appKey ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}` : TFL_JAMCAM_URL; + const resp = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + if (!resp.ok) { + console.warn('[CCTV] TfL JamCam download failed:', resp.status); + return []; + } + const places = await resp.json(); + if (!Array.isArray(places)) return []; + + const cameras = []; + for (const place of places) { + const props = {}; + for (const p of place?.additionalProperties || []) { + if (p?.key) props[p.key] = p.value; + } + if (String(props.available).toLowerCase() !== 'true') continue; + const lat = toFiniteNumber(place?.lat); + const lon = toFiniteNumber(place?.lon); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + const imageUrl = String(props.imageUrl || ''); + if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin + + // "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id). + const rawId = String(place?.id || '').replace(/^JamCams_/, ''); + if (!rawId) continue; + const cameraId = `tfl-${rawId}`; + + cameras.push({ + id: cameraId, + name: String(place?.commonName || `JamCam ${rawId}`), + city: 'London', + cityId: 'london', + provider: 'Transport for London', + lat, + lon, + // No heading signal at all in JamCam data → id-hash fallback, low + // confidence personality (same as headingless Austin cameras). + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: 15, // Thames-basin prior; one-shot snap corrects. + feedType: 'image', // stills-first (owner decision); props.videoUrl deliberately unused + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'tfl-open-data', + license: 'Powered by TfL Open Data', + }); + } + + const maxRaw = Number(process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES); + const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_TFL_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, [LONDON_CENTER]); + console.log(`[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`); + return prioritized; + } catch (error) { + console.warn('[CCTV] TfL JamCam download error:', error?.message || error); + return []; + } +} diff --git a/server/providers/local.js b/server/providers/local.js index 934e7c442..7c8cfd48d 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -1,3 +1,7 @@ +import { cctvProxy } from './cctv.js'; +import { radioBrowserProxy } from './radio.js'; +export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream } from './cctv.js'; +export { createRadioProxyMiddleware, isPublicRadioAddress, normalizeRadioBrowserStation, publicRadioStation, publicRadioHttpsUrl } from './radio.js'; import { terrainHeightsProxy } from './terrain.js'; import { tomtomProxy } from './traffic.js'; import { firmsProxy } from './firms.js'; @@ -48,14 +52,9 @@ import { promises as fsp } from 'node:fs'; import { spawnSync } from 'node:child_process'; import { createHash, randomUUID } from 'node:crypto'; import path from 'node:path'; -import { Readable } from 'node:stream'; -import https from 'node:https'; -import { lookup as lookupDns } from 'node:dns/promises'; -import { directionToHeading } from '../../src/data/directionText.js'; import { fileURLToPath } from 'node:url'; -import { normalizeRadioCountryInput } from '../../src/data/radioCountry.js'; import { normalizeRegionalArticles, normalizeRegionalPlace, @@ -580,530 +579,6 @@ async function readRequestBodyCapped(req, maxBytes) { return Buffer.concat(chunks); } -// --------------------------------------------------------------------------- -// Radio Browser directory proxy -// --------------------------------------------------------------------------- -const RADIO_DIRECTORY_CACHE_MS = 45 * 60 * 1000; -const RADIO_DIRECTORY_STALE_MS = 7 * 24 * 60 * 60 * 1000; -const RADIO_MIRROR_CACHE_MS = 6 * 60 * 60 * 1000; -const RADIO_FETCH_TIMEOUT_MS = 12_000; -const RADIO_RESPONSE_MAX_BYTES = 4 * 1024 * 1024; -const RADIO_DIRECTORY_LIMIT = 750; -const RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES = 5; -const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(RADIO_DIRECTORY_LIMIT / 2); -const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; -const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; -const RADIO_FALLBACK_MIRRORS = Object.freeze([ - 'https://de1.api.radio-browser.info', - 'https://de2.api.radio-browser.info', - 'https://nl1.api.radio-browser.info', -]); - -function cleanRadioText(value, maxLength) { - return String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, maxLength).trim(); -} - -function isNonGlobalIpv4(hostname) { - const pieces = hostname.split('.'); - if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) return false; - const values = pieces.map(Number); - if (values.some((value) => value > 255)) return true; - const [a, b, c] = values; - return a === 0 || a === 10 || a === 127 || a >= 224 - || (a === 100 && b >= 64 && b <= 127) - || (a === 169 && b === 254) - || (a === 172 && b >= 16 && b <= 31) - || (a === 192 && b === 0) - || (a === 192 && b === 88 && c === 99) - || (a === 192 && b === 168) - || (a === 198 && (b === 18 || b === 19)) - || (a === 198 && b === 51 && c === 100) - || (a === 203 && b === 0 && c === 113); -} - -/** Return a normalized public HTTPS URL, or null for local/private targets. */ -export function publicRadioHttpsUrl(value) { - try { - const url = new URL(String(value ?? '')); - const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, ''); - if (url.protocol !== 'https:' || url.username || url.password || !hostname) return null; - if ( - hostname === 'localhost' - || hostname.endsWith('.localhost') - || hostname.endsWith('.local') - || isNonGlobalIpv4(hostname) - || hostname.includes(':') - ) return null; - url.hash = ''; - return url.href; - } catch { - return null; - } -} - -/** Normalize one Radio Browser station and omit favicons and unsafe streams. */ -export function normalizeRadioBrowserStation(raw) { - const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); - const lat = raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); - const lon = raw?.geo_long === null || raw?.geo_long === '' ? null : Number(raw?.geo_long); - const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); - const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url); - if ( - !RADIO_UUID_RE.test(id) - || Number(raw?.lastcheckok) !== 1 - || Number(raw?.hls) === 1 - || !Number.isFinite(lat) || lat < -90 || lat > 90 - || !Number.isFinite(lon) || lon < -180 || lon > 180 - || !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) - || !streamUrl - ) return null; - - const name = cleanRadioText(raw?.name, 140); - if (!name) return null; - const tags = String(raw?.tags ?? '') - .split(',') - .map((tag) => cleanRadioText(tag, 80).toLocaleLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim()) - .filter(Boolean) - .filter((tag, index, all) => all.indexOf(tag) === index) - .slice(0, 24); - const languages = String(raw?.language ?? '') - .split(',') - .map((language) => cleanRadioText(language, 40)) - .filter(Boolean) - .slice(0, 8); - const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); - const normalizedCode = normalizeRadioCountryInput(rawCountryCode); - const normalizedCountry = normalizedCode.valid && !normalizedCode.empty - ? normalizedCode - : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); - const bitrate = Number(raw?.bitrate); - return { - id, - name, - lat, - lon, - streamUrl, - homepage: publicRadioHttpsUrl(raw?.homepage), - tags, - languages, - state: cleanRadioText(raw?.state, 80), - country: normalizedCountry.valid && !normalizedCountry.empty - ? normalizedCountry.name - : cleanRadioText(raw?.country, 80), - countryCode: normalizedCountry.valid ? normalizedCountry.code : '', - metadataTrust: 'untrusted-community', - codec, - bitrate: Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 ? bitrate : null, - clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), - }; -} - -export function publicRadioStation(station) { - return { - id: station.id, - name: station.name, - lat: station.lat, - lon: station.lon, - streamUrl: station.streamUrl, - homepage: station.homepage, - tags: station.tags, - languages: station.languages, - state: station.state, - country: station.country, - countryCode: station.countryCode, - metadataTrust: station.metadataTrust, - codec: station.codec, - bitrate: station.bitrate, - }; -} - -function radioMirrorOrigin(value) { - const hostname = String(value ?? '').toLowerCase().replace(/\.$/, ''); - if (!/^[a-z0-9-]+\.api\.radio-browser\.info$/.test(hostname)) return null; - return `https://${hostname}`; -} - -/** Return whether a resolved Radio Browser address is safe for an outbound request. */ -export function isPublicRadioAddress(value) { - const address = String(value ?? '').trim().toLowerCase().replace(/^\[|\]$/g, ''); - if (!address) return false; - if (!address.includes(':')) { - const ipv4 = address.split('.'); - return ipv4.length === 4 - && ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) - && !isNonGlobalIpv4(address); - } - const pieces = address.split('::'); - if (pieces.length > 2) return false; - const left = pieces[0] ? pieces[0].split(':') : []; - const right = pieces[1] ? pieces[1].split(':') : []; - const missing = 8 - left.length - right.length; - if ((pieces.length === 1 && missing !== 0) || (pieces.length === 2 && missing < 1)) return false; - const groups = [...left, ...Array(Math.max(0, missing)).fill('0'), ...right]; - if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) return false; - const numeric = groups.reduce((total, group) => (total << 16n) | BigInt(`0x${group}`), 0n); - const inCidr = (base, prefix) => { - const shift = 128n - BigInt(prefix); - return (numeric >> shift) === (base >> shift); - }; - const base = (text) => text.split(':').reduce( - (total, group) => (total << 16n) | BigInt(`0x${group || '0'}`), - 0n, - ); - const cidr = (text, prefix) => inCidr(base(text), prefix); - return cidr('2000:0:0:0:0:0:0:0', 3) - && !cidr('2001:0:0:0:0:0:0:0', 23) - && !cidr('2001:db8:0:0:0:0:0:0', 32) - && !cidr('2002:0:0:0:0:0:0:0', 16) - && !cidr('3fff:0:0:0:0:0:0:0', 20); -} - -function radioProxyDestination(value) { - let url; - try { - url = new URL(String(value)); - } catch { - return null; - } - const origin = radioMirrorOrigin(url.hostname); - if ( - !origin - || url.origin !== origin - || url.username - || url.password - || url.port - || url.hash - ) return null; - const discovery = url.hostname.toLowerCase() === 'all.api.radio-browser.info' - && url.pathname === '/json/servers' - && !url.search; - const directory = url.pathname === '/json/stations/search'; - const click = /^\/json\/url\/[0-9a-f-]+$/i.test(url.pathname) && !url.search; - return discovery || directory || click ? url : null; -} - -async function resolveRadioProxyAddresses(hostname, lookupImpl) { - const resolved = await lookupImpl(hostname, { all: true, verbatim: true }); - const rows = Array.isArray(resolved) ? resolved : [resolved]; - const addresses = rows - .map((row) => ({ address: String(row?.address || ''), family: Number(row?.family) || undefined })) - .filter((row) => row.address); - if (!addresses.length || addresses.some((row) => !isPublicRadioAddress(row.address))) { - throw new Error('Radio Browser resolved to a forbidden address'); - } - return addresses; -} - -function fetchPinnedRadioResponse(url, options, addresses) { - return new Promise((resolve, reject) => { - const address = addresses[0]; - const request = https.request(url, { - method: 'GET', - headers: options.headers, - signal: options.signal, - lookup(_hostname, lookupOptions, callback) { - if (lookupOptions?.all) callback(null, addresses); - else callback(null, address.address, address.family); - }, - }, (response) => { - const headers = new Headers(); - for (const [name, value] of Object.entries(response.headers)) { - if (Array.isArray(value)) value.forEach((item) => headers.append(name, item)); - else if (value !== undefined) headers.set(name, String(value)); - } - resolve(new Response(Readable.toWeb(response), { - status: response.statusCode || 500, - statusText: response.statusMessage || '', - headers, - })); - }); - request.on('error', reject); - request.end(); - }); -} - -async function mapRadioConcurrent(values, concurrency, mapper) { - const results = new Array(values.length); - let cursor = 0; - const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { - for (;;) { - const index = cursor++; - if (index >= values.length) return; - results[index] = await mapper(values[index], index); - } - }); - await Promise.all(workers); - return results; -} - -/** Create the testable Connect middleware backing `/api/radio`. */ -export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = lookupDns, now = Date.now } = {}) { - let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 }; - let mirrorPromise = null; - let catalogCache = null; - let catalogGeneration = 0; - // The generation counter is process-local, so it restarts from 1 with the - // server. The instance token scopes each generation sequence: a client that - // sees a new instance must treat the catalog as a fresh sequence, never as a - // repeat ("still generation 1") or a regression ("generation went backward"). - const catalogInstance = randomUUID(); - let servedStationIds = new Set(); - let refreshPromise = null; - - async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) { - const destination = radioProxyDestination(url); - if (!destination) throw new Error('Radio Browser destination is not permitted'); - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), RADIO_FETCH_TIMEOUT_MS); - try { - const addresses = await resolveRadioProxyAddresses(destination.hostname, lookupImpl); - const options = { - headers: { Accept: 'application/json', 'User-Agent': RADIO_USER_AGENT }, - signal: controller.signal, - redirect: 'manual', - }; - const response = fetchImpl - ? await fetchImpl(destination.href, options) - : await fetchPinnedRadioResponse(destination, options, addresses); - if (response.status >= 300 && response.status < 400) { - try { await response.body?.cancel?.(); } catch { /* no-op */ } - throw new Error('Radio Browser redirects are refused'); - } - if (!response.ok) throw new Error(`Radio Browser returned ${response.status}`); - const text = await readResponseTextCapped(response, maxBytes); - return JSON.parse(text); - } finally { - clearTimeout(timer); - } - } - - async function mirrors() { - if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS) return mirrorCache.origins; - if (!mirrorPromise) { - mirrorPromise = (async () => { - try { - const rows = await fetchJson('https://all.api.radio-browser.info/json/servers', 256 * 1024); - const discovered = [...new Set((Array.isArray(rows) ? rows : []).map((row) => radioMirrorOrigin(row?.name)).filter(Boolean))]; - if (discovered.length) { - mirrorCache = { origins: [...discovered, ...RADIO_FALLBACK_MIRRORS.filter((origin) => !discovered.includes(origin))], cachedAt: now() }; - } - } catch { - mirrorCache = { ...mirrorCache, cachedAt: now() }; - } - return mirrorCache.origins; - })().finally(() => { mirrorPromise = null; }); - } - return mirrorPromise; - } - - async function fetchPath(pathname) { - let lastError = null; - for (const origin of await mirrors()) { - try { - return await fetchJson(`${origin}${pathname}`); - } catch (error) { - lastError = error; - } - } - throw lastError || new Error('No Radio Browser mirror is available'); - } - - async function refreshCatalog() { - const queries = [null, 'news', 'talk', 'weather', 'emergency', 'scanner', 'aviation', 'marine', 'traffic']; - const outcomes = await mapRadioConcurrent(queries, 3, async (tag, index) => { - const params = new URLSearchParams({ - has_geo_info: 'true', - is_https: 'true', - hidebroken: 'true', - order: 'clickcount', - reverse: 'true', - limit: index === 0 ? '1800' : '220', - }); - if (tag) params.set('tag', tag); - try { - const rows = await fetchPath(`/json/stations/search?${params}`); - if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - if (!rows.every((row) => ( - row - && typeof row === 'object' - && !Array.isArray(row) - && typeof row.stationuuid === 'string' - && typeof row.name === 'string' - && (typeof row.url_resolved === 'string' || typeof row.url === 'string') - ))) throw new Error('Radio Browser catalog contained a malformed station row'); - const stations = rows.map(normalizeRadioBrowserStation).filter(Boolean); - const requestedTag = cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(); - const requestedTagCovered = !requestedTag || stations.some((station) => ( - station.tags.some((stationTag) => stationTag === requestedTag || stationTag.includes(requestedTag)) - )); - return { - // Query coverage is based on accepted rows, not merely a payload that - // happens to match the upstream schema. Specialist responses must - // also contain an accepted station tagged for the requested category. - succeeded: stations.length > 0 && requestedTagCovered, - stations, - }; - } catch { - return { succeeded: false, stations: [] }; - } - }); - const resultSets = outcomes.map((outcome) => outcome.stations); - - const selected = []; - const seen = new Set(); - const take = (station) => { - if (!station || seen.has(station.id) || selected.length >= RADIO_DIRECTORY_LIMIT) return; - seen.add(station.id); - selected.push(station); - }; - // Seed specialist station-tag queries before popularity fill so operational - // categories remain represented even when global click charts skew musical. - for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take); - resultSets.flat().sort((a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name)).forEach(take); - const timestamp = now(); - const successfulQueries = outcomes.filter((outcome) => outcome.succeeded).length; - const broadQueryHealthy = outcomes[0].succeeded && outcomes[0].stations.length > 0; - const healthReasons = []; - if (!broadQueryHealthy) healthReasons.push('broad-query-unhealthy'); - if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES) healthReasons.push('query-coverage-below-policy'); - if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); - const degraded = healthReasons.length > 0; - const coverage = { - successfulQueries, - totalQueries: queries.length, - stationCount: selected.length, - healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS, - }; - const nextCatalog = { - cachedAt: timestamp, - updatedAt: new Date(timestamp).toISOString(), - stations: selected.map(publicRadioStation), - stationIds: new Set(selected.map((station) => station.id)), - degraded, - degradedReason: degraded ? healthReasons.join(',') : null, - coverage, - }; - if (degraded && catalogCache) { - const error = new Error('Radio Browser catalog refresh did not meet health policy'); - error.radioCatalogDegraded = true; - error.radioDegradedReason = nextCatalog.degradedReason; - error.radioCoverage = coverage; - throw error; - } - if (degraded && !selected.length) { - const error = new Error('Radio Browser catalog refresh returned no usable stations'); - error.radioCatalogDegraded = true; - error.radioDegradedReason = nextCatalog.degradedReason; - error.radioCoverage = coverage; - throw error; - } - if (degraded) { - servedStationIds = nextCatalog.stationIds; - return { ...nextCatalog, acceptedGeneration: null }; - } - catalogCache = { - ...nextCatalog, - acceptedGeneration: ++catalogGeneration, - }; - servedStationIds = catalogCache.stationIds; - return catalogCache; - } - - async function getCatalog() { - if (catalogCache && now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS) { - return { ...catalogCache, stale: false }; - } - if (!refreshPromise) { - refreshPromise = refreshCatalog().finally(() => { refreshPromise = null; }); - } - try { - return { ...await refreshPromise, stale: false }; - } catch (error) { - if (catalogCache && now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS) { - return { - ...catalogCache, - stale: true, - degraded: true, - degradedReason: error?.radioDegradedReason || 'refresh-failed', - coverage: error?.radioCoverage || catalogCache.coverage, - }; - } - throw error; - } - } - - function sendJson(res, status, body) { - res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify(body)); - } - - return async function radioProxyMiddleware(req, res) { - const requestUrl = new URL(req.url || '/', 'http://localhost'); - if (requestUrl.pathname === '/stations') { - if (req.method !== 'GET') { - res.writeHead(405, { Allow: 'GET', 'Cache-Control': 'no-store' }); - res.end(); - return; - } - try { - const catalog = await getCatalog(); - sendJson(res, 200, { - stations: catalog.stations, - updatedAt: catalog.updatedAt, - stale: catalog.stale, - degraded: Boolean(catalog.degraded), - degradedReason: catalog.degradedReason || null, - coverage: catalog.coverage || null, - acceptedGeneration: catalog.acceptedGeneration ?? null, - catalogInstance, - }); - } catch (error) { - sendJson(res, 503, { - error: 'Radio directory is temporarily unavailable', - degraded: Boolean(error?.radioCatalogDegraded), - degradedReason: error?.radioDegradedReason || null, - }); - } - return; - } - - const clickMatch = requestUrl.pathname.match(/^\/click\/([0-9a-f-]+)$/i); - if (clickMatch) { - if (req.method !== 'POST') { - res.writeHead(405, { Allow: 'POST', 'Cache-Control': 'no-store' }); - res.end(); - return; - } - const id = clickMatch[1].toLowerCase(); - if (!RADIO_UUID_RE.test(id) || !servedStationIds.has(id)) { - sendJson(res, 404, { error: 'Unknown radio station' }); - return; - } - res.writeHead(204, { 'Cache-Control': 'no-store' }); - res.end(); - void fetchPath(`/json/url/${id}`).catch(() => {}); - return; - } - - sendJson(res, 404, { error: 'Unknown radio route' }); - }; -} - -function radioBrowserProxy() { - const middleware = createRadioProxyMiddleware(); - const install = (server) => { - server.middlewares.use('/api/radio', middleware); - }; - return { - name: 'radio-browser-proxy', - configureServer: install, - configurePreviewServer: install, - }; -} // Sourced from the shared voice-model registry so the client's cost estimate // can never be computed against a different model than the session runs on. const OPENAI_REALTIME_MODEL_DEFAULT = VOICE_MODELS.standard.id; @@ -1425,1305 +900,6 @@ function overpassProxy() { }; } -/** - * FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random - * values (e.g. hue for synthetic SVG billboards, fallback heading angles). - * - * @param {string} text - * @returns {number} Unsigned 32-bit hash. - */ -function hashSeed(text) { - let h = 2166136261 >>> 0; // FNV offset basis - for (let i = 0; i < text.length; i++) { - h ^= text.charCodeAt(i); - h = Math.imul(h, 16777619); // FNV prime - } - return h >>> 0; -} - -/** - * Escape special XML/HTML characters for safe embedding in SVG text nodes. - * - * @param {string} text - * @returns {string} - */ -function escapeXml(text) { - return String(text || '') - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/'/g, '''); -} - -/** - * Canonicalize a CCTV feed type string to one of: - * 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through. - * - * @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream'). - * @returns {string} Normalized feed type. - */ -function normalizeFeedType(value) { - const raw = String(value || '').trim().toLowerCase(); - if (!raw) return 'image'; - if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; - if (raw === 'mjpg') return 'mjpeg'; - if (raw === 'video') return 'mp4'; - if (raw === 'stream') return 'hls'; - return raw; -} - -/** - * Check whether a normalized feed type represents streaming video. - * - * @param {string} feedType - * @returns {boolean} - */ -function isVideoFeedType(feedType) { - return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls'; -} - -// --------------------------------------------------------------------------- -// CCTV proxy constants and source cache state -// --------------------------------------------------------------------------- -/** Path to the optional static CCTV source list (JSON array). */ -const DEFAULT_CCTV_SOURCE_FILE = 'config/cctv_sources.austin.json'; -/** Austin Open Data portal endpoint for traffic camera records. */ -const DEFAULT_AUSTIN_ROWS_URL = 'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD'; -/** Default cap on Austin cameras after distance-based prioritization. */ -const DEFAULT_AUSTIN_MAX_SOURCES = 250; -/** Global cap on total CCTV sources served by the proxy. */ -const DEFAULT_CCTV_MAX_SOURCES = 900; -/** Reference point for Austin camera prioritization (Congress & 6th). */ -const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 }; -/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */ -const CALTRANS_CCTV_URL = (district) => - `https://cwwp2.dot.ca.gov/data/d${district}/cctv/cctvStatusD${String(district).padStart(2, '0')}.json`; -/** Districts fetched by default: SF Bay (4), LA (7), San Diego (11), Sacramento (3). */ -const DEFAULT_CALTRANS_DISTRICTS = '4,7,11,3'; -const DEFAULT_CALTRANS_MAX_SOURCES = 300; -/** Prioritization anchors: downtown cores of the four default metros. */ -const CALTRANS_ANCHORS = [ - { lat: 37.7793, lon: -122.4193 }, // San Francisco - { lat: 34.0537, lon: -118.2428 }, // Los Angeles - { lat: 32.7157, lon: -117.1611 }, // San Diego - { lat: 38.5816, lon: -121.4944 }, // Sacramento -]; -/** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */ -const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam'; -const TFL_IMAGE_ORIGIN = 'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'; -const DEFAULT_TFL_MAX_SOURCES = 250; -const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 }; -/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL) infrequent. Frames are fetched per-request and are unaffected. */ -const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; -/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one - * stalled upstream can't leave getCctvSources (and thus every CCTV route) - * pending forever — a hung fetch aborts, the loader returns [], and - * serve-stale/other packs take over. */ -const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000; -/** Individual CCTV image fetches must settle before the active 10-second - * client refresh cadence. A bounded miss can fall through to Street View or - * the synthetic frame instead of leaving the browser preview pending. */ -export const CCTV_FRAME_FETCH_TIMEOUT_MS = 8 * 1000; -/** @type {Array} Cached merged + normalized CCTV source list. */ -let _cctvSourceCache = []; -/** @type {number} Epoch-ms when the source cache was last refreshed. */ -let _cctvSourceCacheAt = 0; -/** @type {Promise>|null} In-flight refresh, shared by concurrent - * callers so a post-TTL burst launches ONE refetch, not one per request. */ -let _cctvSourceInflight = null; - -/** - * Coerce a value to a finite number, returning fallback if NaN/Infinity. - * - * @param {*} value - * @param {number} [fallback=NaN] - * @returns {number} - */ -function toFiniteNumber(value, fallback = NaN) { - const num = Number(value); - return Number.isFinite(num) ? num : fallback; -} - -/** - * Normalize a column/field name to a lowercase snake_case key. - * - * @param {string} text - * @returns {string} - */ -function normalizeKey(text) { - return String(text || '') - .trim() - .toLowerCase() - .replace(/[^a-z0-9]+/g, '_') - .replace(/^_+|_+$/g, ''); -} - -/** - * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). - * - * @returns {Array} Array of raw source objects, or [] on error. - */ -function loadSourcesFromFile() { - const sourceFile = process.env.CCTV_SOURCES_FILE || DEFAULT_CCTV_SOURCE_FILE; - const resolved = path.isAbsolute(sourceFile) - ? sourceFile - : path.resolve(__dirname, sourceFile); - try { - if (!fs.existsSync(resolved)) return []; - const raw = fs.readFileSync(resolved, 'utf8'); - const parsed = JSON.parse(raw); - return Array.isArray(parsed) ? parsed : []; - } catch (error) { - console.warn('[CCTV] failed to read source file:', resolved, error?.message || error); - return []; - } -} - -/** - * Load CCTV sources from the CCTV_SOURCES_JSON env variable (inline JSON). - * - * @returns {Array} Array of raw source objects, or [] if unset/invalid. - */ -function loadSourcesFromEnv() { - const raw = process.env.CCTV_SOURCES_JSON; - if (!raw) return []; - try { - const parsed = JSON.parse(raw); - return Array.isArray(parsed) ? parsed : []; - } catch { - return []; - } -} - - -/** - * Parse a WKT POINT string (e.g. "POINT(-97.74 30.27)") into lat/lon. - * - * WKT uses (lon lat) order; returned object uses {lat, lon}. - * - * @param {string} value - * @returns {{lat:number, lon:number}} - */ -function parsePointString(value) { - const match = String(value || '').match(/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i); - if (!match) return { lat: NaN, lon: NaN }; - return { - lon: toFiniteNumber(match[1]), - lat: toFiniteNumber(match[2]), - }; -} - -/** - * Extract lat/lon from a variety of coordinate representations. - * - * Handles WKT POINT strings, and objects with latitude/lat/y or - * longitude/lon/lng/x properties (various casing). - * - * @param {string|object|null} value - * @returns {{lat:number, lon:number}} - */ -function coerceLatLon(value) { - if (!value) return { lat: NaN, lon: NaN }; - - if (typeof value === 'string') { - return parsePointString(value); - } - - if (typeof value !== 'object') { - return { lat: NaN, lon: NaN }; - } - - const lat = toFiniteNumber( - value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat, - NaN - ); - const lon = toFiniteNumber( - value.longitude ?? value.lon ?? value.lng ?? value.x ?? value.Longitude ?? value.Lon, - NaN - ); - return { lat, lon }; -} - -/** - * Extract geographic coordinates from an Austin Open Data camera record. - * - * Tries several candidate fields (location, coordinates, the_geom, - * point, geocoded_column) via coerceLatLon, then falls back to - * explicit latitude/longitude scalar fields. - * - * @param {object} record - Flattened camera record. - * @returns {{lat:number, lon:number}} - */ -function extractAustinCoords(record) { - const candidates = [ - record.location, - record.coordinates, - record.the_geom, - record.point, - record.geocoded_column, - ]; - for (const candidate of candidates) { - const parsed = coerceLatLon(candidate); - if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon)) return parsed; - } - - const lat = toFiniteNumber( - record.latitude ?? record.lat ?? record.camera_latitude ?? record.location_latitude, - NaN - ); - const lon = toFiniteNumber( - record.longitude ?? record.lon ?? record.lng ?? record.camera_longitude ?? record.location_longitude, - NaN - ); - return { lat, lon }; -} - -/** - * Extract a numeric camera ID from an Austin Open Data record. - * - * Tries well-known field names first, then scans any field whose key - * contains "camera"/"cam"/"device" + "id". - * - * @param {object} record - Flattened camera record. - * @returns {string} Numeric ID string, or '' if none found. - */ -function extractAustinCameraId(record) { - const preferredKeys = [ - 'camera_id', - 'cameraid', - 'cam_id', - 'device_id', - 'intersection_id', - 'id', - ]; - for (const key of preferredKeys) { - const value = record[key]; - if (value == null) continue; - const asText = String(value).trim(); - if (!asText) continue; - if (/^\d+$/.test(asText)) return asText; - } - - for (const [key, value] of Object.entries(record)) { - if (!/camera|cam|device/.test(key)) continue; - if (!/id/.test(key)) continue; - const asText = String(value || '').trim(); - if (!asText) continue; - if (/^\d+$/.test(asText)) return asText; - } - - return ''; -} - -/** - * Extract a human-readable camera name from an Austin record. - * - * @param {object} record - Flattened camera record. - * @param {string} cameraId - Fallback identifier if no name field found. - * @returns {string} - */ -function extractAustinName(record, cameraId) { - const preferredKeys = [ - 'camera_name', - 'location_name', - 'intersection_name', - 'location', - 'cross_street', - 'description', - 'name', - ]; - for (const key of preferredKeys) { - const value = record[key]; - if (typeof value !== 'string') continue; - const text = value.trim(); - if (text) return text; - } - return `Austin Camera ${cameraId}`; -} - -/** - * Extract camera heading (compass bearing) from an Austin record. - * - * Tries explicit numeric heading fields first, then direction-keyword - * fields, then infers from the camera name/description text. - * - * @param {object} record - Flattened camera record. - * @returns {number} Heading in degrees [0..360), or NaN if unknown. - */ -function extractAustinHeading(record) { - const direct = toFiniteNumber(record.heading_deg ?? record.heading ?? record.bearing, NaN); - if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360; - - // Dedicated direction fields: bare cardinal words ("West") are real facings. - const directionKeys = ['direction', 'travel_direction', 'facing', 'facing_direction']; - for (const key of directionKeys) { - const heading = directionToHeading(record[key], true); - if (Number.isFinite(heading)) return heading; - } - - // Free-form name/intersection text: only explicit travel forms ("WESTBOUND"/ - // "WB") count — a bare "West" here is a street name ("5TH ST / WEST AVE"), not - // a facing, and must not promote the camera to a false high-confidence heading. - const nameProbe = [ - record.camera_name, - record.location_name, - record.intersection_name, - record.location, - record.cross_street, - record.description, - record.name, - ].filter(Boolean).join(' '); - const inferred = directionToHeading(nameProbe); - if (Number.isFinite(inferred)) return inferred; - - return NaN; -} - -/** - * Bounding-box sanity check: is this coordinate plausibly in the Austin metro area? - * - * @param {number} lat - * @param {number} lon - * @returns {boolean} - */ -function isLikelyAustinCoordinate(lat, lon) { - if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; - return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.40; -} - -/** - * Derive a deterministic fallback heading from a camera ID hash. - * - * Produces one of 16 evenly-spaced compass directions (0, 22.5, 45, ...). - * - * @param {string} cameraId - * @returns {number} Heading in degrees [0..360). - */ -function fallbackHeadingFromId(cameraId) { - return (hashSeed(String(cameraId)) % 16) * 22.5; -} - -/** - * Convert a Socrata rows.json array row into a keyed object using column metadata. - * - * @param {Array} row - Array of cell values from the Socrata payload. - * @param {Array<{fieldName?:string, name?:string}>} columns - Column descriptors. - * @returns {object} Keyed record with normalized snake_case keys. - */ -function rowArrayToObject(row, columns) { - const record = {}; - for (let idx = 0; idx < columns.length; idx++) { - const col = columns[idx]; - const key = normalizeKey(col.fieldName || col.name || `col_${idx}`); - if (!key) continue; - record[key] = row[idx]; - } - return record; -} - -/** - * Distance-prioritizes cameras to a cap: keeps the maxCount cameras closest - * to ANY of the given anchor points (min distance over anchors), tie-broken - * by original array order. Used by every live source pack (Austin: one - * downtown anchor; Caltrans: one anchor per major CA metro; TfL: central - * London) so a cap always keeps the densest, most interesting cores. - * - * @param {Array} cameras - Normalized camera source objects. - * @param {number} maxCount - Cap (<=0 or >= length disables). - * @param {Array<{lat:number,lon:number}>} anchors - At least one anchor. - * @returns {Array} Capped, priority-ordered camera list. - */ -function prioritizeSources(cameras, maxCount, anchors) { - const list = Array.isArray(cameras) ? cameras : []; - const anchorList = (Array.isArray(anchors) ? anchors : []).filter( - (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon) - ); - if (!Number.isFinite(maxCount) || maxCount <= 0 || list.length <= maxCount || !anchorList.length) { - return list; - } - - const scored = list.map((camera, idx) => { - const lat = Number(camera?.lat); - const lon = Number(camera?.lon); - const distKm = Number.isFinite(lat) && Number.isFinite(lon) - ? Math.min(...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon))) - : Number.POSITIVE_INFINITY; - return { camera, idx, distKm }; - }); - - scored.sort((a, b) => { - if (a.distKm !== b.distKm) return a.distKm - b.distKm; - return a.idx - b.idx; - }); - - return scored.slice(0, maxCount).map((entry) => entry.camera); -} - -/** - * Fetch and parse Austin traffic camera records from the city Open Data portal. - * - * Downloads the Socrata rows.json payload, converts each row to a keyed - * record, extracts camera ID / coords / heading / name, validates against - * the Austin bounding box, deduplicates by ID, then distance-prioritizes - * to stay within CCTV_AUSTIN_MAX_SOURCES. - * - * @returns {Promise>} Normalized camera source objects. - */ -async function loadAustinSourcesFromOpenData() { - const endpoint = process.env.CCTV_AUSTIN_ROWS_URL || DEFAULT_AUSTIN_ROWS_URL; - try { - const resp = await fetch(endpoint, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); - if (!resp.ok) { - console.warn('[CCTV] Austin source download failed:', resp.status); - return []; - } - const payload = await resp.json(); - const columns = Array.isArray(payload?.meta?.view?.columns) ? payload.meta.view.columns : []; - const rows = Array.isArray(payload?.data) ? payload.data : []; - if (!columns.length || !rows.length) return []; - - const cameras = []; - for (const row of rows) { - if (!Array.isArray(row)) continue; - const record = rowArrayToObject(row, columns); - const cameraId = extractAustinCameraId(record); - if (!cameraId) continue; - - // Only live cameras: the dataset carries DESIRED (planned, not built), - // REMOVED and VOID rows whose frame URLs never resolve — those cameras - // would render as permanent Street View / synthetic fallbacks. Tolerate - // a missing column (keep the row) so a schema change fails open. - const status = String(record.camera_status || '').trim().toUpperCase(); - if (status && status !== 'TURNED_ON') continue; - - const { lat, lon } = extractAustinCoords(record); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; - if (!isLikelyAustinCoordinate(lat, lon)) continue; - - const extractedHeading = extractAustinHeading(record); - const hasHeading = Number.isFinite(extractedHeading); - const headingDeg = hasHeading ? extractedHeading : fallbackHeadingFromId(cameraId); - cameras.push({ - id: cameraId, - name: extractAustinName(record, cameraId), - city: 'Austin', - cityId: 'austin', - provider: 'Austin Transportation & Public Works', - lat, - lon, - headingDeg, - headingConfidence: hasHeading ? 'high' : 'low', - pitchDeg: hasHeading ? -24 : -18, - fovDeg: hasHeading ? 56 : 44, - rangeM: hasHeading ? 210 : 145, - mountHeightM: hasHeading ? 10 : 8, - groundElevationM: 150, - feedType: 'image', - url: `https://cctv.austinmobility.io/image/${encodeURIComponent(cameraId)}.jpg`, - snapshotUrl: `https://cctv.austinmobility.io/image/${encodeURIComponent(cameraId)}.jpg`, - sourceKind: 'austin-open-data', - license: 'Public city traffic camera frame', - }); - } - - const unique = Array.from(new Map(cameras.map((camera) => [camera.id, camera])).values()); - const maxRaw = Number(process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(300, Math.floor(maxRaw))) : DEFAULT_AUSTIN_MAX_SOURCES; - const prioritized = prioritizeSources(unique, maxCount, [AUSTIN_DOWNTOWN]); - if (prioritized.length < unique.length) { - console.log(`[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`); - } else { - console.log('[CCTV] Loaded Austin camera sources:', prioritized.length); - } - return prioritized; - } catch (error) { - console.warn('[CCTV] Austin source download error:', error?.message || error); - return []; - } -} - -/** - * Fetch Caltrans CCTV cameras for the configured districts (CCTV_CALTRANS_DISTRICTS, - * comma-separated 1..12; empty string disables the pack). One official JSON feed per - * district, identical schema statewide; keyless. Only inService cameras with finite - * coords and a cwwp2.dot.ca.gov https image URL are kept (the image-URL origin check - * is defense-in-depth: the proxy only ever fetches catalog URLs, and this pins the - * catalog to the official host). Districts fetch in parallel and fail independently - * (Promise.allSettled) — one district outage never darkens the others. - * - * @returns {Promise>} Normalized camera source objects. - */ -async function loadCaltransSourcesFromOpenData() { - const districtsRaw = process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS; - const districts = String(districtsRaw) - .split(',') - .map((token) => Number(token.trim())) - .filter((n) => Number.isInteger(n) && n >= 1 && n <= 12); - if (!districts.length) return []; - - const settled = await Promise.allSettled( - districts.map(async (district) => { - const resp = await fetch(CALTRANS_CCTV_URL(district), { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); - if (!resp.ok) throw new Error(`D${district} HTTP ${resp.status}`); - const payload = await resp.json(); - const rows = Array.isArray(payload?.data) ? payload.data : []; - return { district, rows }; - }) - ); - - const cameras = []; - for (const result of settled) { - if (result.status !== 'fulfilled') { - console.warn('[CCTV] Caltrans district fetch failed:', result.reason?.message || result.reason); - continue; - } - const { district, rows } = result.value; - for (const row of rows) { - const cctv = row?.cctv; - if (!cctv || String(cctv.inService).toLowerCase() !== 'true') continue; - const loc = cctv.location || {}; - const lat = toFiniteNumber(loc.latitude); - const lon = toFiniteNumber(loc.longitude); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; - - const imageUrl = String(cctv.imageData?.static?.currentImageURL || ''); - // Official-host pin (see JSDoc). Also drops records with no still image. - if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue; - - const locationName = String(loc.locationName || '').trim(); - // Leading token of locationName is the stable camera code ("TV102 -- I-580 : …"). - const codeMatch = /^([A-Za-z0-9_-]+)\s*--/.exec(locationName); - const code = (codeMatch ? codeMatch[1] : `x${cameras.length}`).toLowerCase(); - const cameraId = `ca-d${district}-${code}`; - - // loc.direction is a dedicated field ("West", "South") → allow bare words. - const heading = directionToHeading(loc.direction, true); - const hasHeading = Number.isFinite(heading); - const label = locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') || `Caltrans D${district} ${code}`; - cameras.push({ - id: cameraId, - name: loc.nearbyPlace ? `${label} (${loc.nearbyPlace})` : label, - city: String(loc.nearbyPlace || `Caltrans D${district}`), - cityId: `ca-d${district}`, - provider: 'Caltrans', - lat, - lon, - headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId), - headingConfidence: hasHeading ? 'high' : 'low', - // Same two fabricated pose personalities as Austin (design §1a): these are - // RAW PRIOR starting points; the client's one-shot ground snap + manual - // calibration own the truth. - pitchDeg: hasHeading ? -24 : -18, - fovDeg: hasHeading ? 56 : 44, - rangeM: hasHeading ? 210 : 145, - mountHeightM: hasHeading ? 10 : 8, - // loc.elevation is reported in FEET (verified: D3 maxes at 7427 ft ≈ - // 2264 m for the Sierra passes — as metres that would top Mt Whitney). - // Convert to metres and clamp to a sane CA-roads range so an occasional - // garbage upstream value can't fling a camera kilometres up. Prior only: - // the client one-shot snap corrects it on 3D-tile stacks — but on a - // no-tileset stack (keyless OSM) the snap misses and this height freezes, - // so it must be right-ish on its own. - groundElevationM: (() => { - const ft = toFiniteNumber(loc.elevation, NaN); - return Number.isFinite(ft) ? Math.max(-100, Math.min(4000, ft * 0.3048)) : 150; - })(), - feedType: 'image', - url: imageUrl, - snapshotUrl: imageUrl, - sourceKind: 'caltrans-open-data', - license: 'Public Caltrans highway camera frame', - }); - } - } - - const maxRaw = Number(process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_CALTRANS_MAX_SOURCES; - const prioritized = prioritizeSources(cameras, maxCount, CALTRANS_ANCHORS); - console.log(`[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`); - return prioritized; -} - -/** - * Fetch TfL JamCams (London). Keyless: the optional TFL_APP_KEY only raises the - * list-endpoint rate limit (frames come from TfL's public S3 bucket, which is not - * rate-limited); the 15-min source cache keeps list hits far below anonymous - * limits anyway. Only `available === "true"` cameras with finite coords and an - * image URL on the official bucket are kept. Attribution: "Powered by TfL Open - * Data" (registered in src/data/dataCredits.js). - * - * @returns {Promise>} Normalized camera source objects. - */ -async function loadTflSourcesFromOpenData() { - try { - const appKey = String(process.env.TFL_APP_KEY || '').trim(); - const url = appKey ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}` : TFL_JAMCAM_URL; - const resp = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); - if (!resp.ok) { - console.warn('[CCTV] TfL JamCam download failed:', resp.status); - return []; - } - const places = await resp.json(); - if (!Array.isArray(places)) return []; - - const cameras = []; - for (const place of places) { - const props = {}; - for (const p of place?.additionalProperties || []) { - if (p?.key) props[p.key] = p.value; - } - if (String(props.available).toLowerCase() !== 'true') continue; - const lat = toFiniteNumber(place?.lat); - const lon = toFiniteNumber(place?.lon); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; - const imageUrl = String(props.imageUrl || ''); - if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin - - // "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id). - const rawId = String(place?.id || '').replace(/^JamCams_/, ''); - if (!rawId) continue; - const cameraId = `tfl-${rawId}`; - - cameras.push({ - id: cameraId, - name: String(place?.commonName || `JamCam ${rawId}`), - city: 'London', - cityId: 'london', - provider: 'Transport for London', - lat, - lon, - // No heading signal at all in JamCam data → id-hash fallback, low - // confidence personality (same as headingless Austin cameras). - headingDeg: fallbackHeadingFromId(cameraId), - headingConfidence: 'low', - pitchDeg: -18, - fovDeg: 44, - rangeM: 145, - mountHeightM: 8, - groundElevationM: 15, // Thames-basin prior; one-shot snap corrects. - feedType: 'image', // stills-first (owner decision); props.videoUrl deliberately unused - url: imageUrl, - snapshotUrl: imageUrl, - sourceKind: 'tfl-open-data', - license: 'Powered by TfL Open Data', - }); - } - - const maxRaw = Number(process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_TFL_MAX_SOURCES; - const prioritized = prioritizeSources(cameras, maxCount, [LONDON_CENTER]); - console.log(`[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`); - return prioritized; - } catch (error) { - console.warn('[CCTV] TfL JamCam download error:', error?.message || error); - return []; - } -} - -/** - * Normalize a raw CCTV source item into a canonical shape with safe defaults. - * - * @param {object} item - Raw source from file, env, or Austin Open Data. - * @returns {object} Normalized source with all expected fields populated. - */ -function normalizeSourceItem(item) { - return { - id: String(item.id || '').trim(), - name: String(item.name || item.id || '').trim(), - city: String(item.city || ''), - cityId: String(item.cityId || ''), - provider: String(item.provider || 'Configured CCTV Source'), - lat: toFiniteNumber(item.lat), - lon: toFiniteNumber(item.lon), - headingDeg: toFiniteNumber(item.headingDeg), - headingConfidence: String(item.headingConfidence || item.headingSource || '').toLowerCase(), - pitchDeg: toFiniteNumber(item.pitchDeg), - fovDeg: toFiniteNumber(item.fovDeg), - rangeM: toFiniteNumber(item.rangeM), - mountHeightM: toFiniteNumber(item.mountHeightM), - groundElevationM: toFiniteNumber(item.groundElevationM), - feedType: normalizeFeedType(item.feedType || item.type || ''), - url: typeof item.url === 'string' ? item.url : '', - snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '', - license: String(item.license || item.licenseNote || ''), - sourceKind: String(item.sourceKind || item.kind || 'configured'), - // Optional CAL badge input (cctv-v2 design §3b/§9.2, additive-only per the - // global constraints — nothing else in this file changes): hand-authored - // file/env catalog entries may declare poseSource:'curated' so the panel - // badge can distinguish them from raw automated priors (e.g. Austin Open - // Data, which never sets this field). Passed through as-is to the client. - poseSource: item.poseSource === 'curated' ? 'curated' : undefined, - }; -} - -/** - * Assemble and cache the merged CCTV source list. - * - * Merges sources from three origins (Austin Open Data, local file, - * env variable), deduplicates by ID, applies the global max cap, and - * caches for CCTV_SOURCE_CACHE_MS. - * - * @returns {Promise>} Deduplicated, capped source list. - */ -async function getCctvSources() { - const now = Date.now(); - if (_cctvSourceCache.length && now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS) { - return _cctvSourceCache; - } - // Single-flight: a burst of requests arriving past the TTL shares ONE refresh - // instead of each launching the full multi-provider refetch. The `.finally` - // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; }); - return _cctvSourceInflight; -} - -/** - * Assemble and cache the merged CCTV source list from file/env + live packs. - * Always resolves (loaders self-catch to []); on a fully-empty refresh with a - * good prior catalog it serves stale rather than blanking the CCTV layer. - * - * @returns {Promise>} Deduplicated, capped source list. - */ -async function refreshCctvSources() { - const fromFile = loadSourcesFromFile(); - const fromEnv = loadSourcesFromEnv(); - - const forceAustin = String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1'; - const preferAustin = String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0'; - // Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack - // is configured and live packs aren't forced — same gate that governed the - // Austin-only fetch, now governing all three. Each pack fails independently. - const needsLiveSources = forceAustin || ((fromFile.length + fromEnv.length) === 0 && preferAustin); - const tflEnabled = String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0'; - - let fromAustin = []; - let fromCaltrans = []; - let fromTfl = []; - if (needsLiveSources) { - const [austinResult, caltransResult, tflResult] = await Promise.allSettled([ - loadAustinSourcesFromOpenData(), - loadCaltransSourcesFromOpenData(), - tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]), - ]); - fromAustin = austinResult.status === 'fulfilled' ? austinResult.value : []; - fromCaltrans = caltransResult.status === 'fulfilled' ? caltransResult.value : []; - fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : []; - } - // Live sources first so file/env overrides win on duplicate IDs (Map last-write). - const merged = [...fromAustin, ...fromCaltrans, ...fromTfl, ...fromFile, ...fromEnv]; - - // Deduplicate by camera ID (last-write wins because of Map.set) - const byId = new Map(); - for (const item of merged) { - if (!item || typeof item !== 'object') continue; - const normalized = normalizeSourceItem(item); - if (!normalized.id) continue; - byId.set(normalized.id, normalized); - } - - const mergedSources = Array.from(byId.values()); - const maxRaw = Number(process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) : DEFAULT_CCTV_MAX_SOURCES; - if (mergedSources.length > maxCount) { - console.warn(`[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`); - } - const capped = mergedSources.length > maxCount ? mergedSources.slice(0, maxCount) : mergedSources; - if (capped.length > 0 || _cctvSourceCache.length === 0) { - _cctvSourceCache = capped; - } else { - // Every source came back empty (all live packs timed out / upstream outage) - // but a good catalog is already cached — serve it stale rather than blanking - // every CCTV route. Advancing the timestamp waits one TTL before retrying, - // which (with single-flight) bounds load on a persistently-down upstream. - console.warn(`[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`); - } - _cctvSourceCacheAt = Date.now(); - return _cctvSourceCache; -} - -/** - * Generate a synthetic SVG billboard image for a CCTV camera placeholder. - * - * Produces a 960x540 SVG with a deterministic gradient (hue derived from - * camera ID hash), scanline overlay, HUD-style grid, and text labels - * showing camera name, city, ID, status, and current timestamp. Used - * when no upstream image or Street View fallback is available. - * - * @param {object} opts - * @param {string} opts.cameraId - * @param {string} opts.label - * @param {string} [opts.city] - * @param {string} [opts.status] - * @returns {string} SVG markup string. - */ -function buildSyntheticCctvSvg({ cameraId, label, city, status }) { - const seed = hashSeed(`${cameraId}:${label}:${city}`); - const hue = seed % 360; - const hue2 = (hue + 46) % 360; - const now = new Date(); - const ts = now.toISOString().replace('T', ' ').replace('Z', 'Z').slice(0, 20); - const safeLabel = escapeXml(label); - const safeCity = escapeXml(city || 'GLOBAL GRID'); - const safeId = escapeXml(cameraId); - const safeStatus = escapeXml(status || 'SYNTHETIC'); - - return ` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - CCTV FEED PLACEHOLDER - ${safeLabel} · ${safeCity} - ${safeId} - ${escapeXml(ts)} - ${safeStatus} - -`.trim(); -} - -/** - * Coerce a fetch() response body to a Node.js Readable stream. - * - * Handles both Node-native streams (.pipe) and web ReadableStreams (.getReader). - * - * @param {ReadableStream|NodeJS.ReadableStream|null} body - * @returns {import('stream').Readable|null} - */ -function toReadable(body) { - if (!body) return null; - if (typeof body.pipe === 'function') return body; - if (typeof body.getReader === 'function') { - return Readable.fromWeb(body); - } - return null; -} - -/** - * Pipe an upstream fetch Response (image or video) to the client HTTP response. - * - * Forwards Content-Type, Content-Length, Content-Range, Accept-Ranges, and - * Cache-Control headers from the upstream. Falls back to buffered arrayBuffer - * if the body is not streamable. - * - * @param {import('http').ServerResponse} res - * @param {Response} upstream - fetch() Response object. - * @param {object} [opts] - * @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header. - */ -async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstream' } = {}) { - const contentType = upstream.headers.get('content-type') || 'application/octet-stream'; - const cacheControl = upstream.headers.get('cache-control') || 'no-store'; - const contentLength = upstream.headers.get('content-length'); - const contentRange = upstream.headers.get('content-range'); - const acceptRanges = upstream.headers.get('accept-ranges'); - const headers = { - 'Content-Type': contentType, - 'Cache-Control': cacheControl, - 'X-CCTV-Source': sourceHeader, - }; - if (contentLength) headers['Content-Length'] = contentLength; - if (contentRange) headers['Content-Range'] = contentRange; - if (acceptRanges) headers['Accept-Ranges'] = acceptRanges; - - // Cheap defense: reject an upstream that DECLARES an oversized fixed body. - // Live MJPEG/HLS streams are unbounded by design and send no content-length, - // so they pipe normally (piping streams to the client, never buffering). - const MEDIA_DECLARED_CAP_BYTES = 64 * 1024 * 1024; - if (Number.isFinite(Number(contentLength)) && Number(contentLength) > MEDIA_DECLARED_CAP_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Upstream media exceeds size cap' })); - try { await upstream.body?.cancel(); } catch { /* no-op */ } - return; - } - - res.writeHead(upstream.status, headers); - - const stream = toReadable(upstream.body); - if (!stream) { - const buf = Buffer.from(await upstream.arrayBuffer()); - res.end(buf); - return; - } - - stream.on('error', () => { - if (!res.writableEnded) res.end(); - }); - stream.pipe(res); -} - -/** - * Fetch one upstream CCTV image within the frame-refresh budget. - * - * A timeout is treated like every other upstream miss so the caller can - * continue through the Street View and synthetic fallback chain. `fetchImpl` - * and `timeoutMs` are injectable only to keep the timeout contract unit-testable. - * - * @param {string} url - Server-registered upstream image URL. - * @param {object} [options] - * @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation. - * @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout. - * @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>} - */ -export async function fetchCctvImageFromUpstream(url, { - fetchImpl = fetch, - timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS, -} = {}) { - if (!url || !/^https?:\/\//i.test(url)) return null; - const controller = new AbortController(); - const timeoutId = setTimeout(() => { - controller.abort(new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError')); - }, timeoutMs); - try { - const upstream = await fetchImpl(url, { - headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }, - signal: controller.signal, - }); - const contentType = upstream.headers.get('content-type') || ''; - if (!upstream.ok || !contentType.startsWith('image/')) return null; - return { - ok: true, - body: Buffer.from(await upstream.arrayBuffer()), - contentType, - }; - } catch { - return null; - } finally { - clearTimeout(timeoutId); - } -} - -/** - * Vite plugin: CCTV camera proxy with source registry, frame/media serving, - * fallback chain (upstream -> Street View -> synthetic SVG), and health tracking. - * - * Endpoints: - * GET /api/cctv/sources — list all registered camera sources - * GET /api/cctv/health — per-camera health/status report - * GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera - * GET /api/cctv/media/:id — proxy live video/image media from upstream - * GET /api/cctv/frame/:id — single frame with fallback chain - * - * @returns {import('vite').Plugin} - */ -function cctvProxy() { - /** @type {Map} */ - const health = new Map(); - /** Cap on health map entries to prevent unbounded growth. Sized to cover the - * full served catalog (CCTV_MAX_SOURCES hard-bounds at 1200) so health/status - * observability isn't silently evicted for a default 800-camera catalog. */ - const HEALTH_MAX_ENTRIES = 1200; - - /** Update the health entry for a camera, evicting the oldest entry if at capacity. */ - const setHealth = (cameraId, patch) => { - // Evict oldest entries if the health map grows beyond the cap - if (!health.has(cameraId) && health.size >= HEALTH_MAX_ENTRIES) { - const oldest = health.keys().next().value; - health.delete(oldest); - } - const prev = health.get(cameraId) || {}; - health.set(cameraId, { - id: cameraId, - status: patch.status || prev.status || 'unknown', - sourceKind: patch.sourceKind || prev.sourceKind || 'unknown', - label: patch.label || prev.label || '', - message: patch.message || prev.message || '', - updatedAt: Date.now(), - }); - }; - - /** Snapshot all camera health entries as an array. */ - const listHealth = () => Array.from(health.values()); - - /** Build a JSON payload describing stream info (feedType, URLs) for a camera. */ - const buildStreamPayload = (source, cameraId) => { - const feedType = normalizeFeedType(source?.feedType || 'image'); - return { - id: cameraId, - feedType, - mediaUrl: isVideoFeedType(feedType) - ? `/api/cctv/media/${encodeURIComponent(cameraId)}` - : null, - frameUrl: `/api/cctv/frame/${encodeURIComponent(cameraId)}`, - provider: source?.provider || '', - sourceKind: source?.sourceKind || (source?.url ? 'configured' : 'fallback'), - }; - }; - - /** - * Fetch a Google Street View static image as a fallback frame. Server-side - * call, never reaches the browser — prefers GOOGLE_MAPS_SERVER_API_KEY - * (#33: a key scoped to Street View Static/Places, restricted by server IP - * rather than HTTP referrer) and falls back to the browser-exposed - * GOOGLE_MAPS_API_KEY for setups that haven't split the two yet. - */ - const streetViewFallback = async ({ lat, lon, heading, fov, pitch }) => { - const streetViewKey = googleServerApiKey(); - if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon)) return null; - try { - const sv = new URL('https://maps.googleapis.com/maps/api/streetview'); - sv.searchParams.set('size', '960x540'); - sv.searchParams.set('location', `${lat},${lon}`); - sv.searchParams.set('heading', String(Number.isFinite(heading) ? heading : 0)); - sv.searchParams.set('fov', String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80)); - sv.searchParams.set('pitch', String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0)); - sv.searchParams.set('source', 'outdoor'); - sv.searchParams.set('return_error_code', 'true'); - sv.searchParams.set('key', streetViewKey); - - const svResp = await fetch(sv.toString(), { - headers: { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }, - signal: AbortSignal.timeout(CCTV_FRAME_FETCH_TIMEOUT_MS), - }); - const svType = svResp.headers.get('content-type') || ''; - if (!svResp.ok || !svType.startsWith('image/')) return null; - - return { - ok: true, - body: Buffer.from(await svResp.arrayBuffer()), - contentType: svType, - }; - } catch { - return null; - } - }; - - return { - name: 'cctv-proxy', - configureServer(server) { - server.middlewares.use('/api/cctv', async (req, res) => { - try { - const sources = await getCctvSources(); - const sourceById = new Map(sources.map((source) => [source.id, source])); - const url = new URL(req.url || '/', 'http://localhost'); - - if (url.pathname === '/sources') { - const body = { - sources: sources.map((source) => ({ - id: source.id, - name: source.name, - city: source.city, - cityId: source.cityId, - provider: source.provider, - lat: source.lat, - lon: source.lon, - headingDeg: source.headingDeg, - headingConfidence: source.headingConfidence || '', - pitchDeg: source.pitchDeg, - fovDeg: source.fovDeg, - rangeM: source.rangeM, - mountHeightM: source.mountHeightM, - groundElevationM: source.groundElevationM, - feedType: normalizeFeedType(source.feedType), - sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), - poseSource: source.poseSource, - license: source.license, - })), - }; - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify(body)); - return; - } - - if (url.pathname === '/health') { - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ cameras: listHealth() })); - return; - } - - if (url.pathname.startsWith('/stream/')) { - const cameraId = decodeURIComponent(url.pathname.replace('/stream/', '').trim()) || 'camera'; - const source = sourceById.get(cameraId); - const payload = buildStreamPayload(source, cameraId); - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify(payload)); - return; - } - - if (url.pathname.startsWith('/media/')) { - const cameraId = decodeURIComponent(url.pathname.replace('/media/', '').trim()) || 'camera'; - const source = sourceById.get(cameraId); - const mediaUrl = source?.url || ''; - const feedType = normalizeFeedType(source?.feedType || 'image'); - - if (!mediaUrl || !/^https?:\/\//i.test(mediaUrl)) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'fallback', - label: source?.provider || 'No upstream URL', - message: 'No stream URL configured', - }); - res.writeHead(404, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'No media URL configured for this camera' })); - return; - } - - try { - const upstreamHeaders = { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }; - const requestRange = req.headers?.range; - if (requestRange) upstreamHeaders.Range = requestRange; - const upstream = await fetch(mediaUrl, { - headers: upstreamHeaders, - }); - const contentType = upstream.headers.get('content-type') || ''; - if (!upstream.ok) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'upstream', - label: source?.provider || 'Configured source', - message: `Upstream HTTP ${upstream.status}`, - }); - res.writeHead(upstream.status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: `Upstream returned ${upstream.status}` })); - return; - } - - if (isVideoFeedType(feedType) && !(contentType.startsWith('video/') || contentType.includes('mpegurl'))) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'upstream', - label: source?.provider || 'Configured source', - message: `Unexpected media type ${contentType || 'unknown'}`, - }); - } else { - setHealth(cameraId, { - status: 'ok', - sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot', - label: source?.provider || 'Configured source', - message: isVideoFeedType(feedType) ? 'Live stream connected' : 'Snapshot feed connected', - }); - } - - await proxyMediaResponse(res, upstream, { - sourceHeader: isVideoFeedType(feedType) ? 'live-media' : 'upstream-image', - }); - return; - } catch (error) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'upstream', - label: source?.provider || 'Configured source', - message: error?.message || 'Media fetch failed', - }); - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Media proxy failed' })); - return; - } - } - - if (!url.pathname.startsWith('/frame/')) { - res.writeHead(404, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'not found' })); - return; - } - - const cameraId = decodeURIComponent(url.pathname.replace('/frame/', '').trim()) || 'camera'; - const source = sourceById.get(cameraId); - const label = url.searchParams.get('label') || source?.name || cameraId; - const city = url.searchParams.get('city') || source?.city || ''; - const lat = Number(url.searchParams.get('lat') || source?.lat); - const lon = Number(url.searchParams.get('lon') || source?.lon); - const heading = Number(url.searchParams.get('heading') || source?.headingDeg); - const fov = Number(url.searchParams.get('fov') || source?.fovDeg); - const pitch = Number(url.searchParams.get('pitch') || source?.pitchDeg); - - // Only use server-registered upstream URLs — never accept client-supplied URLs - // (prevents SSRF via ?upstream= query parameter) - const upstreamCandidate = - source?.snapshotUrl - || (!isVideoFeedType(normalizeFeedType(source?.feedType)) ? source?.url : ''); - - const upstreamImage = await fetchCctvImageFromUpstream(upstreamCandidate); - if (upstreamImage?.ok) { - setHealth(cameraId, { - status: 'ok', - sourceKind: 'snapshot', - label: source?.provider || 'Configured source', - message: 'Upstream snapshot active', - }); - res.writeHead(200, { - 'Content-Type': upstreamImage.contentType, - 'Cache-Control': 'no-store', - 'X-CCTV-Source': 'upstream-image', - }); - res.end(upstreamImage.body); - return; - } - - const sv = await streetViewFallback({ lat, lon, heading, fov, pitch }); - if (sv?.ok) { - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'streetview', - label: 'Google Street View', - message: 'Fallback Street View frame', - }); - res.writeHead(200, { - 'Content-Type': sv.contentType, - 'Cache-Control': 'no-store', - 'X-CCTV-Source': 'streetview', - }); - res.end(sv.body); - return; - } - - const svg = buildSyntheticCctvSvg({ - cameraId, - label, - city, - status: source?.url ? 'UPSTREAM UNAVAILABLE' : 'NO UPSTREAM CONFIGURED', - }); - - setHealth(cameraId, { - status: 'degraded', - sourceKind: 'synthetic', - label: source?.provider || 'Synthetic fallback', - message: source?.url ? 'Upstream unavailable' : 'No source configured', - }); - - res.writeHead(200, { - 'Content-Type': 'image/svg+xml', - 'Cache-Control': 'no-store', - 'X-CCTV-Source': 'synthetic', - }); - res.end(svg); - } catch (error) { - console.error('[CCTV Proxy]', error?.message || String(error)); - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ error: 'CCTV proxy error' })); - } - }); - }, - }; -} /** * Vite plugin: OpenAI Realtime ephemeral client secret. @@ -4658,7 +2834,7 @@ export function localProviderPlugins() { militaryInstallationsProxy(), regionalBriefProxy(), weatherEffectsProxy(), - cctvProxy(), + cctvProxy({ sourceRoot: __dirname }), radioBrowserProxy(), gbfsProxy(), adsbLolProxy(), diff --git a/server/providers/radio.js b/server/providers/radio.js new file mode 100644 index 000000000..c36010745 --- /dev/null +++ b/server/providers/radio.js @@ -0,0 +1,15 @@ +import { createRadioProxyMiddleware } from './radio/catalog.js'; +export { createRadioProxyMiddleware }; +export { isPublicRadioAddress } from './radio/transport.js'; +export { normalizeRadioBrowserStation, publicRadioStation, publicRadioHttpsUrl } from './radio/stations.js'; +export function radioBrowserProxy() { + const middleware = createRadioProxyMiddleware(); + const install = (server) => { + server.middlewares.use('/api/radio', middleware); + }; + return { + name: 'radio-browser-proxy', + configureServer: install, + configurePreviewServer: install, + }; +} diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js new file mode 100644 index 000000000..3abf59af2 --- /dev/null +++ b/server/providers/radio/catalog.js @@ -0,0 +1,277 @@ +import { lookup as lookupDns } from 'node:dns/promises'; +import { randomUUID } from 'node:crypto'; + +import { readResponseTextCapped } from '../common/http.js'; +import { normalizeRadioBrowserStation, publicRadioStation, cleanRadioText } from './stations.js'; +import { radioMirrorOrigin, radioProxyDestination, resolveRadioProxyAddresses, fetchPinnedRadioResponse } from './transport.js'; +import { RADIO_DIRECTORY_CACHE_MS, RADIO_DIRECTORY_STALE_MS, RADIO_MIRROR_CACHE_MS, RADIO_FETCH_TIMEOUT_MS, RADIO_RESPONSE_MAX_BYTES, RADIO_DIRECTORY_LIMIT, RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES, RADIO_CATALOG_HEALTHY_MIN_STATIONS, RADIO_USER_AGENT, RADIO_UUID_RE, RADIO_FALLBACK_MIRRORS } from './constants.js'; +export async function mapRadioConcurrent(values, concurrency, mapper) { + const results = new Array(values.length); + let cursor = 0; + const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { + for (;;) { + const index = cursor++; + if (index >= values.length) return; + results[index] = await mapper(values[index], index); + } + }); + await Promise.all(workers); + return results; +} + +/** Create the testable Connect middleware backing `/api/radio`. */ +export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = lookupDns, now = Date.now } = {}) { + let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 }; + let mirrorPromise = null; + let catalogCache = null; + let catalogGeneration = 0; + // The generation counter is process-local, so it restarts from 1 with the + // server. The instance token scopes each generation sequence: a client that + // sees a new instance must treat the catalog as a fresh sequence, never as a + // repeat ("still generation 1") or a regression ("generation went backward"). + const catalogInstance = randomUUID(); + let servedStationIds = new Set(); + let refreshPromise = null; + + async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) { + const destination = radioProxyDestination(url); + if (!destination) throw new Error('Radio Browser destination is not permitted'); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), RADIO_FETCH_TIMEOUT_MS); + try { + const addresses = await resolveRadioProxyAddresses(destination.hostname, lookupImpl); + const options = { + headers: { Accept: 'application/json', 'User-Agent': RADIO_USER_AGENT }, + signal: controller.signal, + redirect: 'manual', + }; + const response = fetchImpl + ? await fetchImpl(destination.href, options) + : await fetchPinnedRadioResponse(destination, options, addresses); + if (response.status >= 300 && response.status < 400) { + try { await response.body?.cancel?.(); } catch { /* no-op */ } + throw new Error('Radio Browser redirects are refused'); + } + if (!response.ok) throw new Error(`Radio Browser returned ${response.status}`); + const text = await readResponseTextCapped(response, maxBytes); + return JSON.parse(text); + } finally { + clearTimeout(timer); + } + } + + async function mirrors() { + if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS) return mirrorCache.origins; + if (!mirrorPromise) { + mirrorPromise = (async () => { + try { + const rows = await fetchJson('https://all.api.radio-browser.info/json/servers', 256 * 1024); + const discovered = [...new Set((Array.isArray(rows) ? rows : []).map((row) => radioMirrorOrigin(row?.name)).filter(Boolean))]; + if (discovered.length) { + mirrorCache = { origins: [...discovered, ...RADIO_FALLBACK_MIRRORS.filter((origin) => !discovered.includes(origin))], cachedAt: now() }; + } + } catch { + mirrorCache = { ...mirrorCache, cachedAt: now() }; + } + return mirrorCache.origins; + })().finally(() => { mirrorPromise = null; }); + } + return mirrorPromise; + } + + async function fetchPath(pathname) { + let lastError = null; + for (const origin of await mirrors()) { + try { + return await fetchJson(`${origin}${pathname}`); + } catch (error) { + lastError = error; + } + } + throw lastError || new Error('No Radio Browser mirror is available'); + } + + async function refreshCatalog() { + const queries = [null, 'news', 'talk', 'weather', 'emergency', 'scanner', 'aviation', 'marine', 'traffic']; + const outcomes = await mapRadioConcurrent(queries, 3, async (tag, index) => { + const params = new URLSearchParams({ + has_geo_info: 'true', + is_https: 'true', + hidebroken: 'true', + order: 'clickcount', + reverse: 'true', + limit: index === 0 ? '1800' : '220', + }); + if (tag) params.set('tag', tag); + try { + const rows = await fetchPath(`/json/stations/search?${params}`); + if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); + if (!rows.every((row) => ( + row + && typeof row === 'object' + && !Array.isArray(row) + && typeof row.stationuuid === 'string' + && typeof row.name === 'string' + && (typeof row.url_resolved === 'string' || typeof row.url === 'string') + ))) throw new Error('Radio Browser catalog contained a malformed station row'); + const stations = rows.map(normalizeRadioBrowserStation).filter(Boolean); + const requestedTag = cleanRadioText(tag, 80) + .toLocaleLowerCase() + .replace(/[_-]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const requestedTagCovered = !requestedTag || stations.some((station) => ( + station.tags.some((stationTag) => stationTag === requestedTag || stationTag.includes(requestedTag)) + )); + return { + // Query coverage is based on accepted rows, not merely a payload that + // happens to match the upstream schema. Specialist responses must + // also contain an accepted station tagged for the requested category. + succeeded: stations.length > 0 && requestedTagCovered, + stations, + }; + } catch { + return { succeeded: false, stations: [] }; + } + }); + const resultSets = outcomes.map((outcome) => outcome.stations); + + const selected = []; + const seen = new Set(); + const take = (station) => { + if (!station || seen.has(station.id) || selected.length >= RADIO_DIRECTORY_LIMIT) return; + seen.add(station.id); + selected.push(station); + }; + // Seed specialist station-tag queries before popularity fill so operational + // categories remain represented even when global click charts skew musical. + for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take); + resultSets.flat().sort((a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name)).forEach(take); + const timestamp = now(); + const successfulQueries = outcomes.filter((outcome) => outcome.succeeded).length; + const broadQueryHealthy = outcomes[0].succeeded && outcomes[0].stations.length > 0; + const healthReasons = []; + if (!broadQueryHealthy) healthReasons.push('broad-query-unhealthy'); + if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES) healthReasons.push('query-coverage-below-policy'); + if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); + const degraded = healthReasons.length > 0; + const coverage = { + successfulQueries, + totalQueries: queries.length, + stationCount: selected.length, + healthyStationMinimum: RADIO_CATALOG_HEALTHY_MIN_STATIONS, + }; + const nextCatalog = { + cachedAt: timestamp, + updatedAt: new Date(timestamp).toISOString(), + stations: selected.map(publicRadioStation), + stationIds: new Set(selected.map((station) => station.id)), + degraded, + degradedReason: degraded ? healthReasons.join(',') : null, + coverage, + }; + if (degraded && catalogCache) { + const error = new Error('Radio Browser catalog refresh did not meet health policy'); + error.radioCatalogDegraded = true; + error.radioDegradedReason = nextCatalog.degradedReason; + error.radioCoverage = coverage; + throw error; + } + if (degraded && !selected.length) { + const error = new Error('Radio Browser catalog refresh returned no usable stations'); + error.radioCatalogDegraded = true; + error.radioDegradedReason = nextCatalog.degradedReason; + error.radioCoverage = coverage; + throw error; + } + if (degraded) { + servedStationIds = nextCatalog.stationIds; + return { ...nextCatalog, acceptedGeneration: null }; + } + catalogCache = { + ...nextCatalog, + acceptedGeneration: ++catalogGeneration, + }; + servedStationIds = catalogCache.stationIds; + return catalogCache; + } + + async function getCatalog() { + if (catalogCache && now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS) { + return { ...catalogCache, stale: false }; + } + if (!refreshPromise) { + refreshPromise = refreshCatalog().finally(() => { refreshPromise = null; }); + } + try { + return { ...await refreshPromise, stale: false }; + } catch (error) { + if (catalogCache && now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS) { + return { + ...catalogCache, + stale: true, + degraded: true, + degradedReason: error?.radioDegradedReason || 'refresh-failed', + coverage: error?.radioCoverage || catalogCache.coverage, + }; + } + throw error; + } + } + + function sendJson(res, status, body) { + res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(body)); + } + + return async function radioProxyMiddleware(req, res) { + const requestUrl = new URL(req.url || '/', 'http://localhost'); + if (requestUrl.pathname === '/stations') { + if (req.method !== 'GET') { + res.writeHead(405, { Allow: 'GET', 'Cache-Control': 'no-store' }); + res.end(); + return; + } + try { + const catalog = await getCatalog(); + sendJson(res, 200, { + stations: catalog.stations, + updatedAt: catalog.updatedAt, + stale: catalog.stale, + degraded: Boolean(catalog.degraded), + degradedReason: catalog.degradedReason || null, + coverage: catalog.coverage || null, + acceptedGeneration: catalog.acceptedGeneration ?? null, + catalogInstance, + }); + } catch (error) { + sendJson(res, 503, { + error: 'Radio directory is temporarily unavailable', + degraded: Boolean(error?.radioCatalogDegraded), + degradedReason: error?.radioDegradedReason || null, + }); + } + return; + } + + const clickMatch = requestUrl.pathname.match(/^\/click\/([0-9a-f-]+)$/i); + if (clickMatch) { + if (req.method !== 'POST') { + res.writeHead(405, { Allow: 'POST', 'Cache-Control': 'no-store' }); + res.end(); + return; + } + const id = clickMatch[1].toLowerCase(); + if (!RADIO_UUID_RE.test(id) || !servedStationIds.has(id)) { + sendJson(res, 404, { error: 'Unknown radio station' }); + return; + } + res.writeHead(204, { 'Cache-Control': 'no-store' }); + res.end(); + void fetchPath(`/json/url/${id}`).catch(() => {}); + return; + } + + sendJson(res, 404, { error: 'Unknown radio route' }); + }; +} diff --git a/server/providers/radio/constants.js b/server/providers/radio/constants.js new file mode 100644 index 000000000..2d935d83d --- /dev/null +++ b/server/providers/radio/constants.js @@ -0,0 +1,15 @@ +export const RADIO_DIRECTORY_CACHE_MS = 45 * 60 * 1000; +export const RADIO_DIRECTORY_STALE_MS = 7 * 24 * 60 * 60 * 1000; +export const RADIO_MIRROR_CACHE_MS = 6 * 60 * 60 * 1000; +export const RADIO_FETCH_TIMEOUT_MS = 12_000; +export const RADIO_RESPONSE_MAX_BYTES = 4 * 1024 * 1024; +export const RADIO_DIRECTORY_LIMIT = 750; +export const RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES = 5; +export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(RADIO_DIRECTORY_LIMIT / 2); +export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; +export const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export const RADIO_FALLBACK_MIRRORS = Object.freeze([ + 'https://de1.api.radio-browser.info', + 'https://de2.api.radio-browser.info', + 'https://nl1.api.radio-browser.info', +]); diff --git a/server/providers/radio/stations.js b/server/providers/radio/stations.js new file mode 100644 index 000000000..fdf69e62c --- /dev/null +++ b/server/providers/radio/stations.js @@ -0,0 +1,119 @@ +import { RADIO_UUID_RE } from './constants.js'; +import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js'; +export function cleanRadioText(value, maxLength) { + return String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, maxLength).trim(); +} + +export function isNonGlobalIpv4(hostname) { + const pieces = hostname.split('.'); + if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) return false; + const values = pieces.map(Number); + if (values.some((value) => value > 255)) return true; + const [a, b, c] = values; + return a === 0 || a === 10 || a === 127 || a >= 224 + || (a === 100 && b >= 64 && b <= 127) + || (a === 169 && b === 254) + || (a === 172 && b >= 16 && b <= 31) + || (a === 192 && b === 0) + || (a === 192 && b === 88 && c === 99) + || (a === 192 && b === 168) + || (a === 198 && (b === 18 || b === 19)) + || (a === 198 && b === 51 && c === 100) + || (a === 203 && b === 0 && c === 113); +} + +/** Return a normalized public HTTPS URL, or null for local/private targets. */ +export function publicRadioHttpsUrl(value) { + try { + const url = new URL(String(value ?? '')); + const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, ''); + if (url.protocol !== 'https:' || url.username || url.password || !hostname) return null; + if ( + hostname === 'localhost' + || hostname.endsWith('.localhost') + || hostname.endsWith('.local') + || isNonGlobalIpv4(hostname) + || hostname.includes(':') + ) return null; + url.hash = ''; + return url.href; + } catch { + return null; + } +} + +/** Normalize one Radio Browser station and omit favicons and unsafe streams. */ +export function normalizeRadioBrowserStation(raw) { + const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); + const lat = raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); + const lon = raw?.geo_long === null || raw?.geo_long === '' ? null : Number(raw?.geo_long); + const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); + const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url); + if ( + !RADIO_UUID_RE.test(id) + || Number(raw?.lastcheckok) !== 1 + || Number(raw?.hls) === 1 + || !Number.isFinite(lat) || lat < -90 || lat > 90 + || !Number.isFinite(lon) || lon < -180 || lon > 180 + || !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) + || !streamUrl + ) return null; + + const name = cleanRadioText(raw?.name, 140); + if (!name) return null; + const tags = String(raw?.tags ?? '') + .split(',') + .map((tag) => cleanRadioText(tag, 80).toLocaleLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim()) + .filter(Boolean) + .filter((tag, index, all) => all.indexOf(tag) === index) + .slice(0, 24); + const languages = String(raw?.language ?? '') + .split(',') + .map((language) => cleanRadioText(language, 40)) + .filter(Boolean) + .slice(0, 8); + const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); + const normalizedCode = normalizeRadioCountryInput(rawCountryCode); + const normalizedCountry = normalizedCode.valid && !normalizedCode.empty + ? normalizedCode + : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); + const bitrate = Number(raw?.bitrate); + return { + id, + name, + lat, + lon, + streamUrl, + homepage: publicRadioHttpsUrl(raw?.homepage), + tags, + languages, + state: cleanRadioText(raw?.state, 80), + country: normalizedCountry.valid && !normalizedCountry.empty + ? normalizedCountry.name + : cleanRadioText(raw?.country, 80), + countryCode: normalizedCountry.valid ? normalizedCountry.code : '', + metadataTrust: 'untrusted-community', + codec, + bitrate: Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 ? bitrate : null, + clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), + }; +} + +export function publicRadioStation(station) { + return { + id: station.id, + name: station.name, + lat: station.lat, + lon: station.lon, + streamUrl: station.streamUrl, + homepage: station.homepage, + tags: station.tags, + languages: station.languages, + state: station.state, + country: station.country, + countryCode: station.countryCode, + metadataTrust: station.metadataTrust, + codec: station.codec, + bitrate: station.bitrate, + }; +} diff --git a/server/providers/radio/transport.js b/server/providers/radio/transport.js new file mode 100644 index 000000000..d9b220896 --- /dev/null +++ b/server/providers/radio/transport.js @@ -0,0 +1,107 @@ +import https from 'node:https'; +import { Readable } from 'node:stream'; +import { isNonGlobalIpv4 } from './stations.js'; +export function radioMirrorOrigin(value) { + const hostname = String(value ?? '').toLowerCase().replace(/\.$/, ''); + if (!/^[a-z0-9-]+\.api\.radio-browser\.info$/.test(hostname)) return null; + return `https://${hostname}`; +} + +/** Return whether a resolved Radio Browser address is safe for an outbound request. */ +export function isPublicRadioAddress(value) { + const address = String(value ?? '').trim().toLowerCase().replace(/^\[|\]$/g, ''); + if (!address) return false; + if (!address.includes(':')) { + const ipv4 = address.split('.'); + return ipv4.length === 4 + && ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) + && !isNonGlobalIpv4(address); + } + const pieces = address.split('::'); + if (pieces.length > 2) return false; + const left = pieces[0] ? pieces[0].split(':') : []; + const right = pieces[1] ? pieces[1].split(':') : []; + const missing = 8 - left.length - right.length; + if ((pieces.length === 1 && missing !== 0) || (pieces.length === 2 && missing < 1)) return false; + const groups = [...left, ...Array(Math.max(0, missing)).fill('0'), ...right]; + if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) return false; + const numeric = groups.reduce((total, group) => (total << 16n) | BigInt(`0x${group}`), 0n); + const inCidr = (base, prefix) => { + const shift = 128n - BigInt(prefix); + return (numeric >> shift) === (base >> shift); + }; + const base = (text) => text.split(':').reduce( + (total, group) => (total << 16n) | BigInt(`0x${group || '0'}`), + 0n, + ); + const cidr = (text, prefix) => inCidr(base(text), prefix); + return cidr('2000:0:0:0:0:0:0:0', 3) + && !cidr('2001:0:0:0:0:0:0:0', 23) + && !cidr('2001:db8:0:0:0:0:0:0', 32) + && !cidr('2002:0:0:0:0:0:0:0', 16) + && !cidr('3fff:0:0:0:0:0:0:0', 20); +} + +export function radioProxyDestination(value) { + let url; + try { + url = new URL(String(value)); + } catch { + return null; + } + const origin = radioMirrorOrigin(url.hostname); + if ( + !origin + || url.origin !== origin + || url.username + || url.password + || url.port + || url.hash + ) return null; + const discovery = url.hostname.toLowerCase() === 'all.api.radio-browser.info' + && url.pathname === '/json/servers' + && !url.search; + const directory = url.pathname === '/json/stations/search'; + const click = /^\/json\/url\/[0-9a-f-]+$/i.test(url.pathname) && !url.search; + return discovery || directory || click ? url : null; +} + +export async function resolveRadioProxyAddresses(hostname, lookupImpl) { + const resolved = await lookupImpl(hostname, { all: true, verbatim: true }); + const rows = Array.isArray(resolved) ? resolved : [resolved]; + const addresses = rows + .map((row) => ({ address: String(row?.address || ''), family: Number(row?.family) || undefined })) + .filter((row) => row.address); + if (!addresses.length || addresses.some((row) => !isPublicRadioAddress(row.address))) { + throw new Error('Radio Browser resolved to a forbidden address'); + } + return addresses; +} + +export function fetchPinnedRadioResponse(url, options, addresses) { + return new Promise((resolve, reject) => { + const address = addresses[0]; + const request = https.request(url, { + method: 'GET', + headers: options.headers, + signal: options.signal, + lookup(_hostname, lookupOptions, callback) { + if (lookupOptions?.all) callback(null, addresses); + else callback(null, address.address, address.family); + }, + }, (response) => { + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + resolve(new Response(Readable.toWeb(response), { + status: response.statusCode || 500, + statusText: response.statusMessage || '', + headers, + })); + }); + request.on('error', reject); + request.end(); + }); +} diff --git a/src/tooling/mediaProviders.test.mjs b/src/tooling/mediaProviders.test.mjs new file mode 100644 index 000000000..ce713ad0b --- /dev/null +++ b/src/tooling/mediaProviders.test.mjs @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { cctvProxy } from '../../server/providers/cctv.js'; +import { radioBrowserProxy } from '../../server/providers/radio.js'; +import { localProviderPlugins } from '../../server/providers/local.js'; + +function install(plugin) { + let handler; + plugin.configureServer({ middlewares: { use(_route, fn) { handler = fn; } } }); + return async (url) => { + const res = { + writeHead(status, headers) { Object.assign(this, { status, headers }); }, + end(body) { this.body = body; }, + }; + await handler({ url, method: 'GET' }, res); + return res; + }; +} + +function fixture(t, id) { + const root = mkdtempSync(path.join(tmpdir(), 'gev-cctv-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(path.join(root, 'config')); + writeFileSync(path.join(root, 'config/cctv_sources.austin.json'), JSON.stringify([ + { id, name: '', lat: 30.27, lon: -97.74, feedType: 'video' }, + ])); + return root; +} + +function isolate(t) { + for (const name of ['CCTV_SOURCES_FILE', 'CCTV_SOURCES_JSON', 'CCTV_FORCE_AUSTIN', 'GOOGLE_MAPS_SERVER_API_KEY', 'GOOGLE_MAPS_API_KEY']) { + const previous = process.env[name]; + delete process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + } + t.mock.method(globalThis, 'fetch', () => { throw Error('fixture must not fetch'); }); +} + +test('CCTV instances resolve their own application source root and isolate catalogs and health', async (t) => { + isolate(t); + const first = install(cctvProxy({ sourceRoot: fixture(t, 'first') })); + const second = install(cctvProxy({ sourceRoot: fixture(t, 'second') })); + const [a, b] = await Promise.all([first('/sources'), second('/sources')]); + assert.equal(a.status, 200); + assert.equal(b.status, 200); + assert.deepEqual(JSON.parse(a.body).sources.map(s => s.id), ['first']); + assert.deepEqual(JSON.parse(b.body).sources.map(s => s.id), ['second']); + const stream = JSON.parse((await first('/stream/first')).body); + assert.equal(stream.feedType, 'mp4'); + assert.equal(stream.mediaUrl, '/api/cctv/media/first'); + assert.equal((await first('/media/first')).status, 404); + const frame = await first('/frame/first'); + assert.equal(frame.headers['X-CCTV-Source'], 'synthetic'); + assert.match(frame.body, /<Camera & test>/); + assert.equal(JSON.parse((await first('/health')).body).cameras.length, 1); + assert.deepEqual(JSON.parse((await second('/health')).body).cameras, []); +}); + +test('composition creates exactly one CCTV and radio provider without acquisition', (t) => { + isolate(t); + const plugins = localProviderPlugins(); + for (const factory of [cctvProxy, radioBrowserProxy]) { + assert.equal(plugins.filter(p => p.name === factory().name).length, 1); + } +}); From c69416ff6bf6626356dbb73cdb9fbb4d1e632ae5 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Sat, 12 Sep 2026 12:09:04 -0700 Subject: [PATCH 2/2] style: adopt formatting for CCTV and radio provider modules --- scripts/format-scope.json | 14 +- server/providers/cctv.js | 150 ++++++++++++---- server/providers/cctv/catalog.js | 229 ++++++++++++++---------- server/providers/cctv/constants.js | 6 +- server/providers/cctv/media.js | 37 ++-- server/providers/cctv/normalize.js | 77 +++++--- server/providers/cctv/sources.js | 125 ++++++++++--- server/providers/radio.js | 6 +- server/providers/radio/catalog.js | 267 ++++++++++++++++++++-------- server/providers/radio/constants.js | 10 +- server/providers/radio/stations.js | 112 ++++++++---- server/providers/radio/transport.js | 138 ++++++++------ src/tooling/mediaProviders.test.mjs | 55 ++++-- 13 files changed, 862 insertions(+), 364 deletions(-) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index e822ab5a2..83b114819 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -69,5 +69,17 @@ "src/data/tomtomTiles.test.mjs", "src/data/firmsCsv.js", "src/data/firmsCsv.test.mjs", - "src/tooling/environmentProviders.test.mjs" + "src/tooling/environmentProviders.test.mjs", + "server/providers/cctv.js", + "server/providers/cctv/catalog.js", + "server/providers/cctv/constants.js", + "server/providers/cctv/media.js", + "server/providers/cctv/normalize.js", + "server/providers/cctv/sources.js", + "server/providers/radio.js", + "server/providers/radio/catalog.js", + "server/providers/radio/constants.js", + "server/providers/radio/stations.js", + "server/providers/radio/transport.js", + "src/tooling/mediaProviders.test.mjs" ] diff --git a/server/providers/cctv.js b/server/providers/cctv.js index e884a9d72..17235e97c 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -1,6 +1,14 @@ import { createCctvCatalog } from './cctv/catalog.js'; -import { normalizeFeedType, isVideoFeedType, toFiniteNumber } from './cctv/normalize.js'; -import { buildSyntheticCctvSvg, proxyMediaResponse, fetchCctvImageFromUpstream } from './cctv/media.js'; +import { + normalizeFeedType, + isVideoFeedType, + toFiniteNumber, +} from './cctv/normalize.js'; +import { + buildSyntheticCctvSvg, + proxyMediaResponse, + fetchCctvImageFromUpstream, +} from './cctv/media.js'; import { CCTV_FRAME_FETCH_TIMEOUT_MS } from './cctv/constants.js'; import { googleServerApiKey } from './places/google-key.js'; export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; @@ -58,7 +66,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { : null, frameUrl: `/api/cctv/frame/${encodeURIComponent(cameraId)}`, provider: source?.provider || '', - sourceKind: source?.sourceKind || (source?.url ? 'configured' : 'fallback'), + sourceKind: + source?.sourceKind || (source?.url ? 'configured' : 'fallback'), }; }; @@ -71,14 +80,24 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { */ const streetViewFallback = async ({ lat, lon, heading, fov, pitch }) => { const streetViewKey = googleServerApiKey(); - if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon)) return null; + if (!streetViewKey || !Number.isFinite(lat) || !Number.isFinite(lon)) + return null; try { const sv = new URL('https://maps.googleapis.com/maps/api/streetview'); sv.searchParams.set('size', '960x540'); sv.searchParams.set('location', `${lat},${lon}`); - sv.searchParams.set('heading', String(Number.isFinite(heading) ? heading : 0)); - sv.searchParams.set('fov', String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80)); - sv.searchParams.set('pitch', String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0)); + sv.searchParams.set( + 'heading', + String(Number.isFinite(heading) ? heading : 0), + ); + sv.searchParams.set( + 'fov', + String(Number.isFinite(fov) ? Math.max(20, Math.min(120, fov)) : 80), + ); + sv.searchParams.set( + 'pitch', + String(Number.isFinite(pitch) ? Math.max(-40, Math.min(20, pitch)) : 0), + ); sv.searchParams.set('source', 'outdoor'); sv.searchParams.set('return_error_code', 'true'); sv.searchParams.set('key', streetViewKey); @@ -106,7 +125,9 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { server.middlewares.use('/api/cctv', async (req, res) => { try { const sources = await getCctvSources(); - const sourceById = new Map(sources.map((source) => [source.id, source])); + const sourceById = new Map( + sources.map((source) => [source.id, source]), + ); const url = new URL(req.url || '/', 'http://localhost'); if (url.pathname === '/sources') { @@ -127,33 +148,47 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { mountHeightM: source.mountHeightM, groundElevationM: source.groundElevationM, feedType: normalizeFeedType(source.feedType), - sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), + sourceKind: + source.sourceKind || (source.url ? 'configured' : 'fallback'), poseSource: source.poseSource, license: source.license, })), }; - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify(body)); return; } if (url.pathname === '/health') { - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ cameras: listHealth() })); return; } if (url.pathname.startsWith('/stream/')) { - const cameraId = decodeURIComponent(url.pathname.replace('/stream/', '').trim()) || 'camera'; + const cameraId = + decodeURIComponent(url.pathname.replace('/stream/', '').trim()) || + 'camera'; const source = sourceById.get(cameraId); const payload = buildStreamPayload(source, cameraId); - res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify(payload)); return; } if (url.pathname.startsWith('/media/')) { - const cameraId = decodeURIComponent(url.pathname.replace('/media/', '').trim()) || 'camera'; + const cameraId = + decodeURIComponent(url.pathname.replace('/media/', '').trim()) || + 'camera'; const source = sourceById.get(cameraId); const mediaUrl = source?.url || ''; const feedType = normalizeFeedType(source?.feedType || 'image'); @@ -165,13 +200,22 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { label: source?.provider || 'No upstream URL', message: 'No stream URL configured', }); - res.writeHead(404, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'No media URL configured for this camera' })); + res.writeHead(404, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: 'No media URL configured for this camera', + }), + ); return; } try { - const upstreamHeaders = { 'User-Agent': 'gods-eye-view-cctv-proxy/1.0' }; + const upstreamHeaders = { + 'User-Agent': 'gods-eye-view-cctv-proxy/1.0', + }; const requestRange = req.headers?.range; if (requestRange) upstreamHeaders.Range = requestRange; const upstream = await fetch(mediaUrl, { @@ -185,12 +229,25 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { label: source?.provider || 'Configured source', message: `Upstream HTTP ${upstream.status}`, }); - res.writeHead(upstream.status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: `Upstream returned ${upstream.status}` })); + res.writeHead(upstream.status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: `Upstream returned ${upstream.status}`, + }), + ); return; } - if (isVideoFeedType(feedType) && !(contentType.startsWith('video/') || contentType.includes('mpegurl'))) { + if ( + isVideoFeedType(feedType) && + !( + contentType.startsWith('video/') || + contentType.includes('mpegurl') + ) + ) { setHealth(cameraId, { status: 'degraded', sourceKind: 'upstream', @@ -202,12 +259,16 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { status: 'ok', sourceKind: isVideoFeedType(feedType) ? 'live' : 'snapshot', label: source?.provider || 'Configured source', - message: isVideoFeedType(feedType) ? 'Live stream connected' : 'Snapshot feed connected', + message: isVideoFeedType(feedType) + ? 'Live stream connected' + : 'Snapshot feed connected', }); } await proxyMediaResponse(res, upstream, { - sourceHeader: isVideoFeedType(feedType) ? 'live-media' : 'upstream-image', + sourceHeader: isVideoFeedType(feedType) + ? 'live-media' + : 'upstream-image', }); return; } catch (error) { @@ -217,7 +278,10 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { label: source?.provider || 'Configured source', message: error?.message || 'Media fetch failed', }); - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(502, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Media proxy failed' })); return; } @@ -229,23 +293,33 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { return; } - const cameraId = decodeURIComponent(url.pathname.replace('/frame/', '').trim()) || 'camera'; + const cameraId = + decodeURIComponent(url.pathname.replace('/frame/', '').trim()) || + 'camera'; const source = sourceById.get(cameraId); - const label = url.searchParams.get('label') || source?.name || cameraId; + const label = + url.searchParams.get('label') || source?.name || cameraId; const city = url.searchParams.get('city') || source?.city || ''; const lat = Number(url.searchParams.get('lat') || source?.lat); const lon = Number(url.searchParams.get('lon') || source?.lon); - const heading = Number(url.searchParams.get('heading') || source?.headingDeg); + const heading = Number( + url.searchParams.get('heading') || source?.headingDeg, + ); const fov = Number(url.searchParams.get('fov') || source?.fovDeg); - const pitch = Number(url.searchParams.get('pitch') || source?.pitchDeg); + const pitch = Number( + url.searchParams.get('pitch') || source?.pitchDeg, + ); // Only use server-registered upstream URLs — never accept client-supplied URLs // (prevents SSRF via ?upstream= query parameter) const upstreamCandidate = - source?.snapshotUrl - || (!isVideoFeedType(normalizeFeedType(source?.feedType)) ? source?.url : ''); + source?.snapshotUrl || + (!isVideoFeedType(normalizeFeedType(source?.feedType)) + ? source?.url + : ''); - const upstreamImage = await fetchCctvImageFromUpstream(upstreamCandidate); + const upstreamImage = + await fetchCctvImageFromUpstream(upstreamCandidate); if (upstreamImage?.ok) { setHealth(cameraId, { status: 'ok', @@ -262,7 +336,13 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { return; } - const sv = await streetViewFallback({ lat, lon, heading, fov, pitch }); + const sv = await streetViewFallback({ + lat, + lon, + heading, + fov, + pitch, + }); if (sv?.ok) { setHealth(cameraId, { status: 'degraded', @@ -283,14 +363,18 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { cameraId, label, city, - status: source?.url ? 'UPSTREAM UNAVAILABLE' : 'NO UPSTREAM CONFIGURED', + status: source?.url + ? 'UPSTREAM UNAVAILABLE' + : 'NO UPSTREAM CONFIGURED', }); setHealth(cameraId, { status: 'degraded', sourceKind: 'synthetic', label: source?.provider || 'Synthetic fallback', - message: source?.url ? 'Upstream unavailable' : 'No source configured', + message: source?.url + ? 'Upstream unavailable' + : 'No source configured', }); res.writeHead(200, { diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index e27d06528..a883de6a1 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -1,8 +1,16 @@ import fs from 'node:fs'; import path from 'node:path'; -import { DEFAULT_CCTV_SOURCE_FILE, DEFAULT_CCTV_MAX_SOURCES, CCTV_SOURCE_CACHE_MS } from './constants.js'; +import { + DEFAULT_CCTV_SOURCE_FILE, + DEFAULT_CCTV_MAX_SOURCES, + CCTV_SOURCE_CACHE_MS, +} from './constants.js'; import { normalizeSourceItem } from './normalize.js'; -import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, loadTflSourcesFromOpenData } from './sources.js'; +import { + loadAustinSourcesFromOpenData, + loadCaltransSourcesFromOpenData, + loadTflSourcesFromOpenData, +} from './sources.js'; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). * @@ -19,7 +27,11 @@ function loadSourcesFromFile(sourceRoot) { const parsed = JSON.parse(raw); return Array.isArray(parsed) ? parsed : []; } catch (error) { - console.warn('[CCTV] failed to read source file:', resolved, error?.message || error); + console.warn( + '[CCTV] failed to read source file:', + resolved, + error?.message || error, + ); return []; } } @@ -42,99 +54,128 @@ function loadSourcesFromEnv() { /** Create an independent catalog rooted in the consuming application. */ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { -/** @type {Array} Cached merged + normalized CCTV source list. */ -let _cctvSourceCache = []; -/** @type {number} Epoch-ms when the source cache was last refreshed. */ -let _cctvSourceCacheAt = 0; -/** @type {Promise>|null} In-flight refresh, shared by concurrent - * callers so a post-TTL burst launches ONE refetch, not one per request. */ -let _cctvSourceInflight = null; + /** @type {Array} Cached merged + normalized CCTV source list. */ + let _cctvSourceCache = []; + /** @type {number} Epoch-ms when the source cache was last refreshed. */ + let _cctvSourceCacheAt = 0; + /** @type {Promise>|null} In-flight refresh, shared by concurrent + * callers so a post-TTL burst launches ONE refetch, not one per request. */ + let _cctvSourceInflight = null; -/** - * Assemble and cache the merged CCTV source list. - * - * Merges sources from three origins (Austin Open Data, local file, - * env variable), deduplicates by ID, applies the global max cap, and - * caches for CCTV_SOURCE_CACHE_MS. - * - * @returns {Promise>} Deduplicated, capped source list. - */ -async function getCctvSources() { - const now = Date.now(); - if (_cctvSourceCache.length && now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS) { + /** + * Assemble and cache the merged CCTV source list. + * + * Merges sources from three origins (Austin Open Data, local file, + * env variable), deduplicates by ID, applies the global max cap, and + * caches for CCTV_SOURCE_CACHE_MS. + * + * @returns {Promise>} Deduplicated, capped source list. + */ + async function getCctvSources() { + const now = Date.now(); + if ( + _cctvSourceCache.length && + now - _cctvSourceCacheAt <= CCTV_SOURCE_CACHE_MS + ) { + return _cctvSourceCache; + } + // Single-flight: a burst of requests arriving past the TTL shares ONE refresh + // instead of each launching the full multi-provider refetch. The `.finally` + // clears the ref so the next post-TTL cycle starts fresh. + if (_cctvSourceInflight) return _cctvSourceInflight; + _cctvSourceInflight = refreshCctvSources().finally(() => { + _cctvSourceInflight = null; + }); + return _cctvSourceInflight; + } + + /** + * Assemble and cache the merged CCTV source list from file/env + live packs. + * Always resolves (loaders self-catch to []); on a fully-empty refresh with a + * good prior catalog it serves stale rather than blanking the CCTV layer. + * + * @returns {Promise>} Deduplicated, capped source list. + */ + async function refreshCctvSources() { + const fromFile = loadSourcesFromFile(sourceRoot); + const fromEnv = loadSourcesFromEnv(); + + const forceAustin = + String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1'; + const preferAustin = + String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0'; + // Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack + // is configured and live packs aren't forced — same gate that governed the + // Austin-only fetch, now governing all three. Each pack fails independently. + const needsLiveSources = + forceAustin || (fromFile.length + fromEnv.length === 0 && preferAustin); + const tflEnabled = + String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0'; + + let fromAustin = []; + let fromCaltrans = []; + let fromTfl = []; + if (needsLiveSources) { + const [austinResult, caltransResult, tflResult] = + await Promise.allSettled([ + loadAustinSourcesFromOpenData(), + loadCaltransSourcesFromOpenData(), + tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]), + ]); + fromAustin = + austinResult.status === 'fulfilled' ? austinResult.value : []; + fromCaltrans = + caltransResult.status === 'fulfilled' ? caltransResult.value : []; + fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : []; + } + // Live sources first so file/env overrides win on duplicate IDs (Map last-write). + const merged = [ + ...fromAustin, + ...fromCaltrans, + ...fromTfl, + ...fromFile, + ...fromEnv, + ]; + + // Deduplicate by camera ID (last-write wins because of Map.set) + const byId = new Map(); + for (const item of merged) { + if (!item || typeof item !== 'object') continue; + const normalized = normalizeSourceItem(item); + if (!normalized.id) continue; + byId.set(normalized.id, normalized); + } + + const mergedSources = Array.from(byId.values()); + const maxRaw = Number( + process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) + : DEFAULT_CCTV_MAX_SOURCES; + if (mergedSources.length > maxCount) { + console.warn( + `[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`, + ); + } + const capped = + mergedSources.length > maxCount + ? mergedSources.slice(0, maxCount) + : mergedSources; + if (capped.length > 0 || _cctvSourceCache.length === 0) { + _cctvSourceCache = capped; + } else { + // Every source came back empty (all live packs timed out / upstream outage) + // but a good catalog is already cached — serve it stale rather than blanking + // every CCTV route. Advancing the timestamp waits one TTL before retrying, + // which (with single-flight) bounds load on a persistently-down upstream. + console.warn( + `[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`, + ); + } + _cctvSourceCacheAt = Date.now(); return _cctvSourceCache; } - // Single-flight: a burst of requests arriving past the TTL shares ONE refresh - // instead of each launching the full multi-provider refetch. The `.finally` - // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; }); - return _cctvSourceInflight; -} -/** - * Assemble and cache the merged CCTV source list from file/env + live packs. - * Always resolves (loaders self-catch to []); on a fully-empty refresh with a - * good prior catalog it serves stale rather than blanking the CCTV layer. - * - * @returns {Promise>} Deduplicated, capped source list. - */ -async function refreshCctvSources() { - const fromFile = loadSourcesFromFile(sourceRoot); - const fromEnv = loadSourcesFromEnv(); - - const forceAustin = String(process.env.CCTV_FORCE_AUSTIN || '').trim() === '1'; - const preferAustin = String(process.env.CCTV_PREFER_AUSTIN || '1').trim() !== '0'; - // Live open-data packs (Austin + Caltrans + TfL) load unless a file/env pack - // is configured and live packs aren't forced — same gate that governed the - // Austin-only fetch, now governing all three. Each pack fails independently. - const needsLiveSources = forceAustin || ((fromFile.length + fromEnv.length) === 0 && preferAustin); - const tflEnabled = String(process.env.CCTV_TFL_ENABLED || '1').trim() !== '0'; - - let fromAustin = []; - let fromCaltrans = []; - let fromTfl = []; - if (needsLiveSources) { - const [austinResult, caltransResult, tflResult] = await Promise.allSettled([ - loadAustinSourcesFromOpenData(), - loadCaltransSourcesFromOpenData(), - tflEnabled ? loadTflSourcesFromOpenData() : Promise.resolve([]), - ]); - fromAustin = austinResult.status === 'fulfilled' ? austinResult.value : []; - fromCaltrans = caltransResult.status === 'fulfilled' ? caltransResult.value : []; - fromTfl = tflResult.status === 'fulfilled' ? tflResult.value : []; - } - // Live sources first so file/env overrides win on duplicate IDs (Map last-write). - const merged = [...fromAustin, ...fromCaltrans, ...fromTfl, ...fromFile, ...fromEnv]; - - // Deduplicate by camera ID (last-write wins because of Map.set) - const byId = new Map(); - for (const item of merged) { - if (!item || typeof item !== 'object') continue; - const normalized = normalizeSourceItem(item); - if (!normalized.id) continue; - byId.set(normalized.id, normalized); - } - - const mergedSources = Array.from(byId.values()); - const maxRaw = Number(process.env.CCTV_MAX_SOURCES || DEFAULT_CCTV_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) : DEFAULT_CCTV_MAX_SOURCES; - if (mergedSources.length > maxCount) { - console.warn(`[CCTV] source catalog ${mergedSources.length} exceeds cap ${maxCount}; keeping the first ${maxCount} (raise CCTV_MAX_SOURCES or lower a per-pack cap to change which).`); - } - const capped = mergedSources.length > maxCount ? mergedSources.slice(0, maxCount) : mergedSources; - if (capped.length > 0 || _cctvSourceCache.length === 0) { - _cctvSourceCache = capped; - } else { - // Every source came back empty (all live packs timed out / upstream outage) - // but a good catalog is already cached — serve it stale rather than blanking - // every CCTV route. Advancing the timestamp waits one TTL before retrying, - // which (with single-flight) bounds load on a persistently-down upstream. - console.warn(`[CCTV] source refresh returned empty; serving ${_cctvSourceCache.length} stale cameras`); - } - _cctvSourceCacheAt = Date.now(); - return _cctvSourceCache; -} - -return getCctvSources; + return getCctvSources; } diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index b8d733348..2ab5f5715 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -1,6 +1,7 @@ export const DEFAULT_CCTV_SOURCE_FILE = 'config/cctv_sources.austin.json'; /** Austin Open Data portal endpoint for traffic camera records. */ -export const DEFAULT_AUSTIN_ROWS_URL = 'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD'; +export const DEFAULT_AUSTIN_ROWS_URL = + 'https://data.austintexas.gov/api/views/b4k4-adkb/rows.json?accessType=DOWNLOAD'; /** Default cap on Austin cameras after distance-based prioritization. */ export const DEFAULT_AUSTIN_MAX_SOURCES = 250; /** Global cap on total CCTV sources served by the proxy. */ @@ -22,7 +23,8 @@ export const CALTRANS_ANCHORS = [ ]; /** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */ export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam'; -export const TFL_IMAGE_ORIGIN = 'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'; +export const TFL_IMAGE_ORIGIN = + 'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/'; export const DEFAULT_TFL_MAX_SOURCES = 250; export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 }; /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL) infrequent. Frames are fetched per-request and are unaffected. */ diff --git a/server/providers/cctv/media.js b/server/providers/cctv/media.js index e0c298e1a..e7684dacc 100644 --- a/server/providers/cctv/media.js +++ b/server/providers/cctv/media.js @@ -97,8 +97,13 @@ export function toReadable(body) { * @param {object} [opts] * @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header. */ -export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstream' } = {}) { - const contentType = upstream.headers.get('content-type') || 'application/octet-stream'; +export async function proxyMediaResponse( + res, + upstream, + { sourceHeader = 'upstream' } = {}, +) { + const contentType = + upstream.headers.get('content-type') || 'application/octet-stream'; const cacheControl = upstream.headers.get('cache-control') || 'no-store'; const contentLength = upstream.headers.get('content-length'); const contentRange = upstream.headers.get('content-range'); @@ -116,10 +121,20 @@ export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstre // Live MJPEG/HLS streams are unbounded by design and send no content-length, // so they pipe normally (piping streams to the client, never buffering). const MEDIA_DECLARED_CAP_BYTES = 64 * 1024 * 1024; - if (Number.isFinite(Number(contentLength)) && Number(contentLength) > MEDIA_DECLARED_CAP_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + if ( + Number.isFinite(Number(contentLength)) && + Number(contentLength) > MEDIA_DECLARED_CAP_BYTES + ) { + res.writeHead(502, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Upstream media exceeds size cap' })); - try { await upstream.body?.cancel(); } catch { /* no-op */ } + try { + await upstream.body?.cancel(); + } catch { + /* no-op */ + } return; } @@ -151,14 +166,16 @@ export async function proxyMediaResponse(res, upstream, { sourceHeader = 'upstre * @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout. * @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>} */ -export async function fetchCctvImageFromUpstream(url, { - fetchImpl = fetch, - timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS, -} = {}) { +export async function fetchCctvImageFromUpstream( + url, + { fetchImpl = fetch, timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS } = {}, +) { if (!url || !/^https?:\/\//i.test(url)) return null; const controller = new AbortController(); const timeoutId = setTimeout(() => { - controller.abort(new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError')); + controller.abort( + new DOMException('CCTV upstream frame fetch timed out', 'TimeoutError'), + ); }, timeoutMs); try { const upstream = await fetchImpl(url, { diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index 24efbe25a..4d9fde95d 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -39,7 +39,9 @@ export function escapeXml(text) { * @returns {string} Normalized feed type. */ export function normalizeFeedType(value) { - const raw = String(value || '').trim().toLowerCase(); + const raw = String(value || '') + .trim() + .toLowerCase(); if (!raw) return 'image'; if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; if (raw === 'mjpg') return 'mjpeg'; @@ -93,7 +95,9 @@ export function normalizeKey(text) { * @returns {{lat:number, lon:number}} */ export function parsePointString(value) { - const match = String(value || '').match(/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i); + const match = String(value || '').match( + /POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i, + ); if (!match) return { lat: NaN, lon: NaN }; return { lon: toFiniteNumber(match[1]), @@ -123,11 +127,16 @@ export function coerceLatLon(value) { const lat = toFiniteNumber( value.latitude ?? value.lat ?? value.y ?? value.Latitude ?? value.Lat, - NaN + NaN, ); const lon = toFiniteNumber( - value.longitude ?? value.lon ?? value.lng ?? value.x ?? value.Longitude ?? value.Lon, - NaN + value.longitude ?? + value.lon ?? + value.lng ?? + value.x ?? + value.Longitude ?? + value.Lon, + NaN, ); return { lat, lon }; } @@ -152,16 +161,24 @@ export function extractAustinCoords(record) { ]; for (const candidate of candidates) { const parsed = coerceLatLon(candidate); - if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon)) return parsed; + if (Number.isFinite(parsed.lat) && Number.isFinite(parsed.lon)) + return parsed; } const lat = toFiniteNumber( - record.latitude ?? record.lat ?? record.camera_latitude ?? record.location_latitude, - NaN + record.latitude ?? + record.lat ?? + record.camera_latitude ?? + record.location_latitude, + NaN, ); const lon = toFiniteNumber( - record.longitude ?? record.lon ?? record.lng ?? record.camera_longitude ?? record.location_longitude, - NaN + record.longitude ?? + record.lon ?? + record.lng ?? + record.camera_longitude ?? + record.location_longitude, + NaN, ); return { lat, lon }; } @@ -239,11 +256,19 @@ export function extractAustinName(record, cameraId) { * @returns {number} Heading in degrees [0..360), or NaN if unknown. */ export function extractAustinHeading(record) { - const direct = toFiniteNumber(record.heading_deg ?? record.heading ?? record.bearing, NaN); + const direct = toFiniteNumber( + record.heading_deg ?? record.heading ?? record.bearing, + NaN, + ); if (Number.isFinite(direct)) return ((direct % 360) + 360) % 360; // Dedicated direction fields: bare cardinal words ("West") are real facings. - const directionKeys = ['direction', 'travel_direction', 'facing', 'facing_direction']; + const directionKeys = [ + 'direction', + 'travel_direction', + 'facing', + 'facing_direction', + ]; for (const key of directionKeys) { const heading = directionToHeading(record[key], true); if (Number.isFinite(heading)) return heading; @@ -260,7 +285,9 @@ export function extractAustinHeading(record) { record.cross_street, record.description, record.name, - ].filter(Boolean).join(' '); + ] + .filter(Boolean) + .join(' '); const inferred = directionToHeading(nameProbe); if (Number.isFinite(inferred)) return inferred; @@ -276,7 +303,7 @@ export function extractAustinHeading(record) { */ export function isLikelyAustinCoordinate(lat, lon) { if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; - return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.40; + return lat >= 30.02 && lat <= 30.58 && lon >= -98.12 && lon <= -97.4; } /** @@ -324,18 +351,26 @@ export function rowArrayToObject(row, columns) { export function prioritizeSources(cameras, maxCount, anchors) { const list = Array.isArray(cameras) ? cameras : []; const anchorList = (Array.isArray(anchors) ? anchors : []).filter( - (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon) + (a) => Number.isFinite(a?.lat) && Number.isFinite(a?.lon), ); - if (!Number.isFinite(maxCount) || maxCount <= 0 || list.length <= maxCount || !anchorList.length) { + if ( + !Number.isFinite(maxCount) || + maxCount <= 0 || + list.length <= maxCount || + !anchorList.length + ) { return list; } const scored = list.map((camera, idx) => { const lat = Number(camera?.lat); const lon = Number(camera?.lon); - const distKm = Number.isFinite(lat) && Number.isFinite(lon) - ? Math.min(...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon))) - : Number.POSITIVE_INFINITY; + const distKm = + Number.isFinite(lat) && Number.isFinite(lon) + ? Math.min( + ...anchorList.map((a) => haversineKm(lat, lon, a.lat, a.lon)), + ) + : Number.POSITIVE_INFINITY; return { camera, idx, distKm }; }); @@ -363,7 +398,9 @@ export function normalizeSourceItem(item) { lat: toFiniteNumber(item.lat), lon: toFiniteNumber(item.lon), headingDeg: toFiniteNumber(item.headingDeg), - headingConfidence: String(item.headingConfidence || item.headingSource || '').toLowerCase(), + headingConfidence: String( + item.headingConfidence || item.headingSource || '', + ).toLowerCase(), pitchDeg: toFiniteNumber(item.pitchDeg), fovDeg: toFiniteNumber(item.fovDeg), rangeM: toFiniteNumber(item.rangeM), diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 81da1574a..111029564 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1,5 +1,28 @@ -import { DEFAULT_AUSTIN_ROWS_URL, DEFAULT_AUSTIN_MAX_SOURCES, AUSTIN_DOWNTOWN, CALTRANS_CCTV_URL, DEFAULT_CALTRANS_DISTRICTS, DEFAULT_CALTRANS_MAX_SOURCES, CALTRANS_ANCHORS, TFL_JAMCAM_URL, TFL_IMAGE_ORIGIN, DEFAULT_TFL_MAX_SOURCES, LONDON_CENTER, CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; -import { toFiniteNumber, extractAustinCoords, extractAustinCameraId, extractAustinName, extractAustinHeading, isLikelyAustinCoordinate, fallbackHeadingFromId, rowArrayToObject, prioritizeSources } from './normalize.js'; +import { + DEFAULT_AUSTIN_ROWS_URL, + DEFAULT_AUSTIN_MAX_SOURCES, + AUSTIN_DOWNTOWN, + CALTRANS_CCTV_URL, + DEFAULT_CALTRANS_DISTRICTS, + DEFAULT_CALTRANS_MAX_SOURCES, + CALTRANS_ANCHORS, + TFL_JAMCAM_URL, + TFL_IMAGE_ORIGIN, + DEFAULT_TFL_MAX_SOURCES, + LONDON_CENTER, + CCTV_SOURCE_FETCH_TIMEOUT_MS, +} from './constants.js'; +import { + toFiniteNumber, + extractAustinCoords, + extractAustinCameraId, + extractAustinName, + extractAustinHeading, + isLikelyAustinCoordinate, + fallbackHeadingFromId, + rowArrayToObject, + prioritizeSources, +} from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. @@ -14,13 +37,18 @@ import { directionToHeading } from '../../../src/data/directionText.js'; export async function loadAustinSourcesFromOpenData() { const endpoint = process.env.CCTV_AUSTIN_ROWS_URL || DEFAULT_AUSTIN_ROWS_URL; try { - const resp = await fetch(endpoint, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + const resp = await fetch(endpoint, { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); if (!resp.ok) { console.warn('[CCTV] Austin source download failed:', resp.status); return []; } const payload = await resp.json(); - const columns = Array.isArray(payload?.meta?.view?.columns) ? payload.meta.view.columns : []; + const columns = Array.isArray(payload?.meta?.view?.columns) + ? payload.meta.view.columns + : []; const rows = Array.isArray(payload?.data) ? payload.data : []; if (!columns.length || !rows.length) return []; @@ -35,7 +63,9 @@ export async function loadAustinSourcesFromOpenData() { // REMOVED and VOID rows whose frame URLs never resolve — those cameras // would render as permanent Street View / synthetic fallbacks. Tolerate // a missing column (keep the row) so a schema change fails open. - const status = String(record.camera_status || '').trim().toUpperCase(); + const status = String(record.camera_status || '') + .trim() + .toUpperCase(); if (status && status !== 'TURNED_ON') continue; const { lat, lon } = extractAustinCoords(record); @@ -44,7 +74,9 @@ export async function loadAustinSourcesFromOpenData() { const extractedHeading = extractAustinHeading(record); const hasHeading = Number.isFinite(extractedHeading); - const headingDeg = hasHeading ? extractedHeading : fallbackHeadingFromId(cameraId); + const headingDeg = hasHeading + ? extractedHeading + : fallbackHeadingFromId(cameraId); cameras.push({ id: cameraId, name: extractAustinName(record, cameraId), @@ -68,18 +100,29 @@ export async function loadAustinSourcesFromOpenData() { }); } - const unique = Array.from(new Map(cameras.map((camera) => [camera.id, camera])).values()); - const maxRaw = Number(process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(300, Math.floor(maxRaw))) : DEFAULT_AUSTIN_MAX_SOURCES; + const unique = Array.from( + new Map(cameras.map((camera) => [camera.id, camera])).values(), + ); + const maxRaw = Number( + process.env.CCTV_AUSTIN_MAX_SOURCES || DEFAULT_AUSTIN_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(300, Math.floor(maxRaw))) + : DEFAULT_AUSTIN_MAX_SOURCES; const prioritized = prioritizeSources(unique, maxCount, [AUSTIN_DOWNTOWN]); if (prioritized.length < unique.length) { - console.log(`[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`); + console.log( + `[CCTV] Loaded Austin camera sources: ${unique.length} (using nearest ${prioritized.length})`, + ); } else { console.log('[CCTV] Loaded Austin camera sources:', prioritized.length); } return prioritized; } catch (error) { - console.warn('[CCTV] Austin source download error:', error?.message || error); + console.warn( + '[CCTV] Austin source download error:', + error?.message || error, + ); return []; } } @@ -96,7 +139,8 @@ export async function loadAustinSourcesFromOpenData() { * @returns {Promise>} Normalized camera source objects. */ export async function loadCaltransSourcesFromOpenData() { - const districtsRaw = process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS; + const districtsRaw = + process.env.CCTV_CALTRANS_DISTRICTS ?? DEFAULT_CALTRANS_DISTRICTS; const districts = String(districtsRaw) .split(',') .map((token) => Number(token.trim())) @@ -105,18 +149,24 @@ export async function loadCaltransSourcesFromOpenData() { const settled = await Promise.allSettled( districts.map(async (district) => { - const resp = await fetch(CALTRANS_CCTV_URL(district), { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + const resp = await fetch(CALTRANS_CCTV_URL(district), { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); if (!resp.ok) throw new Error(`D${district} HTTP ${resp.status}`); const payload = await resp.json(); const rows = Array.isArray(payload?.data) ? payload.data : []; return { district, rows }; - }) + }), ); const cameras = []; for (const result of settled) { if (result.status !== 'fulfilled') { - console.warn('[CCTV] Caltrans district fetch failed:', result.reason?.message || result.reason); + console.warn( + '[CCTV] Caltrans district fetch failed:', + result.reason?.message || result.reason, + ); continue; } const { district, rows } = result.value; @@ -135,13 +185,17 @@ export async function loadCaltransSourcesFromOpenData() { const locationName = String(loc.locationName || '').trim(); // Leading token of locationName is the stable camera code ("TV102 -- I-580 : …"). const codeMatch = /^([A-Za-z0-9_-]+)\s*--/.exec(locationName); - const code = (codeMatch ? codeMatch[1] : `x${cameras.length}`).toLowerCase(); + const code = ( + codeMatch ? codeMatch[1] : `x${cameras.length}` + ).toLowerCase(); const cameraId = `ca-d${district}-${code}`; // loc.direction is a dedicated field ("West", "South") → allow bare words. const heading = directionToHeading(loc.direction, true); const hasHeading = Number.isFinite(heading); - const label = locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') || `Caltrans D${district} ${code}`; + const label = + locationName.replace(/^([A-Za-z0-9_-]+)\s*--\s*/, '') || + `Caltrans D${district} ${code}`; cameras.push({ id: cameraId, name: loc.nearbyPlace ? `${label} (${loc.nearbyPlace})` : label, @@ -168,7 +222,9 @@ export async function loadCaltransSourcesFromOpenData() { // so it must be right-ish on its own. groundElevationM: (() => { const ft = toFiniteNumber(loc.elevation, NaN); - return Number.isFinite(ft) ? Math.max(-100, Math.min(4000, ft * 0.3048)) : 150; + return Number.isFinite(ft) + ? Math.max(-100, Math.min(4000, ft * 0.3048)) + : 150; })(), feedType: 'image', url: imageUrl, @@ -179,10 +235,16 @@ export async function loadCaltransSourcesFromOpenData() { } } - const maxRaw = Number(process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_CALTRANS_MAX_SOURCES; + const maxRaw = Number( + process.env.CCTV_CALTRANS_MAX_SOURCES || DEFAULT_CALTRANS_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(600, Math.floor(maxRaw))) + : DEFAULT_CALTRANS_MAX_SOURCES; const prioritized = prioritizeSources(cameras, maxCount, CALTRANS_ANCHORS); - console.log(`[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`); + console.log( + `[CCTV] Loaded Caltrans camera sources: ${cameras.length} inService (using nearest ${prioritized.length})`, + ); return prioritized; } @@ -199,8 +261,13 @@ export async function loadCaltransSourcesFromOpenData() { export async function loadTflSourcesFromOpenData() { try { const appKey = String(process.env.TFL_APP_KEY || '').trim(); - const url = appKey ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}` : TFL_JAMCAM_URL; - const resp = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS) }); + const url = appKey + ? `${TFL_JAMCAM_URL}?app_key=${encodeURIComponent(appKey)}` + : TFL_JAMCAM_URL; + const resp = await fetch(url, { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); if (!resp.ok) { console.warn('[CCTV] TfL JamCam download failed:', resp.status); return []; @@ -251,10 +318,16 @@ export async function loadTflSourcesFromOpenData() { }); } - const maxRaw = Number(process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES); - const maxCount = Number.isFinite(maxRaw) ? Math.max(8, Math.min(600, Math.floor(maxRaw))) : DEFAULT_TFL_MAX_SOURCES; + const maxRaw = Number( + process.env.CCTV_TFL_MAX_SOURCES || DEFAULT_TFL_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(600, Math.floor(maxRaw))) + : DEFAULT_TFL_MAX_SOURCES; const prioritized = prioritizeSources(cameras, maxCount, [LONDON_CENTER]); - console.log(`[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`); + console.log( + `[CCTV] Loaded TfL JamCam sources: ${cameras.length} available (using nearest ${prioritized.length})`, + ); return prioritized; } catch (error) { console.warn('[CCTV] TfL JamCam download error:', error?.message || error); diff --git a/server/providers/radio.js b/server/providers/radio.js index c36010745..5dd4b196f 100644 --- a/server/providers/radio.js +++ b/server/providers/radio.js @@ -1,7 +1,11 @@ import { createRadioProxyMiddleware } from './radio/catalog.js'; export { createRadioProxyMiddleware }; export { isPublicRadioAddress } from './radio/transport.js'; -export { normalizeRadioBrowserStation, publicRadioStation, publicRadioHttpsUrl } from './radio/stations.js'; +export { + normalizeRadioBrowserStation, + publicRadioStation, + publicRadioHttpsUrl, +} from './radio/stations.js'; export function radioBrowserProxy() { const middleware = createRadioProxyMiddleware(); const install = (server) => { diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js index 3abf59af2..36a3eb730 100644 --- a/server/providers/radio/catalog.js +++ b/server/providers/radio/catalog.js @@ -2,25 +2,53 @@ import { lookup as lookupDns } from 'node:dns/promises'; import { randomUUID } from 'node:crypto'; import { readResponseTextCapped } from '../common/http.js'; -import { normalizeRadioBrowserStation, publicRadioStation, cleanRadioText } from './stations.js'; -import { radioMirrorOrigin, radioProxyDestination, resolveRadioProxyAddresses, fetchPinnedRadioResponse } from './transport.js'; -import { RADIO_DIRECTORY_CACHE_MS, RADIO_DIRECTORY_STALE_MS, RADIO_MIRROR_CACHE_MS, RADIO_FETCH_TIMEOUT_MS, RADIO_RESPONSE_MAX_BYTES, RADIO_DIRECTORY_LIMIT, RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES, RADIO_CATALOG_HEALTHY_MIN_STATIONS, RADIO_USER_AGENT, RADIO_UUID_RE, RADIO_FALLBACK_MIRRORS } from './constants.js'; +import { + normalizeRadioBrowserStation, + publicRadioStation, + cleanRadioText, +} from './stations.js'; +import { + radioMirrorOrigin, + radioProxyDestination, + resolveRadioProxyAddresses, + fetchPinnedRadioResponse, +} from './transport.js'; +import { + RADIO_DIRECTORY_CACHE_MS, + RADIO_DIRECTORY_STALE_MS, + RADIO_MIRROR_CACHE_MS, + RADIO_FETCH_TIMEOUT_MS, + RADIO_RESPONSE_MAX_BYTES, + RADIO_DIRECTORY_LIMIT, + RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES, + RADIO_CATALOG_HEALTHY_MIN_STATIONS, + RADIO_USER_AGENT, + RADIO_UUID_RE, + RADIO_FALLBACK_MIRRORS, +} from './constants.js'; export async function mapRadioConcurrent(values, concurrency, mapper) { const results = new Array(values.length); let cursor = 0; - const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => { - for (;;) { - const index = cursor++; - if (index >= values.length) return; - results[index] = await mapper(values[index], index); - } - }); + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + for (;;) { + const index = cursor++; + if (index >= values.length) return; + results[index] = await mapper(values[index], index); + } + }, + ); await Promise.all(workers); return results; } /** Create the testable Connect middleware backing `/api/radio`. */ -export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = lookupDns, now = Date.now } = {}) { +export function createRadioProxyMiddleware({ + fetchImpl = null, + lookupImpl = lookupDns, + now = Date.now, +} = {}) { let mirrorCache = { origins: [...RADIO_FALLBACK_MIRRORS], cachedAt: 0 }; let mirrorPromise = null; let catalogCache = null; @@ -35,11 +63,15 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look async function fetchJson(url, maxBytes = RADIO_RESPONSE_MAX_BYTES) { const destination = radioProxyDestination(url); - if (!destination) throw new Error('Radio Browser destination is not permitted'); + if (!destination) + throw new Error('Radio Browser destination is not permitted'); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), RADIO_FETCH_TIMEOUT_MS); try { - const addresses = await resolveRadioProxyAddresses(destination.hostname, lookupImpl); + const addresses = await resolveRadioProxyAddresses( + destination.hostname, + lookupImpl, + ); const options = { headers: { Accept: 'application/json', 'User-Agent': RADIO_USER_AGENT }, signal: controller.signal, @@ -49,10 +81,15 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look ? await fetchImpl(destination.href, options) : await fetchPinnedRadioResponse(destination, options, addresses); if (response.status >= 300 && response.status < 400) { - try { await response.body?.cancel?.(); } catch { /* no-op */ } + try { + await response.body?.cancel?.(); + } catch { + /* no-op */ + } throw new Error('Radio Browser redirects are refused'); } - if (!response.ok) throw new Error(`Radio Browser returned ${response.status}`); + if (!response.ok) + throw new Error(`Radio Browser returned ${response.status}`); const text = await readResponseTextCapped(response, maxBytes); return JSON.parse(text); } finally { @@ -61,20 +98,40 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look } async function mirrors() { - if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS) return mirrorCache.origins; + if (now() - mirrorCache.cachedAt < RADIO_MIRROR_CACHE_MS) + return mirrorCache.origins; if (!mirrorPromise) { mirrorPromise = (async () => { try { - const rows = await fetchJson('https://all.api.radio-browser.info/json/servers', 256 * 1024); - const discovered = [...new Set((Array.isArray(rows) ? rows : []).map((row) => radioMirrorOrigin(row?.name)).filter(Boolean))]; + const rows = await fetchJson( + 'https://all.api.radio-browser.info/json/servers', + 256 * 1024, + ); + const discovered = [ + ...new Set( + (Array.isArray(rows) ? rows : []) + .map((row) => radioMirrorOrigin(row?.name)) + .filter(Boolean), + ), + ]; if (discovered.length) { - mirrorCache = { origins: [...discovered, ...RADIO_FALLBACK_MIRRORS.filter((origin) => !discovered.includes(origin))], cachedAt: now() }; + mirrorCache = { + origins: [ + ...discovered, + ...RADIO_FALLBACK_MIRRORS.filter( + (origin) => !discovered.includes(origin), + ), + ], + cachedAt: now(), + }; } } catch { mirrorCache = { ...mirrorCache, cachedAt: now() }; } return mirrorCache.origins; - })().finally(() => { mirrorPromise = null; }); + })().finally(() => { + mirrorPromise = null; + }); } return mirrorPromise; } @@ -92,68 +149,113 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look } async function refreshCatalog() { - const queries = [null, 'news', 'talk', 'weather', 'emergency', 'scanner', 'aviation', 'marine', 'traffic']; - const outcomes = await mapRadioConcurrent(queries, 3, async (tag, index) => { - const params = new URLSearchParams({ - has_geo_info: 'true', - is_https: 'true', - hidebroken: 'true', - order: 'clickcount', - reverse: 'true', - limit: index === 0 ? '1800' : '220', - }); - if (tag) params.set('tag', tag); - try { - const rows = await fetchPath(`/json/stations/search?${params}`); - if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - if (!rows.every((row) => ( - row - && typeof row === 'object' - && !Array.isArray(row) - && typeof row.stationuuid === 'string' - && typeof row.name === 'string' - && (typeof row.url_resolved === 'string' || typeof row.url === 'string') - ))) throw new Error('Radio Browser catalog contained a malformed station row'); - const stations = rows.map(normalizeRadioBrowserStation).filter(Boolean); - const requestedTag = cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(); - const requestedTagCovered = !requestedTag || stations.some((station) => ( - station.tags.some((stationTag) => stationTag === requestedTag || stationTag.includes(requestedTag)) - )); - return { - // Query coverage is based on accepted rows, not merely a payload that - // happens to match the upstream schema. Specialist responses must - // also contain an accepted station tagged for the requested category. - succeeded: stations.length > 0 && requestedTagCovered, - stations, - }; - } catch { - return { succeeded: false, stations: [] }; - } - }); + const queries = [ + null, + 'news', + 'talk', + 'weather', + 'emergency', + 'scanner', + 'aviation', + 'marine', + 'traffic', + ]; + const outcomes = await mapRadioConcurrent( + queries, + 3, + async (tag, index) => { + const params = new URLSearchParams({ + has_geo_info: 'true', + is_https: 'true', + hidebroken: 'true', + order: 'clickcount', + reverse: 'true', + limit: index === 0 ? '1800' : '220', + }); + if (tag) params.set('tag', tag); + try { + const rows = await fetchPath(`/json/stations/search?${params}`); + if (!Array.isArray(rows)) + throw new Error('Radio Browser catalog payload was not an array'); + if ( + !rows.every( + (row) => + row && + typeof row === 'object' && + !Array.isArray(row) && + typeof row.stationuuid === 'string' && + typeof row.name === 'string' && + (typeof row.url_resolved === 'string' || + typeof row.url === 'string'), + ) + ) + throw new Error( + 'Radio Browser catalog contained a malformed station row', + ); + const stations = rows + .map(normalizeRadioBrowserStation) + .filter(Boolean); + const requestedTag = cleanRadioText(tag, 80) + .toLocaleLowerCase() + .replace(/[_-]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const requestedTagCovered = + !requestedTag || + stations.some((station) => + station.tags.some( + (stationTag) => + stationTag === requestedTag || + stationTag.includes(requestedTag), + ), + ); + return { + // Query coverage is based on accepted rows, not merely a payload that + // happens to match the upstream schema. Specialist responses must + // also contain an accepted station tagged for the requested category. + succeeded: stations.length > 0 && requestedTagCovered, + stations, + }; + } catch { + return { succeeded: false, stations: [] }; + } + }, + ); const resultSets = outcomes.map((outcome) => outcome.stations); const selected = []; const seen = new Set(); const take = (station) => { - if (!station || seen.has(station.id) || selected.length >= RADIO_DIRECTORY_LIMIT) return; + if ( + !station || + seen.has(station.id) || + selected.length >= RADIO_DIRECTORY_LIMIT + ) + return; seen.add(station.id); selected.push(station); }; // Seed specialist station-tag queries before popularity fill so operational // categories remain represented even when global click charts skew musical. for (const rows of resultSets.slice(1)) rows.slice(0, 45).forEach(take); - resultSets.flat().sort((a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name)).forEach(take); + resultSets + .flat() + .sort( + (a, b) => b.clickCount - a.clickCount || a.name.localeCompare(b.name), + ) + .forEach(take); const timestamp = now(); - const successfulQueries = outcomes.filter((outcome) => outcome.succeeded).length; - const broadQueryHealthy = outcomes[0].succeeded && outcomes[0].stations.length > 0; + const successfulQueries = outcomes.filter( + (outcome) => outcome.succeeded, + ).length; + const broadQueryHealthy = + outcomes[0].succeeded && outcomes[0].stations.length > 0; const healthReasons = []; if (!broadQueryHealthy) healthReasons.push('broad-query-unhealthy'); - if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES) healthReasons.push('query-coverage-below-policy'); - if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); + if (successfulQueries < RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES) + healthReasons.push('query-coverage-below-policy'); + if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) + healthReasons.push('station-coverage-below-policy'); const degraded = healthReasons.length > 0; const coverage = { successfulQueries, @@ -171,14 +273,18 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look coverage, }; if (degraded && catalogCache) { - const error = new Error('Radio Browser catalog refresh did not meet health policy'); + const error = new Error( + 'Radio Browser catalog refresh did not meet health policy', + ); error.radioCatalogDegraded = true; error.radioDegradedReason = nextCatalog.degradedReason; error.radioCoverage = coverage; throw error; } if (degraded && !selected.length) { - const error = new Error('Radio Browser catalog refresh returned no usable stations'); + const error = new Error( + 'Radio Browser catalog refresh returned no usable stations', + ); error.radioCatalogDegraded = true; error.radioDegradedReason = nextCatalog.degradedReason; error.radioCoverage = coverage; @@ -197,16 +303,24 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look } async function getCatalog() { - if (catalogCache && now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS) { + if ( + catalogCache && + now() - catalogCache.cachedAt < RADIO_DIRECTORY_CACHE_MS + ) { return { ...catalogCache, stale: false }; } if (!refreshPromise) { - refreshPromise = refreshCatalog().finally(() => { refreshPromise = null; }); + refreshPromise = refreshCatalog().finally(() => { + refreshPromise = null; + }); } try { - return { ...await refreshPromise, stale: false }; + return { ...(await refreshPromise), stale: false }; } catch (error) { - if (catalogCache && now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS) { + if ( + catalogCache && + now() - catalogCache.cachedAt <= RADIO_DIRECTORY_STALE_MS + ) { return { ...catalogCache, stale: true, @@ -220,7 +334,10 @@ export function createRadioProxyMiddleware({ fetchImpl = null, lookupImpl = look } function sendJson(res, status, body) { - res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' }); + res.writeHead(status, { + 'Content-Type': 'application/json; charset=utf-8', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify(body)); } diff --git a/server/providers/radio/constants.js b/server/providers/radio/constants.js index 2d935d83d..996e2af06 100644 --- a/server/providers/radio/constants.js +++ b/server/providers/radio/constants.js @@ -5,9 +5,13 @@ export const RADIO_FETCH_TIMEOUT_MS = 12_000; export const RADIO_RESPONSE_MAX_BYTES = 4 * 1024 * 1024; export const RADIO_DIRECTORY_LIMIT = 750; export const RADIO_CATALOG_MIN_SUCCESSFUL_QUERIES = 5; -export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(RADIO_DIRECTORY_LIMIT / 2); -export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; -export const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil( + RADIO_DIRECTORY_LIMIT / 2, +); +export const RADIO_USER_AGENT = + 'GodsEyeView/1.0 (Radio Browser directory client)'; +export const RADIO_UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; export const RADIO_FALLBACK_MIRRORS = Object.freeze([ 'https://de1.api.radio-browser.info', 'https://de2.api.radio-browser.info', diff --git a/server/providers/radio/stations.js b/server/providers/radio/stations.js index fdf69e62c..4d9bcf35c 100644 --- a/server/providers/radio/stations.js +++ b/server/providers/radio/stations.js @@ -1,40 +1,56 @@ import { RADIO_UUID_RE } from './constants.js'; import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js'; export function cleanRadioText(value, maxLength) { - return String(value ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, maxLength).trim(); + return String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, maxLength) + .trim(); } export function isNonGlobalIpv4(hostname) { const pieces = hostname.split('.'); - if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) return false; + if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) + return false; const values = pieces.map(Number); if (values.some((value) => value > 255)) return true; const [a, b, c] = values; - return a === 0 || a === 10 || a === 127 || a >= 224 - || (a === 100 && b >= 64 && b <= 127) - || (a === 169 && b === 254) - || (a === 172 && b >= 16 && b <= 31) - || (a === 192 && b === 0) - || (a === 192 && b === 88 && c === 99) - || (a === 192 && b === 168) - || (a === 198 && (b === 18 || b === 19)) - || (a === 198 && b === 51 && c === 100) - || (a === 203 && b === 0 && c === 113); + return ( + a === 0 || + a === 10 || + a === 127 || + a >= 224 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 0) || + (a === 192 && b === 88 && c === 99) || + (a === 192 && b === 168) || + (a === 198 && (b === 18 || b === 19)) || + (a === 198 && b === 51 && c === 100) || + (a === 203 && b === 0 && c === 113) + ); } /** Return a normalized public HTTPS URL, or null for local/private targets. */ export function publicRadioHttpsUrl(value) { try { const url = new URL(String(value ?? '')); - const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, ''); - if (url.protocol !== 'https:' || url.username || url.password || !hostname) return null; + const hostname = url.hostname + .toLowerCase() + .replace(/^\[|\]$/g, '') + .replace(/\.$/, ''); + if (url.protocol !== 'https:' || url.username || url.password || !hostname) + return null; if ( - hostname === 'localhost' - || hostname.endsWith('.localhost') - || hostname.endsWith('.local') - || isNonGlobalIpv4(hostname) - || hostname.includes(':') - ) return null; + hostname === 'localhost' || + hostname.endsWith('.localhost') || + hostname.endsWith('.local') || + isNonGlobalIpv4(hostname) || + hostname.includes(':') + ) + return null; url.hash = ''; return url.href; } catch { @@ -45,25 +61,40 @@ export function publicRadioHttpsUrl(value) { /** Normalize one Radio Browser station and omit favicons and unsafe streams. */ export function normalizeRadioBrowserStation(raw) { const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); - const lat = raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); - const lon = raw?.geo_long === null || raw?.geo_long === '' ? null : Number(raw?.geo_long); + const lat = + raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); + const lon = + raw?.geo_long === null || raw?.geo_long === '' + ? null + : Number(raw?.geo_long); const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url); if ( - !RADIO_UUID_RE.test(id) - || Number(raw?.lastcheckok) !== 1 - || Number(raw?.hls) === 1 - || !Number.isFinite(lat) || lat < -90 || lat > 90 - || !Number.isFinite(lon) || lon < -180 || lon > 180 - || !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) - || !streamUrl - ) return null; + !RADIO_UUID_RE.test(id) || + Number(raw?.lastcheckok) !== 1 || + Number(raw?.hls) === 1 || + !Number.isFinite(lat) || + lat < -90 || + lat > 90 || + !Number.isFinite(lon) || + lon < -180 || + lon > 180 || + !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) || + !streamUrl + ) + return null; const name = cleanRadioText(raw?.name, 140); if (!name) return null; const tags = String(raw?.tags ?? '') .split(',') - .map((tag) => cleanRadioText(tag, 80).toLocaleLowerCase().replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim()) + .map((tag) => + cleanRadioText(tag, 80) + .toLocaleLowerCase() + .replace(/[_-]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(), + ) .filter(Boolean) .filter((tag, index, all) => all.indexOf(tag) === index) .slice(0, 24); @@ -74,9 +105,10 @@ export function normalizeRadioBrowserStation(raw) { .slice(0, 8); const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); const normalizedCode = normalizeRadioCountryInput(rawCountryCode); - const normalizedCountry = normalizedCode.valid && !normalizedCode.empty - ? normalizedCode - : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); + const normalizedCountry = + normalizedCode.valid && !normalizedCode.empty + ? normalizedCode + : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); const bitrate = Number(raw?.bitrate); return { id, @@ -88,13 +120,17 @@ export function normalizeRadioBrowserStation(raw) { tags, languages, state: cleanRadioText(raw?.state, 80), - country: normalizedCountry.valid && !normalizedCountry.empty - ? normalizedCountry.name - : cleanRadioText(raw?.country, 80), + country: + normalizedCountry.valid && !normalizedCountry.empty + ? normalizedCountry.name + : cleanRadioText(raw?.country, 80), countryCode: normalizedCountry.valid ? normalizedCountry.code : '', metadataTrust: 'untrusted-community', codec, - bitrate: Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 ? bitrate : null, + bitrate: + Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 + ? bitrate + : null, clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), }; } diff --git a/server/providers/radio/transport.js b/server/providers/radio/transport.js index d9b220896..7bbe33bcd 100644 --- a/server/providers/radio/transport.js +++ b/server/providers/radio/transport.js @@ -2,44 +2,67 @@ import https from 'node:https'; import { Readable } from 'node:stream'; import { isNonGlobalIpv4 } from './stations.js'; export function radioMirrorOrigin(value) { - const hostname = String(value ?? '').toLowerCase().replace(/\.$/, ''); + const hostname = String(value ?? '') + .toLowerCase() + .replace(/\.$/, ''); if (!/^[a-z0-9-]+\.api\.radio-browser\.info$/.test(hostname)) return null; return `https://${hostname}`; } /** Return whether a resolved Radio Browser address is safe for an outbound request. */ export function isPublicRadioAddress(value) { - const address = String(value ?? '').trim().toLowerCase().replace(/^\[|\]$/g, ''); + const address = String(value ?? '') + .trim() + .toLowerCase() + .replace(/^\[|\]$/g, ''); if (!address) return false; if (!address.includes(':')) { const ipv4 = address.split('.'); - return ipv4.length === 4 - && ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) - && !isNonGlobalIpv4(address); + return ( + ipv4.length === 4 && + ipv4.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255) && + !isNonGlobalIpv4(address) + ); } const pieces = address.split('::'); if (pieces.length > 2) return false; const left = pieces[0] ? pieces[0].split(':') : []; const right = pieces[1] ? pieces[1].split(':') : []; const missing = 8 - left.length - right.length; - if ((pieces.length === 1 && missing !== 0) || (pieces.length === 2 && missing < 1)) return false; + if ( + (pieces.length === 1 && missing !== 0) || + (pieces.length === 2 && missing < 1) + ) + return false; const groups = [...left, ...Array(Math.max(0, missing)).fill('0'), ...right]; - if (groups.length !== 8 || groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group))) return false; - const numeric = groups.reduce((total, group) => (total << 16n) | BigInt(`0x${group}`), 0n); - const inCidr = (base, prefix) => { - const shift = 128n - BigInt(prefix); - return (numeric >> shift) === (base >> shift); - }; - const base = (text) => text.split(':').reduce( - (total, group) => (total << 16n) | BigInt(`0x${group || '0'}`), + if ( + groups.length !== 8 || + groups.some((group) => !/^[0-9a-f]{1,4}$/.test(group)) + ) + return false; + const numeric = groups.reduce( + (total, group) => (total << 16n) | BigInt(`0x${group}`), 0n, ); + const inCidr = (base, prefix) => { + const shift = 128n - BigInt(prefix); + return numeric >> shift === base >> shift; + }; + const base = (text) => + text + .split(':') + .reduce( + (total, group) => (total << 16n) | BigInt(`0x${group || '0'}`), + 0n, + ); const cidr = (text, prefix) => inCidr(base(text), prefix); - return cidr('2000:0:0:0:0:0:0:0', 3) - && !cidr('2001:0:0:0:0:0:0:0', 23) - && !cidr('2001:db8:0:0:0:0:0:0', 32) - && !cidr('2002:0:0:0:0:0:0:0', 16) - && !cidr('3fff:0:0:0:0:0:0:0', 20); + return ( + cidr('2000:0:0:0:0:0:0:0', 3) && + !cidr('2001:0:0:0:0:0:0:0', 23) && + !cidr('2001:db8:0:0:0:0:0:0', 32) && + !cidr('2002:0:0:0:0:0:0:0', 16) && + !cidr('3fff:0:0:0:0:0:0:0', 20) + ); } export function radioProxyDestination(value) { @@ -51,16 +74,18 @@ export function radioProxyDestination(value) { } const origin = radioMirrorOrigin(url.hostname); if ( - !origin - || url.origin !== origin - || url.username - || url.password - || url.port - || url.hash - ) return null; - const discovery = url.hostname.toLowerCase() === 'all.api.radio-browser.info' - && url.pathname === '/json/servers' - && !url.search; + !origin || + url.origin !== origin || + url.username || + url.password || + url.port || + url.hash + ) + return null; + const discovery = + url.hostname.toLowerCase() === 'all.api.radio-browser.info' && + url.pathname === '/json/servers' && + !url.search; const directory = url.pathname === '/json/stations/search'; const click = /^\/json\/url\/[0-9a-f-]+$/i.test(url.pathname) && !url.search; return discovery || directory || click ? url : null; @@ -70,9 +95,15 @@ export async function resolveRadioProxyAddresses(hostname, lookupImpl) { const resolved = await lookupImpl(hostname, { all: true, verbatim: true }); const rows = Array.isArray(resolved) ? resolved : [resolved]; const addresses = rows - .map((row) => ({ address: String(row?.address || ''), family: Number(row?.family) || undefined })) + .map((row) => ({ + address: String(row?.address || ''), + family: Number(row?.family) || undefined, + })) .filter((row) => row.address); - if (!addresses.length || addresses.some((row) => !isPublicRadioAddress(row.address))) { + if ( + !addresses.length || + addresses.some((row) => !isPublicRadioAddress(row.address)) + ) { throw new Error('Radio Browser resolved to a forbidden address'); } return addresses; @@ -81,26 +112,33 @@ export async function resolveRadioProxyAddresses(hostname, lookupImpl) { export function fetchPinnedRadioResponse(url, options, addresses) { return new Promise((resolve, reject) => { const address = addresses[0]; - const request = https.request(url, { - method: 'GET', - headers: options.headers, - signal: options.signal, - lookup(_hostname, lookupOptions, callback) { - if (lookupOptions?.all) callback(null, addresses); - else callback(null, address.address, address.family); + const request = https.request( + url, + { + method: 'GET', + headers: options.headers, + signal: options.signal, + lookup(_hostname, lookupOptions, callback) { + if (lookupOptions?.all) callback(null, addresses); + else callback(null, address.address, address.family); + }, }, - }, (response) => { - const headers = new Headers(); - for (const [name, value] of Object.entries(response.headers)) { - if (Array.isArray(value)) value.forEach((item) => headers.append(name, item)); - else if (value !== undefined) headers.set(name, String(value)); - } - resolve(new Response(Readable.toWeb(response), { - status: response.statusCode || 500, - statusText: response.statusMessage || '', - headers, - })); - }); + (response) => { + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + resolve( + new Response(Readable.toWeb(response), { + status: response.statusCode || 500, + statusText: response.statusMessage || '', + headers, + }), + ); + }, + ); request.on('error', reject); request.end(); }); diff --git a/src/tooling/mediaProviders.test.mjs b/src/tooling/mediaProviders.test.mjs index ce713ad0b..a8980fa0c 100644 --- a/src/tooling/mediaProviders.test.mjs +++ b/src/tooling/mediaProviders.test.mjs @@ -9,11 +9,21 @@ import { localProviderPlugins } from '../../server/providers/local.js'; function install(plugin) { let handler; - plugin.configureServer({ middlewares: { use(_route, fn) { handler = fn; } } }); + plugin.configureServer({ + middlewares: { + use(_route, fn) { + handler = fn; + }, + }, + }); return async (url) => { const res = { - writeHead(status, headers) { Object.assign(this, { status, headers }); }, - end(body) { this.body = body; }, + writeHead(status, headers) { + Object.assign(this, { status, headers }); + }, + end(body) { + this.body = body; + }, }; await handler({ url, method: 'GET' }, res); return res; @@ -24,14 +34,29 @@ function fixture(t, id) { const root = mkdtempSync(path.join(tmpdir(), 'gev-cctv-')); t.after(() => rmSync(root, { recursive: true, force: true })); mkdirSync(path.join(root, 'config')); - writeFileSync(path.join(root, 'config/cctv_sources.austin.json'), JSON.stringify([ - { id, name: '', lat: 30.27, lon: -97.74, feedType: 'video' }, - ])); + writeFileSync( + path.join(root, 'config/cctv_sources.austin.json'), + JSON.stringify([ + { + id, + name: '', + lat: 30.27, + lon: -97.74, + feedType: 'video', + }, + ]), + ); return root; } function isolate(t) { - for (const name of ['CCTV_SOURCES_FILE', 'CCTV_SOURCES_JSON', 'CCTV_FORCE_AUSTIN', 'GOOGLE_MAPS_SERVER_API_KEY', 'GOOGLE_MAPS_API_KEY']) { + for (const name of [ + 'CCTV_SOURCES_FILE', + 'CCTV_SOURCES_JSON', + 'CCTV_FORCE_AUSTIN', + 'GOOGLE_MAPS_SERVER_API_KEY', + 'GOOGLE_MAPS_API_KEY', + ]) { const previous = process.env[name]; delete process.env[name]; t.after(() => { @@ -39,7 +64,9 @@ function isolate(t) { else process.env[name] = previous; }); } - t.mock.method(globalThis, 'fetch', () => { throw Error('fixture must not fetch'); }); + t.mock.method(globalThis, 'fetch', () => { + throw Error('fixture must not fetch'); + }); } test('CCTV instances resolve their own application source root and isolate catalogs and health', async (t) => { @@ -49,8 +76,14 @@ test('CCTV instances resolve their own application source root and isolate catal const [a, b] = await Promise.all([first('/sources'), second('/sources')]); assert.equal(a.status, 200); assert.equal(b.status, 200); - assert.deepEqual(JSON.parse(a.body).sources.map(s => s.id), ['first']); - assert.deepEqual(JSON.parse(b.body).sources.map(s => s.id), ['second']); + assert.deepEqual( + JSON.parse(a.body).sources.map((s) => s.id), + ['first'], + ); + assert.deepEqual( + JSON.parse(b.body).sources.map((s) => s.id), + ['second'], + ); const stream = JSON.parse((await first('/stream/first')).body); assert.equal(stream.feedType, 'mp4'); assert.equal(stream.mediaUrl, '/api/cctv/media/first'); @@ -66,6 +99,6 @@ test('composition creates exactly one CCTV and radio provider without acquisitio isolate(t); const plugins = localProviderPlugins(); for (const factory of [cctvProxy, radioBrowserProxy]) { - assert.equal(plugins.filter(p => p.name === factory().name).length, 1); + assert.equal(plugins.filter((p) => p.name === factory().name).length, 1); } });