diff --git a/CHANGELOG.md b/CHANGELOG.md index 9944f0257..a23fa88da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +- Expose portable radio, camera-type and regional source helpers; keep HTTP transport separate from record normalization. + - Separate vessel records and feed acquisition from rendering while preserving selection, partial-feed retention, sea-surface placement and request cancellation. - Separate military-flight records and acquisition from rendering while preserving ground-model ownership, source units and follow behavior. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fbb8e7a31..3e43c1c70 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,7 @@ # God's Eye View Current State +Portable source exports provide Radio Browser station normalization, CCTV feed types and regional records independently of HTTP middleware. Radio normalization accepts an explicit URL policy; the standalone directory retains its existing HTTPS rules. Tile coordinate validation accepts explicit zoom bounds with unchanged traffic defaults. Cache, request and rendering owners remain unchanged. + Vessels separate plain, stable MMSI records and source acquisition from scene resources. The renderer owns geometry and billboards; cards, picking and trails resolve those resources through it. Partial-feed retention, selected-vessel diff --git a/package.json b/package.json index 77e4b33eb..2c67a5ddb 100644 --- a/package.json +++ b/package.json @@ -207,6 +207,9 @@ "./layers/military/records": "./src/layers/military/records.js", "./layers/military/ingestion": "./src/layers/military/ingestion.js", "./layers/vessels/records": "./src/layers/vessels/records.js", - "./layers/vessels/ingestion": "./src/layers/vessels/ingestion.js" + "./layers/vessels/ingestion": "./src/layers/vessels/ingestion.js", + "./sources/radio-browser": "./src/sources/radioBrowser.js", + "./sources/cctv-types": "./src/sources/cctvTypes.js", + "./sources/regional": "./src/data/regionalModel.js" } } diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 9341046c1..5ac2713fd 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -648,5 +648,8 @@ "src/data/directions.js", "src/data/directions.test.mjs", "src/app/layers/directions.js", - "src/layers/directions/index.js" + "src/layers/directions/index.js", + "src/sources/radioBrowser.js", + "src/sources/cctvTypes.js", + "src/sources/protocols.test.mjs" ] diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ae84f6c61..7858c9bb4 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -161,7 +161,8 @@ "server/providers/common/geo.js", "server/providers/places/google-key.js", "scripts/google-server-key.mjs", - "src/data/cctvFootprint.js" + "src/data/cctvFootprint.js", + "src/sources/cctvTypes.js" ], "external": [] }, @@ -175,7 +176,8 @@ "server/providers/radio/stations.js", "server/providers/radio/transport.js", "src/data/radioCountry.js", - "server/providers/common/http.js" + "server/providers/common/http.js", + "src/sources/radioBrowser.js" ], "external": [] }, @@ -2119,5 +2121,20 @@ "exports": ["./layers/directions"], "modules": ["src/layers/directions/index.js", "src/data/routeSteps.js"], "external": ["cesium"] + }, + "radio-browser-source": { + "exports": ["./sources/radio-browser"], + "modules": ["src/sources/radioBrowser.js", "src/data/radioCountry.js"], + "external": [] + }, + "cctv-types-source": { + "exports": ["./sources/cctv-types"], + "modules": ["src/sources/cctvTypes.js"], + "external": [] + }, + "regional-source": { + "exports": ["./sources/regional"], + "modules": ["src/data/regionalModel.js"], + "external": [] } } diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index ffe025b30..b564ef440 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -1,3 +1,8 @@ +import { + normalizeFeedType, + isVideoFeedType, +} from '../../../src/sources/cctvTypes.js'; +export { normalizeFeedType, isVideoFeedType }; import { directionToHeading } from '../../../src/data/directionText.js'; import { haversineKm } from '../common/geo.js'; /** @@ -31,35 +36,6 @@ export function escapeXml(text) { .replace(/'/g, '''); } -/** - * Canonicalize a CCTV feed type string to one of: - * 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through. - * - * @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream'). - * @returns {string} Normalized feed type. - */ -export function normalizeFeedType(value) { - const raw = String(value || '') - .trim() - .toLowerCase(); - if (!raw) return 'image'; - if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; - if (raw === 'mjpg') return 'mjpeg'; - if (raw === 'video') return 'mp4'; - if (raw === 'stream') return 'hls'; - return raw; -} - -/** - * Check whether a normalized feed type represents streaming video. - * - * @param {string} feedType - * @returns {boolean} - */ -export function isVideoFeedType(feedType) { - return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls'; -} - /** * Coerce a value to a finite number, returning fallback if NaN/Infinity. * diff --git a/server/providers/radio/constants.js b/server/providers/radio/constants.js index 996e2af06..0fee93ac6 100644 --- a/server/providers/radio/constants.js +++ b/server/providers/radio/constants.js @@ -10,8 +10,7 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil( ); export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; -export const RADIO_UUID_RE = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js'; export const RADIO_FALLBACK_MIRRORS = Object.freeze([ 'https://de1.api.radio-browser.info', 'https://de2.api.radio-browser.info', diff --git a/server/providers/radio/stations.js b/server/providers/radio/stations.js index 4d9bcf35c..b97b2d3c6 100644 --- a/server/providers/radio/stations.js +++ b/server/providers/radio/stations.js @@ -1,155 +1,7 @@ -import { RADIO_UUID_RE } from './constants.js'; -import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js'; -export function cleanRadioText(value, maxLength) { - return String(value ?? '') - .replace(/[\u0000-\u001f\u007f]/g, ' ') - .replace(/\s+/g, ' ') - .trim() - .slice(0, maxLength) - .trim(); -} - -export function isNonGlobalIpv4(hostname) { - const pieces = hostname.split('.'); - if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) - return false; - const values = pieces.map(Number); - if (values.some((value) => value > 255)) return true; - const [a, b, c] = values; - return ( - a === 0 || - a === 10 || - a === 127 || - a >= 224 || - (a === 100 && b >= 64 && b <= 127) || - (a === 169 && b === 254) || - (a === 172 && b >= 16 && b <= 31) || - (a === 192 && b === 0) || - (a === 192 && b === 88 && c === 99) || - (a === 192 && b === 168) || - (a === 198 && (b === 18 || b === 19)) || - (a === 198 && b === 51 && c === 100) || - (a === 203 && b === 0 && c === 113) - ); -} - -/** Return a normalized public HTTPS URL, or null for local/private targets. */ -export function publicRadioHttpsUrl(value) { - try { - const url = new URL(String(value ?? '')); - const hostname = url.hostname - .toLowerCase() - .replace(/^\[|\]$/g, '') - .replace(/\.$/, ''); - if (url.protocol !== 'https:' || url.username || url.password || !hostname) - return null; - if ( - hostname === 'localhost' || - hostname.endsWith('.localhost') || - hostname.endsWith('.local') || - isNonGlobalIpv4(hostname) || - hostname.includes(':') - ) - return null; - url.hash = ''; - return url.href; - } catch { - return null; - } -} - -/** Normalize one Radio Browser station and omit favicons and unsafe streams. */ -export function normalizeRadioBrowserStation(raw) { - const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); - const lat = - raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); - const lon = - raw?.geo_long === null || raw?.geo_long === '' - ? null - : Number(raw?.geo_long); - const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); - const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url); - if ( - !RADIO_UUID_RE.test(id) || - Number(raw?.lastcheckok) !== 1 || - Number(raw?.hls) === 1 || - !Number.isFinite(lat) || - lat < -90 || - lat > 90 || - !Number.isFinite(lon) || - lon < -180 || - lon > 180 || - !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) || - !streamUrl - ) - return null; - - const name = cleanRadioText(raw?.name, 140); - if (!name) return null; - const tags = String(raw?.tags ?? '') - .split(',') - .map((tag) => - cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(), - ) - .filter(Boolean) - .filter((tag, index, all) => all.indexOf(tag) === index) - .slice(0, 24); - const languages = String(raw?.language ?? '') - .split(',') - .map((language) => cleanRadioText(language, 40)) - .filter(Boolean) - .slice(0, 8); - const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); - const normalizedCode = normalizeRadioCountryInput(rawCountryCode); - const normalizedCountry = - normalizedCode.valid && !normalizedCode.empty - ? normalizedCode - : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); - const bitrate = Number(raw?.bitrate); - return { - id, - name, - lat, - lon, - streamUrl, - homepage: publicRadioHttpsUrl(raw?.homepage), - tags, - languages, - state: cleanRadioText(raw?.state, 80), - country: - normalizedCountry.valid && !normalizedCountry.empty - ? normalizedCountry.name - : cleanRadioText(raw?.country, 80), - countryCode: normalizedCountry.valid ? normalizedCountry.code : '', - metadataTrust: 'untrusted-community', - codec, - bitrate: - Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 - ? bitrate - : null, - clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), - }; -} - -export function publicRadioStation(station) { - return { - id: station.id, - name: station.name, - lat: station.lat, - lon: station.lon, - streamUrl: station.streamUrl, - homepage: station.homepage, - tags: station.tags, - languages: station.languages, - state: station.state, - country: station.country, - countryCode: station.countryCode, - metadataTrust: station.metadataTrust, - codec: station.codec, - bitrate: station.bitrate, - }; -} +export { + cleanRadioText, + isNonGlobalIpv4, + publicRadioHttpsUrl, + normalizeRadioBrowserStation, + publicRadioStation, +} from '../../../src/sources/radioBrowser.js'; diff --git a/src/data/tomtomTiles.js b/src/data/tomtomTiles.js index f0da57e46..af4fc95b3 100644 --- a/src/data/tomtomTiles.js +++ b/src/data/tomtomTiles.js @@ -26,12 +26,26 @@ const MERCATOR_LAT_LIMIT = 85.05112878; * @param {number} z - Zoom level; integer within [MIN_TILE_ZOOM, MAX_TILE_ZOOM]. * @param {number} x - Tile column; integer within [0, 2^z - 1]. * @param {number} y - Tile row; integer within [0, 2^z - 1]. + * @param {{minZoom?: number, maxZoom?: number}} [bounds] Supported source zooms. * @returns {boolean} True when the coordinate is a fetchable tile. */ -export function isValidTileCoord(z, x, y) { +export function isValidTileCoord( + z, + x, + y, + { minZoom = MIN_TILE_ZOOM, maxZoom = MAX_TILE_ZOOM } = {}, +) { if (!Number.isInteger(z) || !Number.isInteger(x) || !Number.isInteger(y)) return false; - if (z < MIN_TILE_ZOOM || z > MAX_TILE_ZOOM) return false; + if ( + !Number.isInteger(minZoom) || + !Number.isInteger(maxZoom) || + minZoom < 0 || + maxZoom > 30 || + minZoom > maxZoom + ) + return false; + if (z < minZoom || z > maxZoom) return false; const n = 2 ** z; return x >= 0 && x < n && y >= 0 && y < n; } diff --git a/src/sources/cctvTypes.js b/src/sources/cctvTypes.js new file mode 100644 index 000000000..2a3f6a6a3 --- /dev/null +++ b/src/sources/cctvTypes.js @@ -0,0 +1,28 @@ +/** + * Canonicalize a CCTV feed type string to one of: + * 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through. + * + * @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream'). + * @returns {string} Normalized feed type. + */ +export function normalizeFeedType(value) { + const raw = String(value || '') + .trim() + .toLowerCase(); + if (!raw) return 'image'; + if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image'; + if (raw === 'mjpg') return 'mjpeg'; + if (raw === 'video') return 'mp4'; + if (raw === 'stream') return 'hls'; + return raw; +} + +/** + * Check whether a normalized feed type represents streaming video. + * + * @param {string} feedType + * @returns {boolean} + */ +export function isVideoFeedType(feedType) { + return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls'; +} diff --git a/src/sources/protocols.test.mjs b/src/sources/protocols.test.mjs new file mode 100644 index 000000000..c4d90e6b0 --- /dev/null +++ b/src/sources/protocols.test.mjs @@ -0,0 +1,116 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + normalizeRadioBrowserStation, + publicRadioStation, + publicRadioHttpsUrl, +} from './radioBrowser.js'; +import { normalizeFeedType, isVideoFeedType } from './cctvTypes.js'; +import { isValidTileCoord } from '../data/tomtomTiles.js'; + +const station = { + stationuuid: 'abcdefff-1234-5678-abcd-123456789abc', + name: ' Test\n Radio ', + geo_lat: '30.2', + geo_long: '-97.7', + url_resolved: 'https://radio.example/stream', + homepage: 'https://radio.example/', + codec: 'mp3', + lastcheckok: 1, + hls: 0, + countrycode: 'US', + country: 'US', + tags: 'News,news,air_traffic', + bitrate: 128, + clickcount: 10, +}; + +test('radio records normalize identity, country and tags and project only public fields', () => { + const result = normalizeRadioBrowserStation(station); + assert.equal(result.name, 'Test Radio'); + assert.equal(result.countryCode, 'US'); + assert.equal(result.country, 'United States'); + assert.deepEqual(result.tags, ['news', 'air traffic']); + assert.equal(result.lat, 30.2); + assert.equal(result.metadataTrust, 'untrusted-community'); + assert.equal('clickCount' in publicRadioStation(result), false); + assert.equal( + 'extra' in publicRadioStation({ ...result, extra: 'not a record field' }), + false, + ); +}); + +test('radio rejects unusable metadata and unsafe default stream destinations', () => { + for (const patch of [ + { geo_lat: null }, + { geo_long: undefined }, + { geo_lat: 91 }, + { stationuuid: 'invalid' }, + { name: '' }, + { hls: 1 }, + { lastcheckok: 0 }, + { codec: 'unknown' }, + ...[ + 'http://radio.example/', + 'https://localhost/', + 'https://127.0.0.1/', + 'https://user:pass@radio.example/', + 'https://[::1]/', + ].map((url_resolved) => ({ url_resolved })), + ]) + assert.equal(normalizeRadioBrowserStation({ ...station, ...patch }), null); +}); + +test('a supplied URL policy applies independently to stream and homepage', () => { + const normalizeUrl = (value) => { + const safe = publicRadioHttpsUrl(value); + return safe && !new URL(safe).port ? safe : null; + }; + assert.equal( + normalizeRadioBrowserStation( + { ...station, url_resolved: 'https://radio.example:8443/' }, + { normalizeUrl }, + ), + null, + ); + const record = normalizeRadioBrowserStation( + { ...station, homepage: 'https://radio.example:8443/' }, + { normalizeUrl }, + ); + assert.equal(record.streamUrl, station.url_resolved); + assert.equal(record.homepage, null); + assert.ok( + normalizeRadioBrowserStation({ + ...station, + url_resolved: 'https://radio.example:8443/', + }), + ); +}); + +test('camera aliases remain separate from player support decisions', () => { + for (const [input, expected] of [ + ['JPEG', 'image'], + ['mjpg', 'mjpeg'], + ['video', 'mp4'], + ['stream', 'hls'], + ['', 'image'], + ['other', 'other'], + ]) { + assert.equal(normalizeFeedType(input), expected); + } + assert.equal(isVideoFeedType('mjpeg'), false); + assert.equal(isVideoFeedType('hls'), true); +}); + +test('XYZ math preserves traffic bounds while allowing explicit source bounds', () => { + assert.equal(isValidTileCoord(7, 0, 0), false); + assert.equal(isValidTileCoord(17, 0, 0), false); + assert.equal(isValidTileCoord(8, 255, 255), true); + assert.equal(isValidTileCoord(8, 256, 0), false); + const bounds = { minZoom: 0, maxZoom: 22 }; + assert.equal(isValidTileCoord(0, 0, 0, bounds), true); + assert.equal(isValidTileCoord(22, 2 ** 22 - 1, 0, bounds), true); + assert.equal(isValidTileCoord(23, 0, 0, bounds), false); + assert.equal(isValidTileCoord(4, 1.5, 0, bounds), false); + assert.equal(isValidTileCoord(4, 0, 0, { minZoom: 5, maxZoom: 2 }), false); +}); diff --git a/src/sources/radioBrowser.js b/src/sources/radioBrowser.js new file mode 100644 index 000000000..8511b6beb --- /dev/null +++ b/src/sources/radioBrowser.js @@ -0,0 +1,166 @@ +import { normalizeRadioCountryInput } from '../data/radioCountry.js'; + +export const RADIO_UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export function cleanRadioText(value, maxLength) { + return String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, maxLength) + .trim(); +} + +export function isNonGlobalIpv4(hostname) { + const pieces = hostname.split('.'); + if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) + return false; + const values = pieces.map(Number); + if (values.some((value) => value > 255)) return true; + const [a, b, c] = values; + return ( + a === 0 || + a === 10 || + a === 127 || + a >= 224 || + (a === 100 && b >= 64 && b <= 127) || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 0) || + (a === 192 && b === 88 && c === 99) || + (a === 192 && b === 168) || + (a === 198 && (b === 18 || b === 19)) || + (a === 198 && b === 51 && c === 100) || + (a === 203 && b === 0 && c === 113) + ); +} + +/** Return a normalized public HTTPS URL, or null for local/private targets. */ +export function publicRadioHttpsUrl(value) { + try { + const url = new URL(String(value ?? '')); + const hostname = url.hostname + .toLowerCase() + .replace(/^\[|\]$/g, '') + .replace(/\.$/, ''); + if (url.protocol !== 'https:' || url.username || url.password || !hostname) + return null; + if ( + hostname === 'localhost' || + hostname.endsWith('.localhost') || + hostname.endsWith('.local') || + isNonGlobalIpv4(hostname) || + hostname.includes(':') + ) + return null; + url.hash = ''; + return url.href; + } catch { + return null; + } +} + +/** + * Normalize one Radio Browser station and omit favicons and unsafe streams. + * @param {object} raw Directory record. + * @param {object} [options] + * @param {(value: unknown) => string|null} [options.normalizeUrl] URL admission + * policy, applied to both stream and homepage; returns a safe URL or null. + */ +export function normalizeRadioBrowserStation( + raw, + { normalizeUrl = publicRadioHttpsUrl } = {}, +) { + const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase(); + const lat = + raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat); + const lon = + raw?.geo_long === null || raw?.geo_long === '' + ? null + : Number(raw?.geo_long); + const codec = cleanRadioText(raw?.codec, 16).toUpperCase(); + const streamUrl = normalizeUrl(raw?.url_resolved || raw?.url); + if ( + !RADIO_UUID_RE.test(id) || + Number(raw?.lastcheckok) !== 1 || + Number(raw?.hls) === 1 || + !Number.isFinite(lat) || + lat < -90 || + lat > 90 || + !Number.isFinite(lon) || + lon < -180 || + lon > 180 || + !/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) || + !streamUrl + ) + return null; + + const name = cleanRadioText(raw?.name, 140); + if (!name) return null; + const tags = String(raw?.tags ?? '') + .split(',') + .map((tag) => + cleanRadioText(tag, 80) + .toLocaleLowerCase() + .replace(/[_-]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(), + ) + .filter(Boolean) + .filter((tag, index, all) => all.indexOf(tag) === index) + .slice(0, 24); + const languages = String(raw?.language ?? '') + .split(',') + .map((language) => cleanRadioText(language, 40)) + .filter(Boolean) + .slice(0, 8); + const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase(); + const normalizedCode = normalizeRadioCountryInput(rawCountryCode); + const normalizedCountry = + normalizedCode.valid && !normalizedCode.empty + ? normalizedCode + : normalizeRadioCountryInput(cleanRadioText(raw?.country, 80)); + const bitrate = Number(raw?.bitrate); + return { + id, + name, + lat, + lon, + streamUrl, + homepage: normalizeUrl(raw?.homepage), + tags, + languages, + state: cleanRadioText(raw?.state, 80), + country: + normalizedCountry.valid && !normalizedCountry.empty + ? normalizedCountry.name + : cleanRadioText(raw?.country, 80), + countryCode: normalizedCountry.valid ? normalizedCountry.code : '', + metadataTrust: 'untrusted-community', + codec, + bitrate: + Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024 + ? bitrate + : null, + clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)), + }; +} + +export function publicRadioStation(station) { + return { + id: station.id, + name: station.name, + lat: station.lat, + lon: station.lon, + streamUrl: station.streamUrl, + homepage: station.homepage, + tags: station.tags, + languages: station.languages, + state: station.state, + country: station.country, + countryCode: station.countryCode, + metadataTrust: station.metadataTrust, + codec: station.codec, + bitrate: station.bitrate, + }; +}