refactor: expose portable source record protocols

This commit is contained in:
Sameh Khamis committed 2026-09-14 19:36:32 -07:00
1 parent 6fa25c349a
commit ea6c8653b3
12 files changed
+370 -192

No files matched your search

+2
View File
@@ -1,5 +1,7 @@
# Changelog # Changelog
- Expose portable radio, camera-type and regional source helpers; keep HTTP transport separate from record normalization.
- Separate vessel records and feed acquisition from rendering while preserving selection, partial-feed retention, sea-surface placement and request cancellation. - Separate vessel records and feed acquisition from rendering while preserving selection, partial-feed retention, sea-surface placement and request cancellation.
- Separate military-flight records and acquisition from rendering while preserving ground-model ownership, source units and follow behavior. - Separate military-flight records and acquisition from rendering while preserving ground-model ownership, source units and follow behavior.
+2
View File
@@ -1,5 +1,7 @@
# God's Eye View Current State # God's Eye View Current State
Portable source exports provide Radio Browser station normalization, CCTV feed types and regional records independently of HTTP middleware. Radio normalization accepts an explicit URL policy; the standalone directory retains its existing HTTPS rules. Tile coordinate validation accepts explicit zoom bounds with unchanged traffic defaults. Cache, request and rendering owners remain unchanged.
Vessels separate plain, stable MMSI records and source acquisition from scene Vessels separate plain, stable MMSI records and source acquisition from scene
resources. The renderer owns geometry and billboards; cards, picking and trails resources. The renderer owns geometry and billboards; cards, picking and trails
resolve those resources through it. Partial-feed retention, selected-vessel resolve those resources through it. Partial-feed retention, selected-vessel
+4 -1
View File
@@ -207,6 +207,9 @@
"./layers/military/records": "./src/layers/military/records.js", "./layers/military/records": "./src/layers/military/records.js",
"./layers/military/ingestion": "./src/layers/military/ingestion.js", "./layers/military/ingestion": "./src/layers/military/ingestion.js",
"./layers/vessels/records": "./src/layers/vessels/records.js", "./layers/vessels/records": "./src/layers/vessels/records.js",
"./layers/vessels/ingestion": "./src/layers/vessels/ingestion.js" "./layers/vessels/ingestion": "./src/layers/vessels/ingestion.js",
"./sources/radio-browser": "./src/sources/radioBrowser.js",
"./sources/cctv-types": "./src/sources/cctvTypes.js",
"./sources/regional": "./src/data/regionalModel.js"
} }
} }
+4 -1
View File
@@ -648,5 +648,8 @@
"src/data/directions.js", "src/data/directions.js",
"src/data/directions.test.mjs", "src/data/directions.test.mjs",
"src/app/layers/directions.js", "src/app/layers/directions.js",
"src/layers/directions/index.js" "src/layers/directions/index.js",
"src/sources/radioBrowser.js",
"src/sources/cctvTypes.js",
"src/sources/protocols.test.mjs"
] ]
+19 -2
View File
@@ -161,7 +161,8 @@
"server/providers/common/geo.js", "server/providers/common/geo.js",
"server/providers/places/google-key.js", "server/providers/places/google-key.js",
"scripts/google-server-key.mjs", "scripts/google-server-key.mjs",
"src/data/cctvFootprint.js" "src/data/cctvFootprint.js",
"src/sources/cctvTypes.js"
], ],
"external": [] "external": []
}, },
@@ -175,7 +176,8 @@
"server/providers/radio/stations.js", "server/providers/radio/stations.js",
"server/providers/radio/transport.js", "server/providers/radio/transport.js",
"src/data/radioCountry.js", "src/data/radioCountry.js",
"server/providers/common/http.js" "server/providers/common/http.js",
"src/sources/radioBrowser.js"
], ],
"external": [] "external": []
}, },
@@ -2119,5 +2121,20 @@
"exports": ["./layers/directions"], "exports": ["./layers/directions"],
"modules": ["src/layers/directions/index.js", "src/data/routeSteps.js"], "modules": ["src/layers/directions/index.js", "src/data/routeSteps.js"],
"external": ["cesium"] "external": ["cesium"]
},
"radio-browser-source": {
"exports": ["./sources/radio-browser"],
"modules": ["src/sources/radioBrowser.js", "src/data/radioCountry.js"],
"external": []
},
"cctv-types-source": {
"exports": ["./sources/cctv-types"],
"modules": ["src/sources/cctvTypes.js"],
"external": []
},
"regional-source": {
"exports": ["./sources/regional"],
"modules": ["src/data/regionalModel.js"],
"external": []
} }
} }
+5 -29
View File
@@ -1,3 +1,8 @@
import {
normalizeFeedType,
isVideoFeedType,
} from '../../../src/sources/cctvTypes.js';
export { normalizeFeedType, isVideoFeedType };
import { directionToHeading } from '../../../src/data/directionText.js'; import { directionToHeading } from '../../../src/data/directionText.js';
import { haversineKm } from '../common/geo.js'; import { haversineKm } from '../common/geo.js';
/** /**
@@ -31,35 +36,6 @@ export function escapeXml(text) {
.replace(/'/g, '''); .replace(/'/g, ''');
} }
/**
* Canonicalize a CCTV feed type string to one of:
* 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through.
*
* @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream').
* @returns {string} Normalized feed type.
*/
export function normalizeFeedType(value) {
const raw = String(value || '')
.trim()
.toLowerCase();
if (!raw) return 'image';
if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image';
if (raw === 'mjpg') return 'mjpeg';
if (raw === 'video') return 'mp4';
if (raw === 'stream') return 'hls';
return raw;
}
/**
* Check whether a normalized feed type represents streaming video.
*
* @param {string} feedType
* @returns {boolean}
*/
export function isVideoFeedType(feedType) {
return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls';
}
/** /**
* Coerce a value to a finite number, returning fallback if NaN/Infinity. * Coerce a value to a finite number, returning fallback if NaN/Infinity.
* *
+1 -2
View File
@@ -10,8 +10,7 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
); );
export const RADIO_USER_AGENT = export const RADIO_USER_AGENT =
'GodsEyeView/1.0 (Radio Browser directory client)'; 'GodsEyeView/1.0 (Radio Browser directory client)';
export const RADIO_UUID_RE = export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js';
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const RADIO_FALLBACK_MIRRORS = Object.freeze([ export const RADIO_FALLBACK_MIRRORS = Object.freeze([
'https://de1.api.radio-browser.info', 'https://de1.api.radio-browser.info',
'https://de2.api.radio-browser.info', 'https://de2.api.radio-browser.info',
+7 -155
View File
@@ -1,155 +1,7 @@
import { RADIO_UUID_RE } from './constants.js'; export {
import { normalizeRadioCountryInput } from '../../../src/data/radioCountry.js'; cleanRadioText,
export function cleanRadioText(value, maxLength) { isNonGlobalIpv4,
return String(value ?? '') publicRadioHttpsUrl,
.replace(/[\u0000-\u001f\u007f]/g, ' ') normalizeRadioBrowserStation,
.replace(/\s+/g, ' ') publicRadioStation,
.trim() } from '../../../src/sources/radioBrowser.js';
.slice(0, maxLength)
.trim();
}
export function isNonGlobalIpv4(hostname) {
const pieces = hostname.split('.');
if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece)))
return false;
const values = pieces.map(Number);
if (values.some((value) => value > 255)) return true;
const [a, b, c] = values;
return (
a === 0 ||
a === 10 ||
a === 127 ||
a >= 224 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 0) ||
(a === 192 && b === 88 && c === 99) ||
(a === 192 && b === 168) ||
(a === 198 && (b === 18 || b === 19)) ||
(a === 198 && b === 51 && c === 100) ||
(a === 203 && b === 0 && c === 113)
);
}
/** Return a normalized public HTTPS URL, or null for local/private targets. */
export function publicRadioHttpsUrl(value) {
try {
const url = new URL(String(value ?? ''));
const hostname = url.hostname
.toLowerCase()
.replace(/^\[|\]$/g, '')
.replace(/\.$/, '');
if (url.protocol !== 'https:' || url.username || url.password || !hostname)
return null;
if (
hostname === 'localhost' ||
hostname.endsWith('.localhost') ||
hostname.endsWith('.local') ||
isNonGlobalIpv4(hostname) ||
hostname.includes(':')
)
return null;
url.hash = '';
return url.href;
} catch {
return null;
}
}
/** Normalize one Radio Browser station and omit favicons and unsafe streams. */
export function normalizeRadioBrowserStation(raw) {
const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase();
const lat =
raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat);
const lon =
raw?.geo_long === null || raw?.geo_long === ''
? null
: Number(raw?.geo_long);
const codec = cleanRadioText(raw?.codec, 16).toUpperCase();
const streamUrl = publicRadioHttpsUrl(raw?.url_resolved || raw?.url);
if (
!RADIO_UUID_RE.test(id) ||
Number(raw?.lastcheckok) !== 1 ||
Number(raw?.hls) === 1 ||
!Number.isFinite(lat) ||
lat < -90 ||
lat > 90 ||
!Number.isFinite(lon) ||
lon < -180 ||
lon > 180 ||
!/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) ||
!streamUrl
)
return null;
const name = cleanRadioText(raw?.name, 140);
if (!name) return null;
const tags = String(raw?.tags ?? '')
.split(',')
.map((tag) =>
cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim(),
)
.filter(Boolean)
.filter((tag, index, all) => all.indexOf(tag) === index)
.slice(0, 24);
const languages = String(raw?.language ?? '')
.split(',')
.map((language) => cleanRadioText(language, 40))
.filter(Boolean)
.slice(0, 8);
const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase();
const normalizedCode = normalizeRadioCountryInput(rawCountryCode);
const normalizedCountry =
normalizedCode.valid && !normalizedCode.empty
? normalizedCode
: normalizeRadioCountryInput(cleanRadioText(raw?.country, 80));
const bitrate = Number(raw?.bitrate);
return {
id,
name,
lat,
lon,
streamUrl,
homepage: publicRadioHttpsUrl(raw?.homepage),
tags,
languages,
state: cleanRadioText(raw?.state, 80),
country:
normalizedCountry.valid && !normalizedCountry.empty
? normalizedCountry.name
: cleanRadioText(raw?.country, 80),
countryCode: normalizedCountry.valid ? normalizedCountry.code : '',
metadataTrust: 'untrusted-community',
codec,
bitrate:
Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024
? bitrate
: null,
clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)),
};
}
export function publicRadioStation(station) {
return {
id: station.id,
name: station.name,
lat: station.lat,
lon: station.lon,
streamUrl: station.streamUrl,
homepage: station.homepage,
tags: station.tags,
languages: station.languages,
state: station.state,
country: station.country,
countryCode: station.countryCode,
metadataTrust: station.metadataTrust,
codec: station.codec,
bitrate: station.bitrate,
};
}
+16 -2
View File
@@ -26,12 +26,26 @@ const MERCATOR_LAT_LIMIT = 85.05112878;
* @param {number} z - Zoom level; integer within [MIN_TILE_ZOOM, MAX_TILE_ZOOM]. * @param {number} z - Zoom level; integer within [MIN_TILE_ZOOM, MAX_TILE_ZOOM].
* @param {number} x - Tile column; integer within [0, 2^z - 1]. * @param {number} x - Tile column; integer within [0, 2^z - 1].
* @param {number} y - Tile row; integer within [0, 2^z - 1]. * @param {number} y - Tile row; integer within [0, 2^z - 1].
* @param {{minZoom?: number, maxZoom?: number}} [bounds] Supported source zooms.
* @returns {boolean} True when the coordinate is a fetchable tile. * @returns {boolean} True when the coordinate is a fetchable tile.
*/ */
export function isValidTileCoord(z, x, y) { export function isValidTileCoord(
z,
x,
y,
{ minZoom = MIN_TILE_ZOOM, maxZoom = MAX_TILE_ZOOM } = {},
) {
if (!Number.isInteger(z) || !Number.isInteger(x) || !Number.isInteger(y)) if (!Number.isInteger(z) || !Number.isInteger(x) || !Number.isInteger(y))
return false; return false;
if (z < MIN_TILE_ZOOM || z > MAX_TILE_ZOOM) return false; if (
!Number.isInteger(minZoom) ||
!Number.isInteger(maxZoom) ||
minZoom < 0 ||
maxZoom > 30 ||
minZoom > maxZoom
)
return false;
if (z < minZoom || z > maxZoom) return false;
const n = 2 ** z; const n = 2 ** z;
return x >= 0 && x < n && y >= 0 && y < n; return x >= 0 && x < n && y >= 0 && y < n;
} }
+28
View File
@@ -0,0 +1,28 @@
/**
* Canonicalize a CCTV feed type string to one of:
* 'image', 'mjpeg', 'mp4', 'webm', 'hls', or pass-through.
*
* @param {string} value - Raw feed type (e.g. 'jpeg', 'mjpg', 'video', 'stream').
* @returns {string} Normalized feed type.
*/
export function normalizeFeedType(value) {
const raw = String(value || '')
.trim()
.toLowerCase();
if (!raw) return 'image';
if (raw === 'jpeg' || raw === 'jpg' || raw === 'png') return 'image';
if (raw === 'mjpg') return 'mjpeg';
if (raw === 'video') return 'mp4';
if (raw === 'stream') return 'hls';
return raw;
}
/**
* Check whether a normalized feed type represents streaming video.
*
* @param {string} feedType
* @returns {boolean}
*/
export function isVideoFeedType(feedType) {
return feedType === 'mp4' || feedType === 'webm' || feedType === 'hls';
}
+116
View File
@@ -0,0 +1,116 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
normalizeRadioBrowserStation,
publicRadioStation,
publicRadioHttpsUrl,
} from './radioBrowser.js';
import { normalizeFeedType, isVideoFeedType } from './cctvTypes.js';
import { isValidTileCoord } from '../data/tomtomTiles.js';
const station = {
stationuuid: 'abcdefff-1234-5678-abcd-123456789abc',
name: ' Test\n Radio ',
geo_lat: '30.2',
geo_long: '-97.7',
url_resolved: 'https://radio.example/stream',
homepage: 'https://radio.example/',
codec: 'mp3',
lastcheckok: 1,
hls: 0,
countrycode: 'US',
country: 'US',
tags: 'News,news,air_traffic',
bitrate: 128,
clickcount: 10,
};
test('radio records normalize identity, country and tags and project only public fields', () => {
const result = normalizeRadioBrowserStation(station);
assert.equal(result.name, 'Test Radio');
assert.equal(result.countryCode, 'US');
assert.equal(result.country, 'United States');
assert.deepEqual(result.tags, ['news', 'air traffic']);
assert.equal(result.lat, 30.2);
assert.equal(result.metadataTrust, 'untrusted-community');
assert.equal('clickCount' in publicRadioStation(result), false);
assert.equal(
'extra' in publicRadioStation({ ...result, extra: 'not a record field' }),
false,
);
});
test('radio rejects unusable metadata and unsafe default stream destinations', () => {
for (const patch of [
{ geo_lat: null },
{ geo_long: undefined },
{ geo_lat: 91 },
{ stationuuid: 'invalid' },
{ name: '' },
{ hls: 1 },
{ lastcheckok: 0 },
{ codec: 'unknown' },
...[
'http://radio.example/',
'https://localhost/',
'https://127.0.0.1/',
'https://user:[email protected]/',
'https://[::1]/',
].map((url_resolved) => ({ url_resolved })),
])
assert.equal(normalizeRadioBrowserStation({ ...station, ...patch }), null);
});
test('a supplied URL policy applies independently to stream and homepage', () => {
const normalizeUrl = (value) => {
const safe = publicRadioHttpsUrl(value);
return safe && !new URL(safe).port ? safe : null;
};
assert.equal(
normalizeRadioBrowserStation(
{ ...station, url_resolved: 'https://radio.example:8443/' },
{ normalizeUrl },
),
null,
);
const record = normalizeRadioBrowserStation(
{ ...station, homepage: 'https://radio.example:8443/' },
{ normalizeUrl },
);
assert.equal(record.streamUrl, station.url_resolved);
assert.equal(record.homepage, null);
assert.ok(
normalizeRadioBrowserStation({
...station,
url_resolved: 'https://radio.example:8443/',
}),
);
});
test('camera aliases remain separate from player support decisions', () => {
for (const [input, expected] of [
['JPEG', 'image'],
['mjpg', 'mjpeg'],
['video', 'mp4'],
['stream', 'hls'],
['', 'image'],
['other', 'other'],
]) {
assert.equal(normalizeFeedType(input), expected);
}
assert.equal(isVideoFeedType('mjpeg'), false);
assert.equal(isVideoFeedType('hls'), true);
});
test('XYZ math preserves traffic bounds while allowing explicit source bounds', () => {
assert.equal(isValidTileCoord(7, 0, 0), false);
assert.equal(isValidTileCoord(17, 0, 0), false);
assert.equal(isValidTileCoord(8, 255, 255), true);
assert.equal(isValidTileCoord(8, 256, 0), false);
const bounds = { minZoom: 0, maxZoom: 22 };
assert.equal(isValidTileCoord(0, 0, 0, bounds), true);
assert.equal(isValidTileCoord(22, 2 ** 22 - 1, 0, bounds), true);
assert.equal(isValidTileCoord(23, 0, 0, bounds), false);
assert.equal(isValidTileCoord(4, 1.5, 0, bounds), false);
assert.equal(isValidTileCoord(4, 0, 0, { minZoom: 5, maxZoom: 2 }), false);
});
+166
View File
@@ -0,0 +1,166 @@
import { normalizeRadioCountryInput } from '../data/radioCountry.js';
export const RADIO_UUID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export function cleanRadioText(value, maxLength) {
return String(value ?? '')
.replace(/[\u0000-\u001f\u007f]/g, ' ')
.replace(/\s+/g, ' ')
.trim()
.slice(0, maxLength)
.trim();
}
export function isNonGlobalIpv4(hostname) {
const pieces = hostname.split('.');
if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece)))
return false;
const values = pieces.map(Number);
if (values.some((value) => value > 255)) return true;
const [a, b, c] = values;
return (
a === 0 ||
a === 10 ||
a === 127 ||
a >= 224 ||
(a === 100 && b >= 64 && b <= 127) ||
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 0) ||
(a === 192 && b === 88 && c === 99) ||
(a === 192 && b === 168) ||
(a === 198 && (b === 18 || b === 19)) ||
(a === 198 && b === 51 && c === 100) ||
(a === 203 && b === 0 && c === 113)
);
}
/** Return a normalized public HTTPS URL, or null for local/private targets. */
export function publicRadioHttpsUrl(value) {
try {
const url = new URL(String(value ?? ''));
const hostname = url.hostname
.toLowerCase()
.replace(/^\[|\]$/g, '')
.replace(/\.$/, '');
if (url.protocol !== 'https:' || url.username || url.password || !hostname)
return null;
if (
hostname === 'localhost' ||
hostname.endsWith('.localhost') ||
hostname.endsWith('.local') ||
isNonGlobalIpv4(hostname) ||
hostname.includes(':')
)
return null;
url.hash = '';
return url.href;
} catch {
return null;
}
}
/**
* Normalize one Radio Browser station and omit favicons and unsafe streams.
* @param {object} raw Directory record.
* @param {object} [options]
* @param {(value: unknown) => string|null} [options.normalizeUrl] URL admission
* policy, applied to both stream and homepage; returns a safe URL or null.
*/
export function normalizeRadioBrowserStation(
raw,
{ normalizeUrl = publicRadioHttpsUrl } = {},
) {
const id = cleanRadioText(raw?.stationuuid, 40).toLowerCase();
const lat =
raw?.geo_lat === null || raw?.geo_lat === '' ? null : Number(raw?.geo_lat);
const lon =
raw?.geo_long === null || raw?.geo_long === ''
? null
: Number(raw?.geo_long);
const codec = cleanRadioText(raw?.codec, 16).toUpperCase();
const streamUrl = normalizeUrl(raw?.url_resolved || raw?.url);
if (
!RADIO_UUID_RE.test(id) ||
Number(raw?.lastcheckok) !== 1 ||
Number(raw?.hls) === 1 ||
!Number.isFinite(lat) ||
lat < -90 ||
lat > 90 ||
!Number.isFinite(lon) ||
lon < -180 ||
lon > 180 ||
!/^(?:MP3|AAC(?:\+|-LC|-HE)?|HE-AAC)$/i.test(codec) ||
!streamUrl
)
return null;
const name = cleanRadioText(raw?.name, 140);
if (!name) return null;
const tags = String(raw?.tags ?? '')
.split(',')
.map((tag) =>
cleanRadioText(tag, 80)
.toLocaleLowerCase()
.replace(/[_-]+/g, ' ')
.replace(/\s+/g, ' ')
.trim(),
)
.filter(Boolean)
.filter((tag, index, all) => all.indexOf(tag) === index)
.slice(0, 24);
const languages = String(raw?.language ?? '')
.split(',')
.map((language) => cleanRadioText(language, 40))
.filter(Boolean)
.slice(0, 8);
const rawCountryCode = cleanRadioText(raw?.countrycode, 2).toUpperCase();
const normalizedCode = normalizeRadioCountryInput(rawCountryCode);
const normalizedCountry =
normalizedCode.valid && !normalizedCode.empty
? normalizedCode
: normalizeRadioCountryInput(cleanRadioText(raw?.country, 80));
const bitrate = Number(raw?.bitrate);
return {
id,
name,
lat,
lon,
streamUrl,
homepage: normalizeUrl(raw?.homepage),
tags,
languages,
state: cleanRadioText(raw?.state, 80),
country:
normalizedCountry.valid && !normalizedCountry.empty
? normalizedCountry.name
: cleanRadioText(raw?.country, 80),
countryCode: normalizedCountry.valid ? normalizedCountry.code : '',
metadataTrust: 'untrusted-community',
codec,
bitrate:
Number.isInteger(bitrate) && bitrate >= 8 && bitrate <= 1024
? bitrate
: null,
clickCount: Math.max(0, Math.min(10_000_000, Number(raw?.clickcount) || 0)),
};
}
export function publicRadioStation(station) {
return {
id: station.id,
name: station.name,
lat: station.lat,
lon: station.lon,
streamUrl: station.streamUrl,
homepage: station.homepage,
tags: station.tags,
languages: station.languages,
state: station.state,
country: station.country,
countryCode: station.countryCode,
metadataTrust: station.metadataTrust,
codec: station.codec,
bitrate: station.bitrate,
};
}