From 32b0304c2ce1bdb71def174cf30ecd2501ec967a Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:23:10 +0100 Subject: [PATCH 1/5] fix(radio): normalize station tags and retain valid mixed rows --- docs/CURRENT-STATE.md | 7 +++++++ server/providers/radio/catalog.js | 17 ++++------------- server/providers/radio/stations.js | 1 + src/data/radio.test.mjs | 18 ++++++++++++++++++ src/data/radioProxy.test.mjs | 8 ++++++++ src/layers/radio/ingestion.js | 12 ++++++------ src/layers/radio/presentation.js | 1 + src/layers/radio/state.js | 1 + src/sources/radioBrowser.js | 18 +++++++++++------- 9 files changed, 57 insertions(+), 26 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..278c4e0bd 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,12 @@ # God's Eye View Current State +Radio tags are lowercased and whitespace-normalized before the final 80-character +cap, including requested-tag coverage matching. Malformed station rows are skipped +individually; valid rows retain their accepted catalog generation. The UI snapshot +reports a bounded `rejectedStationCount` and degraded coverage when rows were +skipped. Malformed envelope/freshness/generation metadata and all-invalid responses +still preserve the previous usable catalog. + Shared distance checks use `src/geoDistance.js` with `{ lat, lon }` points and the IUGG mean spherical Earth radius, 6371.0088 km. `greatCircleKm` is canonical; `greatCircleMeters` is its exact metre conversion. Missing, diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js index 36a3eb730..7eed9f01a 100644 --- a/server/providers/radio/catalog.js +++ b/server/providers/radio/catalog.js @@ -5,7 +5,7 @@ import { readResponseTextCapped } from '../common/http.js'; import { normalizeRadioBrowserStation, publicRadioStation, - cleanRadioText, + normalizeRadioTag, } from './stations.js'; import { radioMirrorOrigin, @@ -177,8 +177,8 @@ export function createRadioProxyMiddleware({ const rows = await fetchPath(`/json/stations/search?${params}`); if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - if ( - !rows.every( + const stations = rows + .filter( (row) => row && typeof row === 'object' && @@ -188,18 +188,9 @@ export function createRadioProxyMiddleware({ (typeof row.url_resolved === 'string' || typeof row.url === 'string'), ) - ) - throw new Error( - 'Radio Browser catalog contained a malformed station row', - ); - const stations = rows .map(normalizeRadioBrowserStation) .filter(Boolean); - const requestedTag = cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(); + const requestedTag = normalizeRadioTag(tag); const requestedTagCovered = !requestedTag || stations.some((station) => diff --git a/server/providers/radio/stations.js b/server/providers/radio/stations.js index b97b2d3c6..49279e2d2 100644 --- a/server/providers/radio/stations.js +++ b/server/providers/radio/stations.js @@ -1,5 +1,6 @@ export { cleanRadioText, + normalizeRadioTag, isNonGlobalIpv4, publicRadioHttpsUrl, normalizeRadioBrowserStation, diff --git a/src/data/radio.test.mjs b/src/data/radio.test.mjs index 58a4101a1..6ab8561c6 100644 --- a/src/data/radio.test.mjs +++ b/src/data/radio.test.mjs @@ -712,6 +712,24 @@ test('unusable directory responses preserve warm client state atomically', async assert.equal(getRadioUIState().stale, true); assert.equal(getRadioUIState().degraded, true); } + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement, { id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 2, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioUIState().stationCount, 1); + assert.equal(getRadioUIState().rejectedStationCount, 1); + assert.equal(getRadioUIState().degraded, true); + assert.equal(getRadioUIState().stale, false); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioAcceptedCatalogSnapshot().stations[0].id, replacement.id); + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [{ id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 3, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioUIState().stale, true); } finally { globalThis.fetch = originalFetch; radioLayer.destroy(); diff --git a/src/data/radioProxy.test.mjs b/src/data/radioProxy.test.mjs index 60463205b..d1c4d2660 100644 --- a/src/data/radioProxy.test.mjs +++ b/src/data/radioProxy.test.mjs @@ -533,3 +533,11 @@ test('a mirror answering an invalid reason phrase rejects instead of crashing', const crashed = mockMirrorStatus(t, 200, 'OK\x7f'); await assert.rejects(Promise.race([fetchMirror(), crashed]), TypeError); }); + + +test('case expansion is capped after tag normalization in the portable and server station path', () => { + const row = normalizeRadioBrowserStation(station({tags: 'İ'.repeat(80) + ', NEWS__RADIO ,news-radio'})); + assert.equal(row.tags[0].length, 80); + assert.equal(row.tags[0], 'i\u0307'.repeat(40)); + assert.deepEqual(row.tags.slice(1), ['news radio']); +}); diff --git a/src/layers/radio/ingestion.js b/src/layers/radio/ingestion.js index 869b8d403..5a0ccfa1d 100644 --- a/src/layers/radio/ingestion.js +++ b/src/layers/radio/ingestion.js @@ -70,11 +70,6 @@ export function createIngestion({ ); if (!acceptedRows.length) throw new Error('Radio directory returned no usable stations'); - if (acceptedRows.length !== rows.length) { - throw new Error( - 'Radio directory response contained malformed stations', - ); - } const acceptedGeneration = body.acceptedGeneration; if ( acceptedGeneration !== null && @@ -147,7 +142,12 @@ export function createIngestion({ } if (!repeatingAcceptedGeneration) layerState._updatedAt = updatedAt; } - layerState._degraded = body.degraded; + layerState._rejectedStationCount = Math.min( + 1000000, + rows.length - acceptedRows.length, + ); + layerState._degraded = + body.degraded || layerState._rejectedStationCount > 0; layerState._stale = body.stale; layerState._error = preservingWarmCatalog ? ERROR_REFRESH_DEGRADED diff --git a/src/layers/radio/presentation.js b/src/layers/radio/presentation.js index 5d6763ca4..c30889018 100644 --- a/src/layers/radio/presentation.js +++ b/src/layers/radio/presentation.js @@ -20,6 +20,7 @@ export function createPresentation({ loading: layerState._loading, stale: layerState._stale, degraded: layerState._degraded, + rejectedStationCount: layerState._rejectedStationCount, error: layerState._error, updatedAt: layerState._updatedAt, filter: layerState._filter, diff --git a/src/layers/radio/state.js b/src/layers/radio/state.js index dca461e03..918ebb989 100644 --- a/src/layers/radio/state.js +++ b/src/layers/radio/state.js @@ -25,6 +25,7 @@ export function createState({ services }) { layerState._stale = false; layerState._degraded = false; + layerState._rejectedStationCount = 0; layerState._error = null; diff --git a/src/sources/radioBrowser.js b/src/sources/radioBrowser.js index 8511b6beb..8329f6257 100644 --- a/src/sources/radioBrowser.js +++ b/src/sources/radioBrowser.js @@ -11,6 +11,16 @@ export function cleanRadioText(value, maxLength) { .trim(); } +/** Normalize a station/filter tag and cap its final, case-folded representation. */ +export function normalizeRadioTag(value) { + return cleanRadioText( + String(value ?? '') + .toLowerCase() + .replace(/[_-]+/g, ' '), + 80, + ); +} + export function isNonGlobalIpv4(hostname) { const pieces = hostname.split('.'); if (pieces.length !== 4 || pieces.some((piece) => !/^\d{1,3}$/.test(piece))) @@ -99,13 +109,7 @@ export function normalizeRadioBrowserStation( if (!name) return null; const tags = String(raw?.tags ?? '') .split(',') - .map((tag) => - cleanRadioText(tag, 80) - .toLocaleLowerCase() - .replace(/[_-]+/g, ' ') - .replace(/\s+/g, ' ') - .trim(), - ) + .map(normalizeRadioTag) .filter(Boolean) .filter((tag, index, all) => all.indexOf(tag) === index) .slice(0, 24); From 34507f606ab99ca2dee5ead78a133045da4315b8 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 00:23:34 +0100 Subject: [PATCH 2/5] docs: record radio fix --- changelog.d/1268-radio-issue-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1268-radio-issue-fix.md diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md new file mode 100644 index 000000000..de6b0bcae --- /dev/null +++ b/changelog.d/1268-radio-issue-fix.md @@ -0,0 +1 @@ +- Normalize station tags and retain valid mixed rows (asturksever, #1268). From 9fd434438b052f6cd5b4c09a30ffb85244a762d5 Mon Sep 17 00:00:00 2001 From: Said Date: Tue, 20 Oct 2026 13:50:42 +0100 Subject: [PATCH 3/5] test: target mounted voice input in radio takeover gate --- changelog.d/1268-radio-issue-fix.md | 1 + scripts/qa-radio.mjs | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md index de6b0bcae..7a5be28b7 100644 --- a/changelog.d/1268-radio-issue-fix.md +++ b/changelog.d/1268-radio-issue-fix.md @@ -1 +1,2 @@ - Normalize station tags and retain valid mixed rows (asturksever, #1268). +- Correct Radio QA push-to-talk setup to target the mounted voice controller (asturksever, #1268). diff --git a/scripts/qa-radio.mjs b/scripts/qa-radio.mjs index 648075bd2..cfc601ca3 100644 --- a/scripts/qa-radio.mjs +++ b/scripts/qa-radio.mjs @@ -2761,13 +2761,16 @@ async function main() { const beforeSpace = radio.getUIState().audioState; // Simulate an already-started hold-Space session; click-started open mic // deliberately ignores Space takeover, and short taps never claim voice. - const priorPushToTalkMode = voice.pushToTalkMode; - voice.pushToTalkMode = true; + // The provider facade forwards reads, not writes. Set the mounted + // controller's input owner; assigning on the facade only shadows it. + const controller = voice.session.adapter.controller; + const priorPushToTalkMode = controller.pushToTalkMode; + controller.pushToTalkMode = true; document.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, cancelable: true, code: 'Space', key: ' ' })); await new Promise((resolve) => setTimeout(resolve, 700)); const afterSpace = radio.getUIState().audioState; document.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, cancelable: true, code: 'Space', key: ' ' })); - voice.pushToTalkMode = priorPushToTalkMode; + controller.pushToTalkMode = priorPushToTalkMode; await radio.play(); voice.setVoiceSpeaker('ai'); return { beforeSpace, afterSpace, afterAi: radio.getUIState().audioState }; From 82190d531a99c507685b2d10e0330bb7f7f7a5b4 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:39:56 +0100 Subject: [PATCH 4/5] Reset the Radio rejection counter on destroy Clear _rejectedStationCount alongside _degraded so a destroyed and re-initialized layer does not report the previous catalog's count. Co-Authored-By: Claude Opus 5.5 --- src/layers/radio/lifecycle.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/layers/radio/lifecycle.js b/src/layers/radio/lifecycle.js index 414cfa5c0..45d66892e 100644 --- a/src/layers/radio/lifecycle.js +++ b/src/layers/radio/lifecycle.js @@ -108,6 +108,7 @@ export function createLifecycle({ layerState._loading = false; layerState._stale = false; layerState._degraded = false; + layerState._rejectedStationCount = 0; layerState._error = null; layerState._updatedAt = null; layerState._managerLifecyclePresentation = null; From f93982f55e4b15885def69e2e69273adc627abd2 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 23 Oct 2026 12:39:56 +0100 Subject: [PATCH 5/5] Carry server structural rejections into Radio diagnostics The proxy now skips malformed upstream rows individually but counts them, capped at 10,000, in the catalog metadata and the /stations envelope (rejectedStationCount). The count never fails health policy, so an otherwise healthy generation is still accepted. The client validates the field, adds its own rejected rows after admission, and reports degraded coverage when either side skipped rows. Adds an end-to-end proxy-to-client test and aligns DATA_SOURCES.md and CURRENT-STATE.md. Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 2 +- changelog.d/1268-radio-issue-fix.md | 2 +- docs/CURRENT-STATE.md | 14 +++++-- server/providers/radio/catalog.js | 39 +++++++++++++------ server/providers/radio/constants.js | 3 ++ src/data/radio.test.mjs | 38 +++++++++++++++++- src/data/radioProxy.test.mjs | 60 ++++++++++++++++++++++++++++- src/layers/radio/ingestion.js | 20 +++++++++- src/layers/radio/policy.js | 4 ++ 9 files changed, 159 insertions(+), 23 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 6d2ee498b..3557c4137 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -177,7 +177,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All sixteen registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Hamilton, Guelph, Burlington and YRT publish route ids that are not the number on the vehicle, so those entries carry a route-name table taken from each operator’s own published GTFS; any id missing from a table renders unchanged rather than blank. A label is never invented: YRT runs a route `9101` that appears nowhere in its published routes, so it stays as the feed sends it. Burlington’s is the one that cannot be shortcut: `351` is route 1 while `3510` is route 10, so the table is a real join rather than a trimmed prefix. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Barrie and YRT/Viva each invite acceptance of their licence through a form, and each licence also states that using the data is itself acceptance; this deployment relies on that clause, which is recorded in both entries. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. Ottawa’s OC Transpo falls to the same keyless admission rule as GO Transit — its realtime feed requires an API key. London Transit and Thunder Bay Transit publish vehicle positions over plain HTTP with no HTTPS endpoint available. Every other Ontario system — Brampton, Oakville, Niagara, Milton, Windsor, Greater Sudbury, UP Express and a dozen smaller operators — publishes static GTFS schedules only, with no realtime vehicle positions to register. Between the nine included and these exclusions, every Ontario realtime vehicle-position feed catalogued in the Mobility Database is accounted for. -- **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. +- **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and non-array or empty successful payloads are rejected atomically. Individual structurally malformed rows are skipped rather than failing the whole query; the response carries their bounded `rejectedStationCount` (at most 10,000), which the client adds to its own rejected rows and reports as degraded coverage without discarding the accepted generation. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. - **Re:Earth Terrain.** Keyless (no API key). Used two ways: (1) `src/mapStackController.js` swaps in a `Cesium.CesiumTerrainProvider` pointed at Re:Earth's `cesium-mesh/ellipsoid` quantized-mesh endpoint for globe stacks without a Cesium ion token (e.g. OSM), replacing a flat `EllipsoidTerrainProvider`; falls back to the flat provider if the endpoint can't be reached. (2) The server-side `/api/terrain/heights` proxy (disk-cached, serve-stale) resolves per-point ellipsoidal ground height for entity placement. Both are best-effort with a keyless-safe fallback (bundled EGM96 geoid math) if Re:Earth is unreachable. diff --git a/changelog.d/1268-radio-issue-fix.md b/changelog.d/1268-radio-issue-fix.md index 7a5be28b7..ab68243d6 100644 --- a/changelog.d/1268-radio-issue-fix.md +++ b/changelog.d/1268-radio-issue-fix.md @@ -1,2 +1,2 @@ -- Normalize station tags and retain valid mixed rows (asturksever, #1268). +- Normalize station tags and retain valid mixed rows; skipped malformed rows still report degraded coverage (asturksever, #1268). - Correct Radio QA push-to-talk setup to target the mounted voice controller (asturksever, #1268). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 12edc4f97..0001b850f 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -11,10 +11,16 @@ search keeps a successful geocode, with no search or view-based region recovery. Radio tags are lowercased and whitespace-normalized before the final 80-character cap, including requested-tag coverage matching. Malformed station rows are skipped -individually; valid rows retain their accepted catalog generation. The UI snapshot -reports a bounded `rejectedStationCount` and degraded coverage when rows were -skipped. Malformed envelope/freshness/generation metadata and all-invalid responses -still preserve the previous usable catalog. +individually on the server and the client; valid rows retain their accepted +catalog generation. The server's `/stations` envelope carries +`rejectedStationCount`, the structurally malformed upstream rows it skipped +(capped at 10,000; policy-filtered rows such as offline or non-HTTPS streams are +not counted). It is diagnostic only and never fails catalog health. The client +adds its own rejected rows after admission, so the UI snapshot reports the +combined, capped `rejectedStationCount` and degraded coverage when either side +skipped rows. Destroy resets the count. Malformed envelope, freshness, generation +or rejection-count metadata and all-invalid responses still preserve the previous +usable catalog. Terrain height requests accept at most 64 points, matching the browser service's existing batch size and one bounded upstream chunk. Larger requests return HTTP diff --git a/server/providers/radio/catalog.js b/server/providers/radio/catalog.js index de66ccbfb..30e6ed4ec 100644 --- a/server/providers/radio/catalog.js +++ b/server/providers/radio/catalog.js @@ -25,6 +25,7 @@ import { RADIO_USER_AGENT, RADIO_UUID_RE, RADIO_FALLBACK_MIRRORS, + RADIO_REJECTED_STATION_COUNT_MAX, } from './constants.js'; async function mapRadioConcurrent(values, concurrency, mapper) { const results = new Array(values.length); @@ -177,17 +178,22 @@ export function createRadioProxyMiddleware({ const rows = await fetchPath(`/json/stations/search?${params}`); if (!Array.isArray(rows)) throw new Error('Radio Browser catalog payload was not an array'); - const stations = rows - .filter( - (row) => - row && - typeof row === 'object' && - !Array.isArray(row) && - typeof row.stationuuid === 'string' && - typeof row.name === 'string' && - (typeof row.url_resolved === 'string' || - typeof row.url === 'string'), - ) + const structuralRows = rows.filter( + (row) => + row && + typeof row === 'object' && + !Array.isArray(row) && + typeof row.stationuuid === 'string' && + typeof row.name === 'string' && + (typeof row.url_resolved === 'string' || + typeof row.url === 'string'), + ); + // Malformed rows are skipped individually, but counted so the + // catalog keeps a diagnostic for upstream schema drift. Policy + // rejections in normalization (offline, non-HTTPS, codec) are normal + // filtering and are not counted. + const rejectedRows = rows.length - structuralRows.length; + const stations = structuralRows .map(normalizeRadioBrowserStation) .filter(Boolean); const requestedTag = normalizeRadioTag(tag); @@ -206,9 +212,10 @@ export function createRadioProxyMiddleware({ // also contain an accepted station tagged for the requested category. succeeded: stations.length > 0 && requestedTagCovered, stations, + rejectedRows, }; } catch { - return { succeeded: false, stations: [] }; + return { succeeded: false, stations: [], rejectedRows: 0 }; } }, ); @@ -248,6 +255,10 @@ export function createRadioProxyMiddleware({ if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS) healthReasons.push('station-coverage-below-policy'); const degraded = healthReasons.length > 0; + const rejectedStationCount = Math.min( + RADIO_REJECTED_STATION_COUNT_MAX, + outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0), + ); const coverage = { successfulQueries, totalQueries: queries.length, @@ -262,6 +273,9 @@ export function createRadioProxyMiddleware({ degraded, degradedReason: degraded ? healthReasons.join(',') : null, coverage, + // Diagnostic only: it never fails health policy, so a usable generation + // with a few skipped rows is still admitted. The client reports it. + rejectedStationCount, }; if (degraded && catalogCache) { const error = new Error( @@ -351,6 +365,7 @@ export function createRadioProxyMiddleware({ coverage: catalog.coverage || null, acceptedGeneration: catalog.acceptedGeneration ?? null, catalogInstance, + rejectedStationCount: catalog.rejectedStationCount ?? 0, }); } catch (error) { sendJson(res, 503, { diff --git a/server/providers/radio/constants.js b/server/providers/radio/constants.js index 0fee93ac6..b0e24babc 100644 --- a/server/providers/radio/constants.js +++ b/server/providers/radio/constants.js @@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil( export const RADIO_USER_AGENT = 'GodsEyeView/1.0 (Radio Browser directory client)'; export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js'; +// Upper bound for the structurally malformed upstream rows reported in the +// /stations envelope. The client layer validates against the same bound. +export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000; export const RADIO_FALLBACK_MIRRORS = Object.freeze([ 'https://de1.api.radio-browser.info', 'https://de2.api.radio-browser.info', diff --git a/src/data/radio.test.mjs b/src/data/radio.test.mjs index 8d24adc19..99963df16 100644 --- a/src/data/radio.test.mjs +++ b/src/data/radio.test.mjs @@ -723,13 +723,47 @@ test('unusable directory responses preserve warm client state atomically', async assert.equal(getRadioUIState().stale, false); assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); assert.equal(getRadioAcceptedCatalogSnapshot().stations[0].id, replacement.id); + // Server-side structural rejections are combined with client rejections + // after admission; the healthy generation is still accepted. globalThis.fetch = async () => new Response(JSON.stringify({ - stations: [{ id: 'malformed' }], stale: false, degraded: false, + stations: [replacement, { id: 'malformed' }], stale: false, degraded: false, acceptedGeneration: 3, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount: 2, })); await radioLayer.update(); - assert.equal(getRadioAcceptedCatalogSnapshot().generation, 2); + assert.equal(getRadioUIState().rejectedStationCount, 3); + assert.equal(getRadioUIState().degraded, true); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 3); + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement], stale: false, degraded: false, + acceptedGeneration: 4, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount: 10_000, + })); + await radioLayer.update(); + assert.equal(getRadioUIState().rejectedStationCount, 10_000); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); + for (const rejectedStationCount of [-1, 1.5, '2', null, 10_001]) { + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [replacement], stale: false, degraded: false, + acceptedGeneration: 5, catalogInstance: 'qa-instance-a', updatedAt: now, + rejectedStationCount, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); + assert.equal(getRadioUIState().stale, true); + } + globalThis.fetch = async () => new Response(JSON.stringify({ + stations: [{ id: 'malformed' }], stale: false, degraded: false, + acceptedGeneration: 5, catalogInstance: 'qa-instance-a', updatedAt: now, + })); + await radioLayer.update(); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 4); assert.equal(getRadioUIState().stale, true); + assert.equal(getRadioUIState().rejectedStationCount, 10_000); + radioLayer.destroy(); + radioLayer.init(viewer); + assert.equal(getRadioUIState().rejectedStationCount, 0); + assert.equal(getRadioUIState().degraded, false); } finally { globalThis.fetch = originalFetch; radioLayer.destroy(); diff --git a/src/data/radioProxy.test.mjs b/src/data/radioProxy.test.mjs index d1c4d2660..e9df10678 100644 --- a/src/data/radioProxy.test.mjs +++ b/src/data/radioProxy.test.mjs @@ -11,7 +11,14 @@ import { publicRadioStation, publicRadioHttpsUrl, } from '../../vite.config.js'; -import { rankRadioStationsForRequest } from './radio.js'; +import { + getRadioAcceptedCatalogSnapshot, + getRadioUIState, + radioLayer, + rankRadioStationsForRequest, +} from './radio.js'; +import { RADIO_REJECTED_STATION_COUNT_MAX as SERVER_REJECTED_MAX } from '../../server/providers/radio/constants.js'; +import { RADIO_REJECTED_STATION_COUNT_MAX as CLIENT_REJECTED_MAX } from '../layers/radio/policy.js'; const UUID = '12345678-1234-4234-8234-123456789abc'; const publicLookup = async () => [{ address: '93.184.216.34', family: 4 }]; @@ -541,3 +548,54 @@ test('case expansion is capped after tag normalization in the portable and serve assert.equal(row.tags[0], 'i\u0307'.repeat(40)); assert.deepEqual(row.tags.slice(1), ['news radio']); }); + +test('server structural rejections reach the client diagnostic without discarding a healthy generation', async () => { + const originalFetch = globalThis.fetch; + const middleware = createProxy({ + now: () => Date.now(), + fetchImpl: async (url) => { + if (String(url).includes('/json/servers')) return responseJson([{ name: 'de1.api.radio-browser.info' }]); + const rows = healthyRowsForQuery(url); + if (queryTag(url)) return responseJson(rows); + // 400 valid broad rows, one row missing its required name, and one + // schema-valid row that normalization filters by policy (not counted). + const { name: _name, ...nameless } = station({ stationuuid: '50000000-0000-4000-8000-000000000001' }); + return responseJson([...rows, nameless, station({ stationuuid: '50000000-0000-4000-8000-000000000002', lastcheckok: 0 })]); + }, + }); + const response = await invoke(middleware, '/stations'); + assert.equal(response.status, 200); + const body = JSON.parse(response.body); + assert.equal(body.degraded, false); + assert.equal(body.degradedReason, null); + assert.equal(body.acceptedGeneration, 1); + assert.equal(body.rejectedStationCount, 1); + assert.equal(SERVER_REJECTED_MAX, CLIENT_REJECTED_MAX); + + const viewer = { + camera: { positionWC: { x: 7_000_000, y: 0, z: 0 } }, + scene: { canvas: { disableRootEvents: true, onwheel: null, addEventListener() {}, removeEventListener() {} } }, + dataSources: { add() {}, remove() {} }, + entities: { add(entity) { return entity; }, remove() {} }, + }; + radioLayer.destroy(); + try { + globalThis.fetch = async () => new Response(response.body, { + status: response.status, + headers: { 'Content-Type': 'application/json' }, + }); + radioLayer.init(viewer); + radioLayer.enable(); + await radioLayer.update(); + const state = getRadioUIState(); + assert.equal(state.stationCount, body.stations.length); + assert.equal(state.rejectedStationCount, 1); + assert.equal(state.degraded, true); + assert.equal(state.stale, false); + assert.equal(getRadioAcceptedCatalogSnapshot().generation, 1); + assert.equal(getRadioAcceptedCatalogSnapshot().instance, body.catalogInstance); + } finally { + globalThis.fetch = originalFetch; + radioLayer.destroy(); + } +}); diff --git a/src/layers/radio/ingestion.js b/src/layers/radio/ingestion.js index 5a0ccfa1d..79f348469 100644 --- a/src/layers/radio/ingestion.js +++ b/src/layers/radio/ingestion.js @@ -2,6 +2,7 @@ import { msg } from '../../i18n/index.js'; import { RADIO_DIRECTORY_STALE_MS, RADIO_DIRECTORY_FUTURE_SKEW_MS, + RADIO_REJECTED_STATION_COUNT_MAX, } from './policy.js'; // Directory errors stay English in layer state and stats; the layer row and @@ -64,6 +65,18 @@ export function createIngestion({ typeof body.degraded !== 'boolean' ) throw new Error('Radio directory freshness metadata was malformed'); + // Rows the server skipped as structurally malformed. Older envelopes + // omit the field; a present value must be a bounded count. + const serverRejectedStationCount = + body.rejectedStationCount === undefined + ? 0 + : body.rejectedStationCount; + if ( + !Number.isSafeInteger(serverRejectedStationCount) || + serverRejectedStationCount < 0 || + serverRejectedStationCount > RADIO_REJECTED_STATION_COUNT_MAX + ) + throw new Error('Radio directory diagnostic metadata was malformed'); const rows = body.stations; const acceptedRows = rows.filter( parts.catalogModel.isValidRadioDirectoryStation, @@ -142,9 +155,12 @@ export function createIngestion({ } if (!repeatingAcceptedGeneration) layerState._updatedAt = updatedAt; } + // Combine server-side structural rejections with rows this client + // refused. The count is diagnostic: it marks coverage degraded but does + // not undo the generation admitted above. layerState._rejectedStationCount = Math.min( - 1000000, - rows.length - acceptedRows.length, + RADIO_REJECTED_STATION_COUNT_MAX, + serverRejectedStationCount + rows.length - acceptedRows.length, ); layerState._degraded = body.degraded || layerState._rejectedStationCount > 0; diff --git a/src/layers/radio/policy.js b/src/layers/radio/policy.js index 0d4ad174a..e0438c780 100644 --- a/src/layers/radio/policy.js +++ b/src/layers/radio/policy.js @@ -24,6 +24,10 @@ export const RADIO_DIRECTORY_STALE_MS = 7 * 24 * 60 * 60 * 1000; export const RADIO_DIRECTORY_FUTURE_SKEW_MS = 5 * 60 * 1000; +// Bound for the server's `rejectedStationCount` envelope field and the +// combined client counter; matches server/providers/radio/constants.js. +export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000; + export const RADIO_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;