From a1786f162ccb753758724409b613ffdc22dd644f Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Tue, 13 Oct 2026 21:49:25 +0200 Subject: [PATCH] fix: recover GNSS feed startup and honor retry cooldowns --- CHANGELOG.md | 4 + docs/CURRENT-STATE.md | 2 +- scripts/qa-gnss.mjs | 96 ++++++++++++++++++++- src/layers/gnss/index.js | 113 +++++++++++++++++++------ src/layers/gnss/ownership.test.mjs | 129 ++++++++++++++++++++++++++++- src/layers/gnss/source.js | 30 ++++++- src/layers/gnss/source.test.mjs | 35 ++++++++ 7 files changed, 376 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d81467a25..8f7adf929 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -189,6 +189,10 @@ are navigation-integrity anomalies; suspected GNSS interference is a secondary, unvalidated interpretation. The layer and the proxy expose machine-readable provenance (`provenance`, `classifier: gev-nic-nacp-v1`). + Transient feed failures no longer reject GNSS activation: the layer stays + enabled, explains unavailable/degraded feed health and retries automatically. + Provider Retry-After deadlines survive movement and toggling; other failures + back off for 30/60/120 seconds. Disable/destroy cancels retry work. Automatically refreshes after camera pan/zoom without toggling, coalesces nearby views, and uses the Earth point under the screen centre at globe scale. Keyless regional sampling wherever adsb.lol has receivers; the row labels diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d62e733e1..f96d4a805 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3101,7 +3101,7 @@ its criteria cannot be silently ignored. | GNSS Integrity ๐Ÿ“ก (`gnss-interference`) | adsb.lol point snapshot, low navigation accuracy binned per 0.5ยฐ cell (GEV NIC/NACp threshold + gpsjam.org formula and bands, keyless, ODbL) | `src/layers/gnss/` via `src/app/layers/gnss.js` | `/api/gnss-integrity` | camera-driven (600 ms debounce, minimum 2.5 s between reads) + 60s poll (server cache 60 s; 30 min rolling window) | | Fire Perimeters ๐Ÿ”ฅ | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | -GNSS Integrity automatically refreshes after camera pan/zoom (600 ms debounce, minimum 2.5 s between reads, shared whole-degree cache keys) through the manager refresh lane, so counts and loading/error state update without toggling. The anchor is the Earth point under the screen centre, including at globe scale where the visible rectangle can cover the entire world. It fetches a capped 250-nautical-mile-radius adsb.lol snapshot around that anchor, keeps each aircraft once per cell over 30 minutes as the camera pans, and draws low/medium/high ground rectangles only for cells with at least 3 aircraft. The row legend counts cells per band and visibly labels the regional 250 nm sampling radius. Zooming out does not turn this public point feed into worldwide coverage; blank areas can lie outside the sampled region or have fewer than three qualifying aircraft. The method is a GEV threshold plus gpsjam.org aggregation, not the gpsjam method: the per-aircraft rule (NIC < 7 or NACp < 8, borrowed from the US 14 CFR 91.227 ADS-B Out minima, or readsb's GPS-loss flag) and the 30-minute visited-view window are GEV-defined and unvalidated; only the cell formula and 2% / 10% bands come from gpsjam.org. `layer.provenance` (also on `getStats()`) records each part, and the proxy tags its rows with `classifier: gev-nic-nacp-v1`. Cells carry `state: navigation-integrity-anomaly` (medium/high) or `nominal`; `interpretation: suspected-gnss-interference` is a secondary reading, not a detection or a located emitter. Only DO-260A/B (ADS-B versions 1 and 2) direct 1090ES positions with a known `seen_pos` of 0โ€“60 s count. Presence and degraded evidence age separately: a degraded report keeps the aircraft degraded in its cell for 30 minutes after that report, however many healthy reports follow, and an aircraft unheard for 30 minutes starts fresh. Rows are dated by the snapshot's observation time, which the proxy keeps on cached and stale answers and reports as `ageMs` on its own clock (so browser clock skew does not move the window); a replayed snapshot never restarts the window; the proxy stops serving a stale snapshot once it is older than 30 minutes, the source refuses an undated one, and cells age out on schedule during fetch failures or while there is no view anchor. +GNSS display activation stays enabled through transient feed failures, including the first read. The row reports unavailable/degraded source health, a specific HTTP cause and the next automatic retry time; successful recovery clears the warning. Retry-After is honored in full for provider cooldowns, with 30/60/120 s bounded backoff otherwise. Camera movement, polling and toggling cannot bypass the deadline; disable/destroy cancels retries. Genuine non-retryable requests and renderer failures remain lifecycle failures. GNSS Integrity automatically refreshes after camera pan/zoom (600 ms debounce, minimum 2.5 s between reads, shared whole-degree cache keys) through the manager refresh lane, so counts and loading/error state update without toggling. The anchor is the Earth point under the screen centre, including at globe scale where the visible rectangle can cover the entire world. It fetches a capped 250-nautical-mile-radius adsb.lol snapshot around that anchor, keeps each aircraft once per cell over 30 minutes as the camera pans, and draws low/medium/high ground rectangles only for cells with at least 3 aircraft. The row legend counts cells per band and visibly labels the regional 250 nm sampling radius. Zooming out does not turn this public point feed into worldwide coverage; blank areas can lie outside the sampled region or have fewer than three qualifying aircraft. The method is a GEV threshold plus gpsjam.org aggregation, not the gpsjam method: the per-aircraft rule (NIC < 7 or NACp < 8, borrowed from the US 14 CFR 91.227 ADS-B Out minima, or readsb's GPS-loss flag) and the 30-minute visited-view window are GEV-defined and unvalidated; only the cell formula and 2% / 10% bands come from gpsjam.org. `layer.provenance` (also on `getStats()`) records each part, and the proxy tags its rows with `classifier: gev-nic-nacp-v1`. Cells carry `state: navigation-integrity-anomaly` (medium/high) or `nominal`; `interpretation: suspected-gnss-interference` is a secondary reading, not a detection or a located emitter. Only DO-260A/B (ADS-B versions 1 and 2) direct 1090ES positions with a known `seen_pos` of 0โ€“60 s count. Presence and degraded evidence age separately: a degraded report keeps the aircraft degraded in its cell for 30 minutes after that report, however many healthy reports follow, and an aircraft unheard for 30 minutes starts fresh. Rows are dated by the snapshot's observation time, which the proxy keeps on cached and stale answers and reports as `ageMs` on its own clock (so browser clock skew does not move the window); a replayed snapshot never restarts the window; the proxy stops serving a stale snapshot once it is older than 30 minutes, the source refuses an undated one, and cells age out on schedule during fetch failures or while there is no view anchor. Fire Perimeters uses capped, timed server reads with stale-on-error caching and a per-client limit. Unchanged snapshots retain geometry; link checks cancel on disable or selection change, and the row legend shows reported containment. diff --git a/scripts/qa-gnss.mjs b/scripts/qa-gnss.mjs index bebed56aa..bf265b9e4 100644 --- a/scripts/qa-gnss.mjs +++ b/scripts/qa-gnss.mjs @@ -239,23 +239,37 @@ function fixturePayload() { } /** Answer /api/gnss-integrity in-page from `fixturePayload()`. */ -async function stubRoute(page, { ageMs = 0 } = {}) { +async function stubRoute(page, { ageMs = 0, initialFailures = 0, stale } = {}) { await page.evaluateOnNewDocument( - (payload, age) => { + (payload, age, failures, staleOverride) => { window.__gnssQaAgeMs = age; window.__gnssQaRequests = []; + window.__gnssQaFailures = failures; const realFetch = window.fetch.bind(window); window.__gnssQaMode = 'fixture'; window.fetch = (input, init) => { const url = typeof input === 'string' ? input : input?.url || ''; if (!url.includes('/api/gnss-integrity')) return realFetch(input, init); window.__gnssQaRequests.push(url); + if (window.__gnssQaFailures > 0) { + window.__gnssQaFailures--; + return Promise.resolve( + Response.json( + { error: 'rate_limited' }, + { + status: 429, + headers: { 'Retry-After': '5' }, + }, + ), + ); + } const ok = window.__gnssQaMode === 'fixture'; const body = ok ? { ...payload, fetchedAt: Date.now() - (window.__gnssQaAgeMs || 0), ageMs: window.__gnssQaAgeMs || 0, + stale: staleOverride ?? payload.stale, } : { error: 'gnss_integrity_unavailable' }; return Promise.resolve( @@ -268,6 +282,8 @@ async function stubRoute(page, { ageMs = 0 } = {}) { }, fixturePayload(), ageMs, + initialFailures, + stale, ); } @@ -702,6 +718,82 @@ async function main() { await phone.screenshot({ path: path.join(SHOTS_DIR, 'gnss-phone.png') }); await phone.close(); + console.log( + '\nSTARTUP โ€” initial rate limit stays enabled and recovers automatically...', + ); + const retry = await browser.newPage(); + await retry.setViewport({ width: 1440, height: 900 }); + retry.on('pageerror', (e) => errors.push(e.message)); + await stubRoute(retry, { initialFailures: 1, stale: false }); + await boot(retry, { lon: 18.6, lat: 54.6, height: 450_000 }); + await showLayerRow(retry); + await retry.evaluate((id) => { + window.__gnssQaFailedStarts = 0; + window.__godsEyeView.dataManager.subscribe((change) => { + if (change.layerId === id && change.type === 'visibility-failed') + window.__gnssQaFailedStarts++; + }); + }, LAYER_ID); + const initial = await enableAndSettle(retry); + const limited = await retry.evaluate((id) => { + const dm = window.__godsEyeView.dataManager; + const row = document.querySelector(`[data-layer-id="${id}"]`); + return { + enabled: dm.isEnabled(id), + failedStarts: window.__gnssQaFailedStarts, + label: row.querySelector('.data-toggle-btn').textContent, + info: row.querySelector('.data-toggle-controls-info').textContent, + }; + }, LAYER_ID); + record( + 'STARTUP: first 429 keeps the layer enabled and shows automatic retry timing', + limited.enabled && + limited.failedStarts === 0 && + limited.label === 'UNAVAILABLE' && + initial.stats.count === 0 && + initial.stats.retryInSec > 0 && + /rate limited.*Retrying automatically at/.test(limited.info), + JSON.stringify(limited), + ); + await retry.screenshot({ + path: path.join(SHOTS_DIR, 'gnss-startup-retrying.png'), + }); + await retry.waitForFunction( + (id) => { + const layer = window.__godsEyeView.dataManager.layers.get(id); + return ( + layer.enabled && + layer.module.getStats().error === null && + layer.module.getStats().count === 4 + ); + }, + { timeout: 20000 }, + LAYER_ID, + ); + const recovered = await retry.evaluate( + (id) => ({ + requests: window.__gnssQaRequests.length, + enabled: window.__godsEyeView.dataManager.isEnabled(id), + label: document.querySelector( + `[data-layer-id="${id}"] .data-toggle-btn`, + ).textContent, + info: document.querySelector( + `[data-layer-id="${id}"] .data-toggle-controls-info`, + ).textContent, + }), + LAYER_ID, + ); + record( + 'STARTUP: one automatic retry recovers to ON without toggling and clears the warning', + recovered.enabled && + recovered.requests === 2 && + recovered.label === 'ON' && + !/Retrying/.test(recovered.info), + JSON.stringify(recovered), + ); + await shoot(retry, 'startup-recovered'); + await retry.close(); + // โ”€โ”€ (vii) AGE โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ console.log( '\n(vii) AGE โ€” a replayed stale snapshot keeps its observation time...', diff --git a/src/layers/gnss/index.js b/src/layers/gnss/index.js index 5ad764d16..85146d832 100644 --- a/src/layers/gnss/index.js +++ b/src/layers/gnss/index.js @@ -73,12 +73,15 @@ export function createGnssIntegrityLayer({ let _stale = false; let _manager = null; let _cameraTimer = null; - let _cameraRefresh = null; + let _backgroundRefresh = null; let _lastQueryKey = null; let _lastRequestAt = -Infinity; let _removeCameraEnd = null; let _removeCameraChanged = null; let _coverage = null; + let _retryTimer = null; + let _retryAt = 0; + let _failureCount = 0; const _observations = new Map(); const provenance = gnssProvenance({ windowMs }); @@ -89,6 +92,7 @@ export function createGnssIntegrityLayer({ function scheduleCameraRefresh() { if (!_enabled || !_dataSource) return; + if (now() < _retryAt) return; const key = queryKey(viewAnchor(_viewer)); if (!key || key === _lastQueryKey) return; clearTimeout(_cameraTimer); @@ -97,34 +101,48 @@ export function createGnssIntegrityLayer({ _cameraTimer = setTimeout( async () => { _cameraTimer = null; - if (!_enabled || _cameraRefresh) return; + if (!_enabled || _backgroundRefresh) return; const key = queryKey(viewAnchor(_viewer)); if (!key || key === _lastQueryKey) return; - const controller = new AbortController(); - _cameraRefresh = controller; - try { - if (_manager) { - // Use the normal refresh lane so counts, loading/errors and render - // demand settle together; wait out an older periodic request. - await _manager.refreshLayer(layer.id, { - signal: controller.signal, - }); - } else { - await layer.update(undefined, { signal: controller.signal }); - } - } finally { - if (_cameraRefresh === controller) { - _cameraRefresh = null; - // Movement during an in-flight request is folded into one update - // for the latest view, rather than lost until the minute tick. - scheduleCameraRefresh(); - } - } + await refreshInBackground(); }, Math.max(600, 2500 - (now() - _lastRequestAt)), ); } + async function refreshInBackground() { + if (!_enabled || _backgroundRefresh) return; + const controller = new AbortController(); + _backgroundRefresh = controller; + try { + if (_manager) + await _manager.refreshLayer(layer.id, { signal: controller.signal }); + else await layer.update(undefined, { signal: controller.signal }); + } finally { + if (_backgroundRefresh === controller) { + _backgroundRefresh = null; + scheduleCameraRefresh(); + scheduleRetry(); + } + } + } + + function scheduleRetry() { + clearTimeout(_retryTimer); + _retryTimer = null; + if (!_enabled || !_retryAt) return; + // Long provider cooldowns keep their full deadline. Browser timers cannot + // exceed ~24 days, so wake in bounded chunks without fetching early. + _retryTimer = setTimeout( + () => { + _retryTimer = null; + void refreshInBackground(); + }, + Math.min(2147483647, Math.max(0, _retryAt - now())), + ); + _retryTimer?.unref?.(); + } + /** Age the store without new evidence, so expired cells leave the globe. */ function expire() { pruneGnssObservations(_observations, now(), { windowMs }); @@ -205,8 +223,10 @@ export function createGnssIntegrityLayer({ disable() { clearTimeout(_cameraTimer); _cameraTimer = null; - _cameraRefresh?.abort(); - _cameraRefresh = null; + clearTimeout(_retryTimer); + _retryTimer = null; + _backgroundRefresh?.abort(); + _backgroundRefresh = null; _request?.abort(); _request = null; _enabled = false; @@ -215,6 +235,14 @@ export function createGnssIntegrityLayer({ async update(_viewerArg, { signal } = {}) { if (!_enabled || !_dataSource || signal?.aborted) return false; + if (now() < _retryAt) { + expire(); + scheduleRetry(); + return true; + } + _retryAt = 0; + clearTimeout(_retryTimer); + _retryTimer = null; const anchor = viewAnchor(_viewer); // No anchor yet (camera still settling) is not a failure: returning // false would make the manager reject the enable. The next interval @@ -236,12 +264,14 @@ export function createGnssIntegrityLayer({ longitude: Math.round(anchor.longitude), radiusNm: 250, }; + let received = false; try { const snapshot = await source.getSnapshot(anchor, { signal: request.signal, }); if (request.signal.aborted || _request !== request || !_enabled) return false; + received = true; // Date the rows by when the proxy observed them: a cached or stale // replay keeps its original age instead of restarting the window. // The proxy reports that age, so client and server clocks never mix. @@ -255,19 +285,37 @@ export function createGnssIntegrityLayer({ _stale = snapshot.stale === true; _lastUpdate = now(); _lastError = null; + _retryAt = 0; + _failureCount = 0; return true; } catch (e) { if (request.signal.aborted || _request !== request || !_enabled) return false; + // A renderer/programming failure remains a real lifecycle failure. + // Only acquisition failures are kept enabled for recovery. + if (received) throw e; console.warn('[Data:GNSS] Fetch error:', e); _lastError = e?.message || 'GNSS integrity source unavailable'; // Keep the last cells only while their evidence is inside the window. expire(); - return false; + if (e?.retryable === false) return false; + _failureCount = Math.min(3, _failureCount + 1); + const delay = + Number.isFinite(e?.retryAfterMs) && e.retryAfterMs > 0 + ? e.retryAfterMs + : Math.min(120000, 30000 * 2 ** (_failureCount - 1)); + _retryAt = Math.max( + _retryAt, + Math.min(Number.MAX_VALUE, now() + delay), + ); + // Enabling the display succeeded. Stats still report an unavailable + // or degraded feed; the manager keeps its normal refresh failure lane. + return true; } finally { signal?.removeEventListener('abort', onAbort); if (_request === request) _request = null; scheduleCameraRefresh(); + scheduleRetry(); } }, @@ -292,6 +340,8 @@ export function createGnssIntegrityLayer({ _lastError = null; _stale = false; _coverage = null; + _retryAt = 0; + _failureCount = 0; _lastQueryKey = null; _manager = null; }, @@ -299,7 +349,15 @@ export function createGnssIntegrityLayer({ getRowControls() { return { chips: [], - info: 'Regional sample ยท 250 nm radius ยท pan to sample another area', + info: + 'Regional sample ยท 250 nm radius ยท pan to sample another area' + + (_lastError && _retryAt + ? ` ยท ${_lastError} ยท Retrying automatically${ + _retryAt <= 8640000000000000 + ? ` at ${new Date(_retryAt).toISOString().slice(11, 19)} UTC` + : ' after provider cooldown' + }` + : ''), infoTitle: 'Updates automatically after pan or zoom and once a minute. The public feed covers a 250-nautical-mile circle around the view centre; zooming out does not make it a worldwide snapshot. Blank cells have insufficient observations.', legend: ['high', 'medium', 'low'].map((level, index) => ({ @@ -320,6 +378,9 @@ export function createGnssIntegrityLayer({ count: _cells.length, lastUpdate: _lastUpdate, error: _lastError, + retryInSec: _retryAt + ? Math.max(0, Math.ceil((_retryAt - now()) / 1000)) + : null, stale: _stale, provenance, coverage: _coverage, diff --git a/src/layers/gnss/ownership.test.mjs b/src/layers/gnss/ownership.test.mjs index 5996d5e2b..a26afa1e6 100644 --- a/src/layers/gnss/ownership.test.mjs +++ b/src/layers/gnss/ownership.test.mjs @@ -3,6 +3,7 @@ import test from 'node:test'; import * as Cesium from 'cesium'; import { createGnssIntegrityLayer, defaultGnssViewAnchor } from './index.js'; import { LayerLifecycle } from '../../data/lifecycle.js'; +import { layerFeedState } from '../../data/feedState.js'; const row = (hex, lat, degraded) => ({ hex, @@ -281,7 +282,7 @@ test('a failed refresh keeps the last cells and reports the error', async () => await layer.update(); assert.equal(layer.getStats().stale, true); fail = true; - assert.equal(await layer.update(), false); + assert.equal(await layer.update(), true); assert.equal(sources[0].entities.values.length, 1); assert.equal(layer.getStats().error, 'adsb.lol HTTP 502'); assert.equal(layer.getStats().count, 1); @@ -376,10 +377,10 @@ test('cells expire during prolonged fetch failures', async () => { await layer.update(); fail = true; advance(20 * MIN); - assert.equal(await layer.update(), false); + assert.equal(await layer.update(), true); assert.equal(layer.getStats().count, 1, 'still inside the window'); advance(11 * MIN); - assert.equal(await layer.update(), false); + assert.equal(await layer.update(), true); assert.equal(layer.getStats().count, 0); assert.equal(sources[0].entities.values.length, 0); assert.equal(layer.getStats().error, 'adsb.lol HTTP 502'); @@ -432,3 +433,125 @@ test('an identical snapshot leaves the drawn entities untouched', async () => { assert.equal(sources[0].entities.values.length, 1); assert.equal(sources[0].entities.values[0], first); }); + +test('an initial feed failure stays enabled, reports unavailable, and recovers automatically', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout', 'setInterval'] }); + let calls = 0; + const h = harness( + async () => { + calls++; + if (calls === 1) throw new Error('adsb.lol HTTP 502'); + return { rows: threeHealthy() }; + }, + { initialize: false }, + ); + const manager = new LayerLifecycle(h.viewer); + manager.register(h.layer); + h.layer.attachDataManager(manager); + const changes = []; + manager.subscribe((c) => changes.push(c.type)); + assert.equal(await manager.setEnabled(h.layer.id, true), true); + assert.equal(manager.isEnabled(h.layer.id), true); + assert.equal(layerFeedState(h.layer.getStats()), 'unavailable'); + assert.equal(h.layer.getStats().retryInSec, 30); + assert.match( + h.layer.getRowControls().info, + /HTTP 502.*Retrying automatically at/, + ); + assert.equal(changes.includes('visibility-failed'), false); + assert.equal( + await manager.refreshLayer(h.layer.id), + false, + 'feed remains honestly unhealthy during cooldown', + ); + assert.equal(calls, 1, 'a manual refresh cannot bypass cooldown'); + h.advance(30000); + t.mock.timers.tick(30000); + await flush(); + assert.equal(calls, 2); + assert.equal(manager.isEnabled(h.layer.id), true); + assert.equal(layerFeedState(h.layer.getStats()), 'nominal'); + assert.equal(h.layer.getStats().count, 1); + assert.equal(h.layer.getStats().retryInSec, null); + assert.doesNotMatch(h.layer.getRowControls().info, /Retrying/); + await manager.destroyAll(); +}); + +test('provider cooldown is preserved across movement and toggling; disable cancels retry work', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let calls = 0; + const h = harness(async () => { + calls++; + if (calls === 1) + throw Object.assign(new Error('Rate limited (HTTP 429)'), { + retryAfterMs: 3600000, + }); + return { rows: threeHealthy() }; + }); + h.layer.enable(); + await h.layer.update(); + h.setAnchor({ latitude: 10, longitude: 20 }); + h.viewer.camera.moveEnd.raiseEvent(); + h.advance(30000); + t.mock.timers.tick(30000); + await flush(); + await h.layer.update(); + assert.equal(calls, 1); + assert.equal(h.layer.getStats().retryInSec, 3570); + h.layer.disable(); + h.advance(30000); + t.mock.timers.tick(30000); + await flush(); + h.layer.enable(); + await h.layer.update(); + assert.equal(calls, 1, 'reenabling does not erase a provider deadline'); + h.advance(3540000); + t.mock.timers.tick(3540000); + await flush(); + assert.equal(calls, 2); + assert.deepEqual(h.anchors.at(-1), { latitude: 10, longitude: 20 }); + h.layer.destroy(); + + const stopped = harness(async () => { + throw new Error('network unavailable'); + }); + stopped.layer.enable(); + await stopped.layer.update(); + stopped.layer.disable(); + stopped.advance(30000); + t.mock.timers.tick(30000); + await flush(); + assert.equal(stopped.anchors.length, 1, 'disabled layers do not retry'); + stopped.layer.destroy(); +}); + +test('repeated acquisition failures back off, while genuine non-retryable failures reject startup', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let calls = 0; + const h = harness(async () => { + calls++; + throw new Error('network unavailable'); + }); + h.layer.enable(); + await h.layer.update(); + assert.equal(h.layer.getStats().retryInSec, 30); + h.advance(30000); + t.mock.timers.tick(30000); + await flush(); + assert.equal(h.layer.getStats().retryInSec, 60); + h.advance(60000); + t.mock.timers.tick(60000); + await flush(); + assert.equal(h.layer.getStats().retryInSec, 120); + assert.equal(calls, 3); + h.layer.destroy(); + const fatal = harness(async () => { + throw Object.assign(new Error('invalid request (HTTP 400)'), { + retryable: false, + }); + }); + fatal.layer.enable(); + assert.equal(await fatal.layer.update(), false); + assert.equal(fatal.layer.getStats().retryInSec, null); + fatal.layer.destroy(); +}); diff --git a/src/layers/gnss/source.js b/src/layers/gnss/source.js index ae56e45c5..bfe7eaf17 100644 --- a/src/layers/gnss/source.js +++ b/src/layers/gnss/source.js @@ -1,8 +1,18 @@ import { readResponseJsonCapped } from '../../sources/httpBody.js'; +function retryDelay(raw, nowMs) { + if (!raw) return null; + const seconds = Number(raw); + const ms = seconds * 1000; + if (Number.isFinite(ms) && seconds > 0) return Math.max(5000, ms); + const at = Date.parse(raw); + return Number.isFinite(at) && at > nowMs ? Math.max(5000, at - nowMs) : null; +} + /** Request integrity facts around a view anchor through the same-origin adsb.lol proxy. */ export function createAdsbGnssSource({ fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), } = {}) { return { async getSnapshot({ latitude, longitude } = {}, { signal } = {}) { @@ -16,7 +26,25 @@ export function createAdsbGnssSource({ const response = await fetchImpl(`/api/gnss-integrity?${params}`, { signal, }); - if (!response.ok) throw new Error(`adsb.lol HTTP ${response.status}`); + if (!response.ok) { + const limited = response.status === 429; + const retryAfterMs = retryDelay( + response.headers.get('retry-after'), + now(), + ); + throw Object.assign( + new Error( + limited + ? 'adsb.lol rate limited (HTTP 429)' + : `adsb.lol feed unavailable (HTTP ${response.status})`, + ), + { + status: response.status, + retryable: limited || response.status >= 500, + retryAfterMs: retryAfterMs ?? (limited ? 60000 : null), + }, + ); + } const payload = await readResponseJsonCapped( response, 4 * 1024 * 1024, diff --git a/src/layers/gnss/source.test.mjs b/src/layers/gnss/source.test.mjs index 7bab0d7d5..aba3f3e9a 100644 --- a/src/layers/gnss/source.test.mjs +++ b/src/layers/gnss/source.test.mjs @@ -74,3 +74,38 @@ test('the source reports stale proxy data and rejects bad input or responses', a ), ); }); + +test('HTTP failures distinguish rate limiting, preserve Retry-After and classify retryability', async () => { + const at = Date.UTC(2026, 9, 13, 12); + for (const [status, header, delay, retryable] of [ + [429, '3600', 3600000, true], + [429, new Date(at + 3600000).toUTCString(), 3600000, true], + [429, null, 60000, true], + [429, 'Infinity', 60000, true], + [502, '5', 5000, true], + [502, null, null, true], + [400, null, null, false], + ]) { + const source = createAdsbGnssSource({ + now: () => at, + fetchImpl: async () => + Response.json( + {}, + { status, headers: header ? { 'Retry-After': header } : {} }, + ), + }); + await assert.rejects( + source.getSnapshot({ latitude: 50, longitude: 20 }), + (e) => { + assert.equal(e.status, status); + assert.equal(e.retryAfterMs, delay); + assert.equal(e.retryable, retryable); + assert.match( + e.message, + status === 429 ? /rate limited/ : /feed unavailable/, + ); + return true; + }, + ); + } +});