diff --git a/CHANGELOG.md b/CHANGELOG.md index df26ae5d3..cd2c4ccdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,10 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ### Fixed +- Reuse cached military aircraft during adsb.lol rate limits and server errors, + honor bounded retry delays, and preserve cached observation times and stale + indicators. Show installation zoom guidance without a false LOAD FAILED. + - GBFS rejects upstream redirects, caps streamed responses at 5 MiB, and keeps its deadline active through body reads. Rejected downloads are cancelled. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index bc8ae4646..53ff64060 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,19 @@ # God's Eye View Current State +## Military feed cooldown and loading guidance + +The adsb.lol military proxy reuses its last response during upstream 429/5xx +failures and observes Retry-After, bounded to 5–120 seconds (defaults: 30 seconds +for rate limits, 15 seconds for server errors). A failure with no cached data +still reports an error. Cached responses carry their age; the browser preserves +observation timestamps and marks fallback data stale instead of inventing fresh +positions or reporting a failed load. Fresh responses clear that stale state. + +Mapped-installation zoom guidance appears in the layer row without counting as +a global loading failure. Guidance statuses do not suppress independent refresh +errors. + + ## Places and CCTV request bounds With a Google key configured, nearby and text search reject missing, blank, diff --git a/server/providers/aircraft/adsb-lol.js b/server/providers/aircraft/adsb-lol.js index 7a34588da..fc5ab0bd3 100644 --- a/server/providers/aircraft/adsb-lol.js +++ b/server/providers/aircraft/adsb-lol.js @@ -1,8 +1,16 @@ /** * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache. * - * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. On upstream - * failure, serves a stale cached response if one exists. + * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. When upstream + * fails — a thrown fetch OR a non-OK status such as 429 — the proxy serves its + * cached body as STALE instead of relaying the failure, and it backs off from + * upstream for the Retry-After period (bounded) so a rate limit is never + * hammered mid-cooldown. A failure with nothing cached is relayed as-is. + * + * Why the non-OK case matters: the browser layer treats a relayed 429 as a + * 45 s cooldown during which EVERY refresh (the 15 s poll and any tracking + * resolve) reports "adsb.lol rate limited", which the global status chip + * renders as LOAD FAILED while the map is still fully populated. * * @returns {import('vite').Plugin} */ @@ -11,43 +19,130 @@ export function adsbLolProxy() { let _cache = null; /** @type {number} Epoch-ms when the cache was populated. */ let _cacheAt = 0; + /** @type {number} Epoch-ms until which upstream is not contacted. */ + let _cooldownUntil = 0; + /** @type {number} Upstream status that started the current cooldown. */ + let _cooldownStatus = 0; /** Response cache TTL (ms). */ const CACHE_MS = 12000; + /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */ + const RATE_LIMIT_COOLDOWN_MS = 30000; + /** Cooldown after a 5xx (ms). */ + const SERVER_ERROR_COOLDOWN_MS = 15000; + /** Bounds for an upstream-supplied Retry-After (ms). */ + const COOLDOWN_MIN_MS = 5000; + const COOLDOWN_MAX_MS = 120000; + + const clampCooldown = (ms) => + Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms)); + + /** Cooldown (ms) an upstream failure earns, honouring Retry-After when sane. */ + function cooldownFor(upstream, now) { + const raw = upstream.headers?.get?.('retry-after'); + if (raw) { + const seconds = Number(raw); + if (Number.isFinite(seconds) && seconds > 0) + return clampCooldown(seconds * 1000); + const at = Date.parse(raw); + if (Number.isFinite(at) && at > now) return clampCooldown(at - now); + } + return upstream.status === 429 + ? RATE_LIMIT_COOLDOWN_MS + : SERVER_ERROR_COOLDOWN_MS; + } + + function serve(res, status, body, cacheStatus, extra = {}) { + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'X-ADS-B-Cache': cacheStatus, + ...(['HIT', 'STALE'].includes(cacheStatus) + ? { + 'X-ADS-B-Cache-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)), + } + : {}), + ...extra, + }); + res.end(body); + } + const installMiddleware = (server) => { server.middlewares.use('/api/adsblol/mil', async (req, res) => { try { const now = Date.now(); if (_cache && now - _cacheAt < CACHE_MS) { - res.writeHead(200, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - 'X-ADS-B-Cache': 'HIT', - }); - res.end(_cache); + serve(res, 200, _cache, 'HIT'); + return; + } + if (now < _cooldownUntil) { + const retryAfter = String( + Math.ceil((_cooldownUntil - Date.now()) / 1000), + ); + if (_cache) { + serve(res, 200, _cache, 'STALE', { + 'X-ADS-B-Upstream-Status': String(_cooldownStatus), + 'X-ADS-B-Retry-After-Seconds': retryAfter, + }); + return; + } + serve( + res, + _cooldownStatus || 503, + JSON.stringify({ error: 'adsb.lol upstream cooling down' }), + 'NONE', + { 'Retry-After': retryAfter }, + ); return; } const upstream = await fetch('https://api.adsb.lol/v2/mil', { headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' }, }); - const body = await upstream.text(); if (upstream.ok) { + const body = await upstream.text(); _cache = body; - _cacheAt = now; + _cacheAt = Date.now(); + _cooldownUntil = 0; + _cooldownStatus = 0; + serve(res, 200, body, 'MISS'); + return; } - res.writeHead(upstream.status, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - 'X-ADS-B-Cache': 'MISS', - }); - res.end(body); + if (upstream.status === 429 || upstream.status >= 500) { + const failedAt = Date.now(); + _cooldownUntil = failedAt + cooldownFor(upstream, failedAt); + _cooldownStatus = upstream.status; + console.warn( + `[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`, + ); + if (_cache) { + // Do not wait for a failing response body before serving usable data. + upstream.body?.cancel().catch(() => {}); + serve(res, 200, _cache, 'STALE', { + 'X-ADS-B-Upstream-Status': String(upstream.status), + 'X-ADS-B-Retry-After-Seconds': String( + Math.ceil((_cooldownUntil - Date.now()) / 1000), + ), + }); + return; + } + } + const body = await upstream.text(); + serve( + res, + upstream.status, + body, + 'MISS', + _cooldownUntil > Date.now() + ? { + 'Retry-After': String( + Math.ceil((_cooldownUntil - Date.now()) / 1000), + ), + } + : {}, + ); } catch (e) { console.error('[adsb.lol Proxy]', e.message); if (_cache) { - res.writeHead(200, { - 'Content-Type': 'application/json', - 'X-ADS-B-Cache': 'STALE', - }); - res.end(_cache); + serve(res, 200, _cache, 'STALE'); return; } res.writeHead(502, { 'Content-Type': 'application/json' }); diff --git a/src/data/manager.js b/src/data/manager.js index 7db25def4..9ba4ae24c 100644 --- a/src/data/manager.js +++ b/src/data/manager.js @@ -1,4 +1,5 @@ import { governorRequestRender } from '../renderGovernor.js'; +import { GUIDANCE_STATUSES } from '../loadingFeedback.js'; import { markDetectionSourcesChanged } from './detection.js'; function cloneLayerParams(value) { if (Array.isArray(value)) return value.map(cloneLayerParams); @@ -80,7 +81,7 @@ export function layerFeedState(stats = {}) { if ( (presentedError || state.unavailable === true || state.available === false) && !hasPriorData - && !['zoom-in', 'empty', 'idle'].includes(status) + && !GUIDANCE_STATUSES.includes(status) ) { return 'unavailable'; } @@ -89,7 +90,7 @@ export function layerFeedState(stats = {}) { // operation, not feed faults. One honesty carve-out: layers keep their // rendered records through the guidance state, so a genuinely stale cache // still reads STALE; a guidance prompt alone never reads DEGRADED. - if (['zoom-in', 'empty', 'idle'].includes(status)) { + if (GUIDANCE_STATUSES.includes(status)) { return state.stale ? 'stale' : 'nominal'; } if ( @@ -2233,6 +2234,12 @@ export class DataLayerManager { } return `${stateLabel} · ${source} · ${presentedError}`; } + // A guidance status carries its prompt in `statusMessage`, not `error`, so + // the row still tells the operator what to do without reporting a fault. + if (GUIDANCE_STATUSES.includes(String(stats.status || '').toLowerCase()) + && typeof stats.statusMessage === 'string' && stats.statusMessage.trim()) { + return `${source} · ${stats.statusMessage.trim()}`; + } const ago = stats.lastUpdate ? this._timeAgo(stats.lastUpdate) : 'never'; if (stats.loading) { const loadingLabel = typeof stats.loadingLabel === 'string' && stats.loadingLabel.trim() diff --git a/src/data/manager.test.mjs b/src/data/manager.test.mjs index 60b7b3fdc..73e3d4bad 100644 --- a/src/data/manager.test.mjs +++ b/src/data/manager.test.mjs @@ -3256,3 +3256,14 @@ test('a layer that surrenders its row controls hides the block entirely', async else globalThis.document = originalDocument; } }); + +test('layer metadata shows a guidance prompt without reporting it as a fault', () => { + const mgr = new DataLayerManager({}); + const text = mgr._buildMetaText({ + enabled: true, + source: 'OpenStreetMap', + stats: { status: 'zoom-in', error: null, count: 0, statusMessage: 'Zoom in to search mapped installations' }, + }); + assert.match(text, /Zoom in to search mapped installations$/); + assert.doesNotMatch(text, /UNAVAILABLE|DEGRADED/); +}); diff --git a/src/data/militaryFlights.js b/src/data/militaryFlights.js index cdacb64ef..220747dcd 100644 --- a/src/data/militaryFlights.js +++ b/src/data/militaryFlights.js @@ -2833,7 +2833,9 @@ const militaryFlightsLayer = { _retryAt = 0; _lastError = null; const currentIcaos = new Set(); - const receiptNowMs = Date.now(); + const cacheAgeMs = Number(response.headers?.get?.('x-ads-b-cache-age-ms')); + const receiptNowMs = Date.now() - (Number.isFinite(cacheAgeMs) && cacheAgeMs > 0 ? cacheAgeMs : 0); + _backoff = response.headers?.get?.('x-ads-b-cache') === 'STALE'; // Field-test fix (RS46): coarse floor cells to warm for the below-ground // clamp — collected during the loop (low airborne contacts only), // batch-resolved once after it. Never a fetch inside the loop. @@ -3215,7 +3217,7 @@ const militaryFlightsLayer = { registerMilitaryIcaos(currentIcaos); _count = _billboards.size; - _lastUpdate = Date.now(); + _lastUpdate = receiptNowMs; _lastTrackingRefreshOutcome = { epoch: trackingRefreshEpoch, status: 'accepted', diff --git a/src/data/militaryFlights.test.mjs b/src/data/militaryFlights.test.mjs index 74ce13832..122b6b132 100644 --- a/src/data/militaryFlights.test.mjs +++ b/src/data/militaryFlights.test.mjs @@ -219,6 +219,49 @@ test('military poll refreshes tracked callsign/altitude/kts and marks a missed p } }); +test('cached military snapshots retain their observation time and expose stale fallback', async (t) => { + let now = 1_800_000_000_000; + const receivedAt = now; + const history = []; + const viewer = { camera: { positionCartographic: null }, scene: {} }; + _setTrackedMilitaryRefreshStateForTest({ + icao24: 'ae01ce', + entity: { gevLabelModel: { title: 'RCH451', details: [] } }, + billboard: { position: Cesium.Cartesian3.fromDegrees(-97, 31, 8000), show: false }, + billboardCollection: { show: false, remove() {} }, + viewer, + history, + meta: { rawLat: 31, rawLon: -97, onGround: false }, + }); + let cache = 'MISS'; + t.mock.method(Date, 'now', () => now); + t.mock.method(globalThis, 'fetch', async () => Response.json({ + ac: [{ hex: 'ae01ce', lon: -97, lat: 31, alt_baro: 28000, + track: 95, gs: 400, seen: 1, seen_pos: 2, flight: 'RCH451' }], + }, { headers: { + 'X-ADS-B-Cache': cache, + ...(cache === 'MISS' ? {} : { 'X-ADS-B-Cache-Age-Ms': String(now - receivedAt) }), + } })); + await militaryFlightsLayer.update(viewer); + assert.equal(history.length, 1); + assert.equal(history[0].epochMs, receivedAt - 2000); + for (const [status, age] of [['HIT', 5000], ['STALE', 45000]]) { + cache = status; + now = receivedAt + age; + await militaryFlightsLayer.update(viewer); + assert.equal(history.length, 1, 'replayed positions cannot create a new fix'); + assert.equal(militaryFlightsLayer.getStats().lastUpdate, receivedAt); + assert.equal(militaryFlightsLayer.getStats().stale, status === 'STALE'); + assert.equal(militaryFlightsLayer.getStats().error, null); + } + cache = 'MISS'; + now += 15000; + await militaryFlightsLayer.update(viewer); + assert.equal(history.length, 2, 'fresh data resumes position updates'); + assert.equal(militaryFlightsLayer.getStats().lastUpdate, now); + assert.equal(militaryFlightsLayer.getStats().stale, false); +}); + test('real military track path creates no native label and publishes every cached host line', () => { const icao24 = 'ae01ce'; const position = Cesium.Cartesian3.fromDegrees(-97.03, 31.05, 8_534.4); diff --git a/src/data/militaryInstallations.js b/src/data/militaryInstallations.js index 26e1fe736..0013146d1 100644 --- a/src/data/militaryInstallations.js +++ b/src/data/militaryInstallations.js @@ -425,12 +425,18 @@ function scheduleLoad() { async function loadInstallations() { if (!state.enabled || !state.viewer) return; const box = viewportBox(state.viewer); + // Guidance, not a fault: the layer chose not to query because the view is + // unbounded (a global view, or Cockpit looking at the horizon). Keep it out + // of `error` — the manager derives refresh failures and the global status + // chip derives LOAD FAILED from that field, and a "zoom in" prompt turned + // every Cockpit refresh into a reported failure. The row and toast read the + // prompt from `statusMessage` (installationFeedback) instead. if (!box) { state.abort?.abort(); state.abort = null; state.loading = false; clearUnavailableRetry(); - setInstallationStatus('zoom-in', 'Zoom in to load mapped installation context'); + setInstallationStatus('zoom-in'); return; } state.abort?.abort(); diff --git a/src/data/militaryInstallations.test.mjs b/src/data/militaryInstallations.test.mjs index d002dcccc..fff2c6eef 100644 --- a/src/data/militaryInstallations.test.mjs +++ b/src/data/militaryInstallations.test.mjs @@ -758,6 +758,8 @@ test('zoom-out aborts an active installation request and returns non-loading gui assert.equal(observedSignal.aborted, true); assert.equal(militaryInstallationsLayer.getStats().loading, false); assert.equal(militaryInstallationsLayer.getStats().status, 'zoom-in'); + assert.equal(militaryInstallationsLayer.getStats().error, null, 'guidance is not a fault'); + assert.match(militaryInstallationsLayer.getStats().statusMessage, /zoom in/i); } finally { militaryInstallationsLayer.destroy(viewer); globalThis.fetch = originalFetch; diff --git a/src/loadingFeedback.js b/src/loadingFeedback.js index 5ff5dd2de..a5540e1ce 100644 --- a/src/loadingFeedback.js +++ b/src/loadingFeedback.js @@ -5,6 +5,8 @@ export const LOADING_TERMINAL_DWELL_MS = 2200; export const LOADING_FAILURE_DWELL_MS = 5000; export const LOADING_LONG_THRESHOLD_MS = 30000; export const TRAFFIC_SYNC_CONFIRM_MS = 1500; +/** Layer statuses that are user guidance, not feed faults (see manager.js layerFeedState). */ +export const GUIDANCE_STATUSES = Object.freeze(['zoom-in', 'empty', 'idle']); function finiteCount(value) { const number = Number(value); @@ -20,7 +22,12 @@ export function normalizeLayerLoading(layer = {}) { const loading = lifecycleState === 'enabling' || disabling || stats.loading === true || stats.refreshing === true; const count = finiteCount(stats.count); const stoppingInstallations = layer.id === 'military-installations' && disabling; - const error = stoppingInstallations ? null : stats.error || stats.lastError || stats.managerRefreshError || null; + // Guidance statuses ask the user to act (zoom in, run a search). They are + // normal operation, never a batch failure — mirrors layerFeedState's carve-out + // so a prompt stored alongside the status cannot turn the chip red. + const guidance = GUIDANCE_STATUSES.includes(status); + const error = stoppingInstallations ? null + : (!guidance && stats.error) || stats.lastError || stats.managerRefreshError || null; const unavailable = !stoppingInstallations && (stats.unavailable === true || stats.available === false || ['unavailable', 'offline', 'down', 'error'].includes(status)); diff --git a/src/loadingFeedback.test.mjs b/src/loadingFeedback.test.mjs index 51b1da5f6..7537efb06 100644 --- a/src/loadingFeedback.test.mjs +++ b/src/loadingFeedback.test.mjs @@ -685,3 +685,28 @@ test('the loading ticker never runs hidden and stops after loading and notices s 'the resample handler must be removed on teardown', ); }); + +test('a guidance status such as zoom-in never counts as a participant failure', () => { + const zoomIn = { id: 'military-installations', name: 'Mapped Installations', enabled: true, + stats: { status: 'zoom-in', error: 'Zoom in to load mapped installation context', loading: true, count: 0 } }; + const loading = aggregateLayerLoading([zoomIn]); + assert.equal(loading.records[0].error, null); + assert.equal(loading.records[0].degraded, false); + let state = reduceLoadingFeedback(createLoadingFeedbackState(), loading, 1000); + state = reduceLoadingFeedback(state, loading, 1200); + const settled = aggregateLayerLoading([{ ...zoomIn, stats: { ...zoomIn.stats, loading: false } }]); + state = reduceLoadingFeedback(state, settled, 1500); + assert.equal(state.terminal, 'complete'); + assert.equal(presentLoadingFeedback(state, settled, 1500).label, 'MAPPED SITES LOADED'); +}); + + +test('guidance does not suppress independent manager and feed failures', () => { + for (const field of ['lastError', 'managerRefreshError']) { + const record = normalizeLayerLoading({ + id: 'militaryInstallations', enabled: true, + stats: { status: 'zoom-in', error: 'Zoom in to load mapped sites.', [field]: 'Network unavailable' }, + }); + assert.equal(record.error, 'Network unavailable'); + } +}); diff --git a/src/tooling/liveProviders.test.mjs b/src/tooling/liveProviders.test.mjs index edbc94e14..250924b42 100644 --- a/src/tooling/liveProviders.test.mjs +++ b/src/tooling/liveProviders.test.mjs @@ -230,3 +230,126 @@ test('AIS preview route ingests through the socket, returns tracks and disposes for (let i = 0; i < 100 && sockets.length < 2; i++) await delay(10); assert.equal(sockets.length, 2); }); + +test('military aircraft route serves stale cache on an upstream 429 and cools down for Retry-After', async (t) => { + let now = Date.now(); + let calls = 0; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + if (calls === 1) return Response.json({ ac: [{ hex: 'abc123' }] }); + return new Response(JSON.stringify({ error: 'rate limited' }), { + status: 429, + headers: { 'Retry-After': '20' }, + }); + }); + const request = install(providers.adsbLolProxy()); + const first = await request('/api/adsblol/mil'); + assert.equal(first.statusCode, 200); + now += 13_000; + const limited = await request('/api/adsblol/mil'); + assert.equal(calls, 2); + assert.equal( + limited.statusCode, + 200, + 'a 429 with a cached body is never relayed', + ); + assert.equal(limited.body, first.body); + assert.equal(limited.headers['x-ads-b-cache'], 'STALE'); + assert.equal(limited.headers['x-ads-b-upstream-status'], '429'); + assert.equal(limited.headers['x-ads-b-cache-age-ms'], '13000'); + now += 5_000; + const cooling = await request('/api/adsblol/mil'); + assert.equal(calls, 2, 'no upstream call inside the Retry-After window'); + assert.equal(cooling.headers['x-ads-b-cache'], 'STALE'); + assert.equal(cooling.headers['x-ads-b-cache-age-ms'], '18000'); + now += 16_000; + await request('/api/adsblol/mil'); + assert.equal(calls, 3, 'upstream is retried once Retry-After elapses'); +}); + +test('military aircraft route relays an upstream 429 when nothing is cached', async (t) => { + t.mock.method(console, 'warn', () => {}); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return new Response('{"error":"rate limited"}', { status: 429 }); + }); + const request = install(providers.adsbLolProxy()); + const limited = await request('/api/adsblol/mil'); + assert.equal(limited.statusCode, 429); + assert.ok(limited.headers['retry-after']); + const again = await request('/api/adsblol/mil'); + assert.equal(again.statusCode, 429); + assert.equal( + calls, + 1, + 'the cooldown still protects upstream with nothing cached', + ); + assert.ok( + again.headers['retry-after'], + 'a cooling-down miss carries Retry-After', + ); +}); + +test('military fallback cancels a stalled 5xx body and starts cooldown at receipt', async (t) => { + let now = 1_800_000_000_000; + let calls = 0; + let cancelled = false; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + if (++calls === 1) return Response.json({ ac: [] }); + now += 10000; + return new Response( + new ReadableStream({ + cancel() { + cancelled = true; + }, + }), + { + status: 503, + headers: { 'Retry-After': new Date(now + 20000).toUTCString() }, + }, + ); + }); + const request = install(providers.adsbLolProxy()); + await request('/api/adsblol/mil'); + now += 1000; + const hit = await request('/api/adsblol/mil'); + assert.equal(hit.headers['x-ads-b-cache-age-ms'], '1000'); + now += 12000; + const fallback = await request('/api/adsblol/mil'); + assert.equal(fallback.statusCode, 200); + assert.equal(cancelled, true); + assert.equal(fallback.headers['x-ads-b-cache-age-ms'], '23000'); + assert.equal(fallback.headers['x-ads-b-retry-after-seconds'], '20'); + now += 19000; + await request('/api/adsblol/mil'); + assert.equal(calls, 2); + now += 2000; + await request('/api/adsblol/mil'); + assert.equal(calls, 3); +}); + +test('military cooldown bounds untrusted Retry-After and defaults server errors', async (t) => { + t.mock.method(console, 'warn', () => {}); + for (const [status, raw, seconds] of [ + [429, '1', 5], + [429, '99999', 120], + [503, 'invalid', 15], + ]) { + t.mock.method( + globalThis, + 'fetch', + async () => + new Response('{}', { + status, + headers: { 'Retry-After': raw }, + }), + ); + const result = await install(providers.adsbLolProxy())('/api/adsblol/mil'); + assert.equal(Number(result.headers['retry-after']), seconds); + } +});