Merge remote-tracking branch 'origin/main' into review/bilawal-issue-forms-367

CHANGELOG.md

@@ -26,6 +26,10 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md
 
 ### Fixed
 
+- Reuse cached military aircraft during adsb.lol rate limits and server errors,
+  honor bounded retry delays, and preserve cached observation times and stale
+  indicators. Show installation zoom guidance without a false LOAD FAILED.
+
 - GBFS rejects upstream redirects, caps streamed responses at 5 MiB, and keeps
   its deadline active through body reads. Rejected downloads are cancelled.
 

docs/CURRENT-STATE.md

@@ -1,5 +1,19 @@
 # God's Eye View Current State
 
+## Military feed cooldown and loading guidance
+
+The adsb.lol military proxy reuses its last response during upstream 429/5xx
+failures and observes Retry-After, bounded to 5–120 seconds (defaults: 30 seconds
+for rate limits, 15 seconds for server errors). A failure with no cached data
+still reports an error. Cached responses carry their age; the browser preserves
+observation timestamps and marks fallback data stale instead of inventing fresh
+positions or reporting a failed load. Fresh responses clear that stale state.
+
+Mapped-installation zoom guidance appears in the layer row without counting as
+a global loading failure. Guidance statuses do not suppress independent refresh
+errors.
+
+
 ## Places and CCTV request bounds
 
 With a Google key configured, nearby and text search reject missing, blank,

server/providers/aircraft/adsb-lol.js

@@ -1,8 +1,16 @@
 /**
  * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
  *
- * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. On upstream
- * failure, serves a stale cached response if one exists.
+ * Proxies GET /api/adsblol/mil to https://api.adsb.lol/v2/mil. When upstream
+ * fails — a thrown fetch OR a non-OK status such as 429 — the proxy serves its
+ * cached body as STALE instead of relaying the failure, and it backs off from
+ * upstream for the Retry-After period (bounded) so a rate limit is never
+ * hammered mid-cooldown. A failure with nothing cached is relayed as-is.
+ *
+ * Why the non-OK case matters: the browser layer treats a relayed 429 as a
+ * 45 s cooldown during which EVERY refresh (the 15 s poll and any tracking
+ * resolve) reports "adsb.lol rate limited", which the global status chip
+ * renders as LOAD FAILED while the map is still fully populated.
  *
  * @returns {import('vite').Plugin}
  */
@@ -11,43 +19,130 @@ export function adsbLolProxy() {
   let _cache = null;
   /** @type {number} Epoch-ms when the cache was populated. */
   let _cacheAt = 0;
+  /** @type {number} Epoch-ms until which upstream is not contacted. */
+  let _cooldownUntil = 0;
+  /** @type {number} Upstream status that started the current cooldown. */
+  let _cooldownStatus = 0;
   /** Response cache TTL (ms). */
   const CACHE_MS = 12000;
+  /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
+  const RATE_LIMIT_COOLDOWN_MS = 30000;
+  /** Cooldown after a 5xx (ms). */
+  const SERVER_ERROR_COOLDOWN_MS = 15000;
+  /** Bounds for an upstream-supplied Retry-After (ms). */
+  const COOLDOWN_MIN_MS = 5000;
+  const COOLDOWN_MAX_MS = 120000;
+
+  const clampCooldown = (ms) =>
+    Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
+
+  /** Cooldown (ms) an upstream failure earns, honouring Retry-After when sane. */
+  function cooldownFor(upstream, now) {
+    const raw = upstream.headers?.get?.('retry-after');
+    if (raw) {
+      const seconds = Number(raw);
+      if (Number.isFinite(seconds) && seconds > 0)
+        return clampCooldown(seconds * 1000);
+      const at = Date.parse(raw);
+      if (Number.isFinite(at) && at > now) return clampCooldown(at - now);
+    }
+    return upstream.status === 429
+      ? RATE_LIMIT_COOLDOWN_MS
+      : SERVER_ERROR_COOLDOWN_MS;
+  }
+
+  function serve(res, status, body, cacheStatus, extra = {}) {
+    res.writeHead(status, {
+      'Content-Type': 'application/json',
+      'Cache-Control': 'no-store',
+      'X-ADS-B-Cache': cacheStatus,
+      ...(['HIT', 'STALE'].includes(cacheStatus)
+        ? {
+            'X-ADS-B-Cache-Age-Ms': String(Math.max(0, Date.now() - _cacheAt)),
+          }
+        : {}),
+      ...extra,
+    });
+    res.end(body);
+  }
+
   const installMiddleware = (server) => {
     server.middlewares.use('/api/adsblol/mil', async (req, res) => {
       try {
         const now = Date.now();
         if (_cache && now - _cacheAt < CACHE_MS) {
-          res.writeHead(200, {
-            'Content-Type': 'application/json',
-            'Cache-Control': 'no-store',
-            'X-ADS-B-Cache': 'HIT',
-          });
-          res.end(_cache);
+          serve(res, 200, _cache, 'HIT');
+          return;
+        }
+        if (now < _cooldownUntil) {
+          const retryAfter = String(
+            Math.ceil((_cooldownUntil - Date.now()) / 1000),
+          );
+          if (_cache) {
+            serve(res, 200, _cache, 'STALE', {
+              'X-ADS-B-Upstream-Status': String(_cooldownStatus),
+              'X-ADS-B-Retry-After-Seconds': retryAfter,
+            });
+            return;
+          }
+          serve(
+            res,
+            _cooldownStatus || 503,
+            JSON.stringify({ error: 'adsb.lol upstream cooling down' }),
+            'NONE',
+            { 'Retry-After': retryAfter },
+          );
           return;
         }
         const upstream = await fetch('https://api.adsb.lol/v2/mil', {
           headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
         });
-        const body = await upstream.text();
         if (upstream.ok) {
+          const body = await upstream.text();
           _cache = body;
-          _cacheAt = now;
+          _cacheAt = Date.now();
+          _cooldownUntil = 0;
+          _cooldownStatus = 0;
+          serve(res, 200, body, 'MISS');
+          return;
         }
-        res.writeHead(upstream.status, {
-          'Content-Type': 'application/json',
-          'Cache-Control': 'no-store',
-          'X-ADS-B-Cache': 'MISS',
-        });
-        res.end(body);
+        if (upstream.status === 429 || upstream.status >= 500) {
+          const failedAt = Date.now();
+          _cooldownUntil = failedAt + cooldownFor(upstream, failedAt);
+          _cooldownStatus = upstream.status;
+          console.warn(
+            `[adsb.lol Proxy] upstream ${upstream.status}; cooling down ${Math.round((_cooldownUntil - Date.now()) / 1000)} s${_cache ? ', serving stale' : ''}`,
+          );
+          if (_cache) {
+            // Do not wait for a failing response body before serving usable data.
+            upstream.body?.cancel().catch(() => {});
+            serve(res, 200, _cache, 'STALE', {
+              'X-ADS-B-Upstream-Status': String(upstream.status),
+              'X-ADS-B-Retry-After-Seconds': String(
+                Math.ceil((_cooldownUntil - Date.now()) / 1000),
+              ),
+            });
+            return;
+          }
+        }
+        const body = await upstream.text();
+        serve(
+          res,
+          upstream.status,
+          body,
+          'MISS',
+          _cooldownUntil > Date.now()
+            ? {
+                'Retry-After': String(
+                  Math.ceil((_cooldownUntil - Date.now()) / 1000),
+                ),
+              }
+            : {},
+        );
       } catch (e) {
         console.error('[adsb.lol Proxy]', e.message);
         if (_cache) {
-          res.writeHead(200, {
-            'Content-Type': 'application/json',
-            'X-ADS-B-Cache': 'STALE',
-          });
-          res.end(_cache);
+          serve(res, 200, _cache, 'STALE');
           return;
         }
         res.writeHead(502, { 'Content-Type': 'application/json' });

src/data/manager.js

@@ -1,4 +1,5 @@
 import { governorRequestRender } from '../renderGovernor.js';
+import { GUIDANCE_STATUSES } from '../loadingFeedback.js';
 import { markDetectionSourcesChanged } from './detection.js';
 function cloneLayerParams(value) {
   if (Array.isArray(value)) return value.map(cloneLayerParams);
@@ -80,7 +81,7 @@ export function layerFeedState(stats = {}) {
   if (
     (presentedError || state.unavailable === true || state.available === false)
     && !hasPriorData
-    && !['zoom-in', 'empty', 'idle'].includes(status)
+    && !GUIDANCE_STATUSES.includes(status)
   ) {
     return 'unavailable';
   }
@@ -89,7 +90,7 @@ export function layerFeedState(stats = {}) {
   // operation, not feed faults. One honesty carve-out: layers keep their
   // rendered records through the guidance state, so a genuinely stale cache
   // still reads STALE; a guidance prompt alone never reads DEGRADED.
-  if (['zoom-in', 'empty', 'idle'].includes(status)) {
+  if (GUIDANCE_STATUSES.includes(status)) {
     return state.stale ? 'stale' : 'nominal';
   }
   if (
@@ -2233,6 +2234,12 @@ export class DataLayerManager {
       }
       return `${stateLabel} · ${source} · ${presentedError}`;
     }
+    // A guidance status carries its prompt in `statusMessage`, not `error`, so
+    // the row still tells the operator what to do without reporting a fault.
+    if (GUIDANCE_STATUSES.includes(String(stats.status || '').toLowerCase())
+        && typeof stats.statusMessage === 'string' && stats.statusMessage.trim()) {
+      return `${source} · ${stats.statusMessage.trim()}`;
+    }
     const ago = stats.lastUpdate ? this._timeAgo(stats.lastUpdate) : 'never';
     if (stats.loading) {
       const loadingLabel = typeof stats.loadingLabel === 'string' && stats.loadingLabel.trim()

src/data/manager.test.mjs

@@ -3256,3 +3256,14 @@ test('a layer that surrenders its row controls hides the block entirely', async
     else globalThis.document = originalDocument;
   }
 });
+
+test('layer metadata shows a guidance prompt without reporting it as a fault', () => {
+  const mgr = new DataLayerManager({});
+  const text = mgr._buildMetaText({
+    enabled: true,
+    source: 'OpenStreetMap',
+    stats: { status: 'zoom-in', error: null, count: 0, statusMessage: 'Zoom in to search mapped installations' },
+  });
+  assert.match(text, /Zoom in to search mapped installations$/);
+  assert.doesNotMatch(text, /UNAVAILABLE|DEGRADED/);
+});

src/data/militaryFlights.js

@@ -2833,7 +2833,9 @@ const militaryFlightsLayer = {
       _retryAt = 0;
       _lastError = null;
       const currentIcaos = new Set();
-      const receiptNowMs = Date.now();
+      const cacheAgeMs = Number(response.headers?.get?.('x-ads-b-cache-age-ms'));
+      const receiptNowMs = Date.now() - (Number.isFinite(cacheAgeMs) && cacheAgeMs > 0 ? cacheAgeMs : 0);
+      _backoff = response.headers?.get?.('x-ads-b-cache') === 'STALE';
       // Field-test fix (RS46): coarse floor cells to warm for the below-ground
       // clamp — collected during the loop (low airborne contacts only),
       // batch-resolved once after it. Never a fetch inside the loop.
@@ -3215,7 +3217,7 @@ const militaryFlightsLayer = {
       registerMilitaryIcaos(currentIcaos);
 
       _count = _billboards.size;
-      _lastUpdate = Date.now();
+      _lastUpdate = receiptNowMs;
       _lastTrackingRefreshOutcome = {
         epoch: trackingRefreshEpoch,
         status: 'accepted',

src/data/militaryFlights.test.mjs

@@ -219,6 +219,49 @@ test('military poll refreshes tracked callsign/altitude/kts and marks a missed p
   }
 });
 
+test('cached military snapshots retain their observation time and expose stale fallback', async (t) => {
+  let now = 1_800_000_000_000;
+  const receivedAt = now;
+  const history = [];
+  const viewer = { camera: { positionCartographic: null }, scene: {} };
+  _setTrackedMilitaryRefreshStateForTest({
+    icao24: 'ae01ce',
+    entity: { gevLabelModel: { title: 'RCH451', details: [] } },
+    billboard: { position: Cesium.Cartesian3.fromDegrees(-97, 31, 8000), show: false },
+    billboardCollection: { show: false, remove() {} },
+    viewer,
+    history,
+    meta: { rawLat: 31, rawLon: -97, onGround: false },
+  });
+  let cache = 'MISS';
+  t.mock.method(Date, 'now', () => now);
+  t.mock.method(globalThis, 'fetch', async () => Response.json({
+    ac: [{ hex: 'ae01ce', lon: -97, lat: 31, alt_baro: 28000,
+      track: 95, gs: 400, seen: 1, seen_pos: 2, flight: 'RCH451' }],
+  }, { headers: {
+    'X-ADS-B-Cache': cache,
+    ...(cache === 'MISS' ? {} : { 'X-ADS-B-Cache-Age-Ms': String(now - receivedAt) }),
+  } }));
+  await militaryFlightsLayer.update(viewer);
+  assert.equal(history.length, 1);
+  assert.equal(history[0].epochMs, receivedAt - 2000);
+  for (const [status, age] of [['HIT', 5000], ['STALE', 45000]]) {
+    cache = status;
+    now = receivedAt + age;
+    await militaryFlightsLayer.update(viewer);
+    assert.equal(history.length, 1, 'replayed positions cannot create a new fix');
+    assert.equal(militaryFlightsLayer.getStats().lastUpdate, receivedAt);
+    assert.equal(militaryFlightsLayer.getStats().stale, status === 'STALE');
+    assert.equal(militaryFlightsLayer.getStats().error, null);
+  }
+  cache = 'MISS';
+  now += 15000;
+  await militaryFlightsLayer.update(viewer);
+  assert.equal(history.length, 2, 'fresh data resumes position updates');
+  assert.equal(militaryFlightsLayer.getStats().lastUpdate, now);
+  assert.equal(militaryFlightsLayer.getStats().stale, false);
+});
+
 test('real military track path creates no native label and publishes every cached host line', () => {
   const icao24 = 'ae01ce';
   const position = Cesium.Cartesian3.fromDegrees(-97.03, 31.05, 8_534.4);

src/data/militaryInstallations.js

@@ -425,12 +425,18 @@ function scheduleLoad() {
 async function loadInstallations() {
   if (!state.enabled || !state.viewer) return;
   const box = viewportBox(state.viewer);
+  // Guidance, not a fault: the layer chose not to query because the view is
+  // unbounded (a global view, or Cockpit looking at the horizon). Keep it out
+  // of `error` — the manager derives refresh failures and the global status
+  // chip derives LOAD FAILED from that field, and a "zoom in" prompt turned
+  // every Cockpit refresh into a reported failure. The row and toast read the
+  // prompt from `statusMessage` (installationFeedback) instead.
   if (!box) {
     state.abort?.abort();
     state.abort = null;
     state.loading = false;
     clearUnavailableRetry();
-    setInstallationStatus('zoom-in', 'Zoom in to load mapped installation context');
+    setInstallationStatus('zoom-in');
     return;
   }
   state.abort?.abort();

src/data/militaryInstallations.test.mjs

@@ -758,6 +758,8 @@ test('zoom-out aborts an active installation request and returns non-loading gui
     assert.equal(observedSignal.aborted, true);
     assert.equal(militaryInstallationsLayer.getStats().loading, false);
     assert.equal(militaryInstallationsLayer.getStats().status, 'zoom-in');
+    assert.equal(militaryInstallationsLayer.getStats().error, null, 'guidance is not a fault');
+    assert.match(militaryInstallationsLayer.getStats().statusMessage, /zoom in/i);
   } finally {
     militaryInstallationsLayer.destroy(viewer);
     globalThis.fetch = originalFetch;

src/loadingFeedback.js

@@ -5,6 +5,8 @@ export const LOADING_TERMINAL_DWELL_MS = 2200;
 export const LOADING_FAILURE_DWELL_MS = 5000;
 export const LOADING_LONG_THRESHOLD_MS = 30000;
 export const TRAFFIC_SYNC_CONFIRM_MS = 1500;
+/** Layer statuses that are user guidance, not feed faults (see manager.js layerFeedState). */
+export const GUIDANCE_STATUSES = Object.freeze(['zoom-in', 'empty', 'idle']);
 
 function finiteCount(value) {
   const number = Number(value);
@@ -20,7 +22,12 @@ export function normalizeLayerLoading(layer = {}) {
   const loading = lifecycleState === 'enabling' || disabling || stats.loading === true || stats.refreshing === true;
   const count = finiteCount(stats.count);
   const stoppingInstallations = layer.id === 'military-installations' && disabling;
-  const error = stoppingInstallations ? null : stats.error || stats.lastError || stats.managerRefreshError || null;
+  // Guidance statuses ask the user to act (zoom in, run a search). They are
+  // normal operation, never a batch failure — mirrors layerFeedState's carve-out
+  // so a prompt stored alongside the status cannot turn the chip red.
+  const guidance = GUIDANCE_STATUSES.includes(status);
+  const error = stoppingInstallations ? null
+    : (!guidance && stats.error) || stats.lastError || stats.managerRefreshError || null;
   const unavailable = !stoppingInstallations && (stats.unavailable === true
     || stats.available === false
     || ['unavailable', 'offline', 'down', 'error'].includes(status));

src/loadingFeedback.test.mjs

@@ -685,3 +685,28 @@ test('the loading ticker never runs hidden and stops after loading and notices s
     'the resample handler must be removed on teardown',
   );
 });
+
+test('a guidance status such as zoom-in never counts as a participant failure', () => {
+  const zoomIn = { id: 'military-installations', name: 'Mapped Installations', enabled: true,
+    stats: { status: 'zoom-in', error: 'Zoom in to load mapped installation context', loading: true, count: 0 } };
+  const loading = aggregateLayerLoading([zoomIn]);
+  assert.equal(loading.records[0].error, null);
+  assert.equal(loading.records[0].degraded, false);
+  let state = reduceLoadingFeedback(createLoadingFeedbackState(), loading, 1000);
+  state = reduceLoadingFeedback(state, loading, 1200);
+  const settled = aggregateLayerLoading([{ ...zoomIn, stats: { ...zoomIn.stats, loading: false } }]);
+  state = reduceLoadingFeedback(state, settled, 1500);
+  assert.equal(state.terminal, 'complete');
+  assert.equal(presentLoadingFeedback(state, settled, 1500).label, 'MAPPED SITES LOADED');
+});
+
+
+test('guidance does not suppress independent manager and feed failures', () => {
+  for (const field of ['lastError', 'managerRefreshError']) {
+    const record = normalizeLayerLoading({
+      id: 'militaryInstallations', enabled: true,
+      stats: { status: 'zoom-in', error: 'Zoom in to load mapped sites.', [field]: 'Network unavailable' },
+    });
+    assert.equal(record.error, 'Network unavailable');
+  }
+});

src/tooling/liveProviders.test.mjs

@@ -230,3 +230,126 @@ test('AIS preview route ingests through the socket, returns tracks and disposes
   for (let i = 0; i < 100 && sockets.length < 2; i++) await delay(10);
   assert.equal(sockets.length, 2);
 });
+
+test('military aircraft route serves stale cache on an upstream 429 and cools down for Retry-After', async (t) => {
+  let now = Date.now();
+  let calls = 0;
+  t.mock.method(Date, 'now', () => now);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    if (calls === 1) return Response.json({ ac: [{ hex: 'abc123' }] });
+    return new Response(JSON.stringify({ error: 'rate limited' }), {
+      status: 429,
+      headers: { 'Retry-After': '20' },
+    });
+  });
+  const request = install(providers.adsbLolProxy());
+  const first = await request('/api/adsblol/mil');
+  assert.equal(first.statusCode, 200);
+  now += 13_000;
+  const limited = await request('/api/adsblol/mil');
+  assert.equal(calls, 2);
+  assert.equal(
+    limited.statusCode,
+    200,
+    'a 429 with a cached body is never relayed',
+  );
+  assert.equal(limited.body, first.body);
+  assert.equal(limited.headers['x-ads-b-cache'], 'STALE');
+  assert.equal(limited.headers['x-ads-b-upstream-status'], '429');
+  assert.equal(limited.headers['x-ads-b-cache-age-ms'], '13000');
+  now += 5_000;
+  const cooling = await request('/api/adsblol/mil');
+  assert.equal(calls, 2, 'no upstream call inside the Retry-After window');
+  assert.equal(cooling.headers['x-ads-b-cache'], 'STALE');
+  assert.equal(cooling.headers['x-ads-b-cache-age-ms'], '18000');
+  now += 16_000;
+  await request('/api/adsblol/mil');
+  assert.equal(calls, 3, 'upstream is retried once Retry-After elapses');
+});
+
+test('military aircraft route relays an upstream 429 when nothing is cached', async (t) => {
+  t.mock.method(console, 'warn', () => {});
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return new Response('{"error":"rate limited"}', { status: 429 });
+  });
+  const request = install(providers.adsbLolProxy());
+  const limited = await request('/api/adsblol/mil');
+  assert.equal(limited.statusCode, 429);
+  assert.ok(limited.headers['retry-after']);
+  const again = await request('/api/adsblol/mil');
+  assert.equal(again.statusCode, 429);
+  assert.equal(
+    calls,
+    1,
+    'the cooldown still protects upstream with nothing cached',
+  );
+  assert.ok(
+    again.headers['retry-after'],
+    'a cooling-down miss carries Retry-After',
+  );
+});
+
+test('military fallback cancels a stalled 5xx body and starts cooldown at receipt', async (t) => {
+  let now = 1_800_000_000_000;
+  let calls = 0;
+  let cancelled = false;
+  t.mock.method(Date, 'now', () => now);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(globalThis, 'fetch', async () => {
+    if (++calls === 1) return Response.json({ ac: [] });
+    now += 10000;
+    return new Response(
+      new ReadableStream({
+        cancel() {
+          cancelled = true;
+        },
+      }),
+      {
+        status: 503,
+        headers: { 'Retry-After': new Date(now + 20000).toUTCString() },
+      },
+    );
+  });
+  const request = install(providers.adsbLolProxy());
+  await request('/api/adsblol/mil');
+  now += 1000;
+  const hit = await request('/api/adsblol/mil');
+  assert.equal(hit.headers['x-ads-b-cache-age-ms'], '1000');
+  now += 12000;
+  const fallback = await request('/api/adsblol/mil');
+  assert.equal(fallback.statusCode, 200);
+  assert.equal(cancelled, true);
+  assert.equal(fallback.headers['x-ads-b-cache-age-ms'], '23000');
+  assert.equal(fallback.headers['x-ads-b-retry-after-seconds'], '20');
+  now += 19000;
+  await request('/api/adsblol/mil');
+  assert.equal(calls, 2);
+  now += 2000;
+  await request('/api/adsblol/mil');
+  assert.equal(calls, 3);
+});
+
+test('military cooldown bounds untrusted Retry-After and defaults server errors', async (t) => {
+  t.mock.method(console, 'warn', () => {});
+  for (const [status, raw, seconds] of [
+    [429, '1', 5],
+    [429, '99999', 120],
+    [503, 'invalid', 15],
+  ]) {
+    t.mock.method(
+      globalThis,
+      'fetch',
+      async () =>
+        new Response('{}', {
+          status,
+          headers: { 'Retry-After': raw },
+        }),
+    );
+    const result = await install(providers.adsbLolProxy())('/api/adsblol/mil');
+    assert.equal(Number(result.headers['retry-after']), seconds);
+  }
+});